From pedronis at codespeak.net Thu Nov 1 12:02:36 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:02:36 +0100 (CET) Subject: [pypy-svn] r48231 - in pypy/extradoc/talk: roadshow-google roadshow-ibm Message-ID: <20071101110236.D04A681AB@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:02:36 2007 New Revision: 48231 Added: pypy/extradoc/talk/roadshow-google/ pypy/extradoc/talk/roadshow-ibm/ pypy/extradoc/talk/roadshow-ibm/talk.txt - copied, changed from r48230, pypy/extradoc/talk/ep2007/pypy_10_and_jit/talk.txt Log: directories for our two main talks on the US trip From pedronis at codespeak.net Thu Nov 1 12:16:19 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:16:19 +0100 (CET) Subject: [pypy-svn] r48232 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101111619.980B581AD@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:16:19 2007 New Revision: 48232 Added: pypy/extradoc/talk/roadshow-ibm/graph0.dot pypy/extradoc/talk/roadshow-ibm/graph1.dot pypy/extradoc/talk/roadshow-ibm/overview2.png - copied unchanged from r48230, pypy/extradoc/talk/ep2007/pypy_10_and_jit/overview2.png Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: our slides about the jit arch Added: pypy/extradoc/talk/roadshow-ibm/graph0.dot ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/graph0.dot Thu Nov 1 12:16:19 2007 @@ -0,0 +1,19 @@ +digraph _generated__graph { +subgraph _generated__ { +_generated__ [shape="box", label="generated", color="black", fillcolor="#a5e6f0", style="filled", width="0.75"]; +edge [label="startblock", style="dashed", color="black", dir="forward", weight="5"]; +_generated__ -> _generated____1 +_generated____1 [shape="box", label="generated__1\ninputargs: v6024\n\nv6025 = cast_pointer(v6024)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6025", style="solid", color="black", dir="forward", weight="5"]; +_generated____1 -> _generated____2 +_generated____2 [shape="box", label="generated__2\ninputargs: v6026\n\nv6027 = getfield(v6026, ('typeptr'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6026 v6027", style="solid", color="black", dir="forward", weight="5"]; +_generated____2 -> _generated____3 +_generated____3 [shape="box", label="generated__3\ninputargs: v6030 v6028\n\nv6029 = cast_ptr_to_int(v6028)\l\lexitswitch: v6029", color="black", fillcolor="#e5ffff", style="filled", width="0.75"]; +edge [label="'default': v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____4 +_generated____4 [shape="box", label="generated__4\ninputargs: v6039 v6035\n\nv6031 = getfield((<* struct ExcData { ex...=... }>), ('exc_type'))\lv6032 = getfield((<* struct ExcData { ex...=... }>), ('exc_value'))\lv6033 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), (<* None>))\lv6034 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), (<* None>))\lv6036 = direct_call((<* fn ll_continue_comp...blePtr>), (<* struct object { typ...=... }>), v6035)\lv6037 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), v6031)\lv6038 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), v6032)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6039 v6035", style="solid", color="black", dir="forward", weight="5"]; +_generated____4 -> _generated____3 +} +} \ No newline at end of file Added: pypy/extradoc/talk/roadshow-ibm/graph1.dot ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/graph1.dot Thu Nov 1 12:16:19 2007 @@ -0,0 +1,90 @@ +digraph _generated__graph { +subgraph _generated__ { +_generated__ [shape="box", label="generated", color="black", fillcolor="#a5e6f0", style="filled", width="0.75"]; +edge [label="startblock", style="dashed", color="black", dir="forward", weight="5"]; +_generated__ -> _generated____1 +_generated____1 [shape="box", label="generated__1\ninputargs: v6024\n\nv6025 = cast_pointer(v6024)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6025", style="solid", color="black", dir="forward", weight="5"]; +_generated____1 -> _generated____2 +_generated____2 [shape="box", label="generated__2\ninputargs: v6026\n\nv6027 = getfield(v6026, ('typeptr'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6026 v6027", style="solid", color="black", dir="forward", weight="5"]; +_generated____2 -> _generated____3 +_generated____3 [shape="octagon", label="generated__3\ninputargs: v6030 v6028\n\nv6029 = cast_ptr_to_int(v6028)\l\lexitswitch: v6029", color="red", fillcolor="#e5ffff", style="filled", width="0.75"]; +edge [label="24647528: v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____4 +edge [label="'default': v6030 v6028", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____5 +_generated____4 [shape="box", label="generated__4\ninputargs: v10217 v10220\n\nv10218 = cast_pointer(v10217)\lv10219 = getfield(v10218, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10219 v10217", style="solid", color="black", dir="forward", weight="5"]; +_generated____4 -> _generated____6 +_generated____6 [shape="box", label="generated__6\ninputargs: v10221 v10223\n\nv10222 = int_lt((1), v10221)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10222 v10223", style="solid", color="black", dir="forward", weight="5"]; +_generated____6 -> _generated____7 +_generated____7 [shape="octagon", label="generated__7\ninputargs: v10224 v10225\n\n\lexitswitch: v10224", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: (1)", style="dotted", color="red", dir="forward", weight="5"]; +_generated____7 -> _generated____8 +edge [label="True: v10225", style="dotted", color="red", dir="forward", weight="5"]; +_generated____7 -> _generated____9 +_generated____8 [shape="box", label="generated__8\ninputargs: v10230\n\nv10226 = malloc((GcStruct pypy.jit.tl.tlc.IntObj))\lv10227 = cast_pointer(v10226)\lv10228 = setfield(v10227, ('typeptr'), (<* struct object_vtabl...=... }>))\lv10229 = cast_pointer(v10227)\lv10231 = setfield(v10229, ('inst_value'), v10230)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10227", style="solid", color="black", dir="forward", weight="5"]; +_generated____8 -> _generated____10 +_generated____10 [shape="box", label="generated__10\ninputargs: v10232\n\nv10233 = cast_pointer(v10232)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10233", style="solid", color="black", dir="forward", weight="5"]; +_generated____10 -> _generated____11 +_generated____11 [shape="box", label="generated__11\ninputargs: v10234\n\nreturn v10234", color="black", fillcolor="green", style="filled", width="0.75"]; +_generated____9 [shape="box", label="generated__9\ninputargs: v10236\n\nv10235 = same_as((0))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10236 v10235", style="solid", color="black", dir="forward", weight="5"]; +_generated____9 -> _generated____12 +_generated____12 [shape="box", label="generated__12\ninputargs: v10239 v10237\n\nv10238 = int_is_true(v10237)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10238 v10239", style="solid", color="black", dir="forward", weight="5"]; +_generated____12 -> _generated____13 +_generated____13 [shape="octagon", label="generated__13\ninputargs: v10240 v10241\n\n\lexitswitch: v10240", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10241", style="dotted", color="red", dir="forward", weight="5"]; +_generated____13 -> _generated____14 +edge [label="True: v10241", style="dotted", color="red", dir="forward", weight="5"]; +_generated____13 -> _generated____15 +_generated____14 [shape="box", label="generated__14\ninputargs: v10242\n\nv10243 = cast_pointer(v10242)\lv10244 = getfield(v10243, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10244 v10242", style="solid", color="black", dir="forward", weight="5"]; +_generated____14 -> _generated____16 +_generated____16 [shape="box", label="generated__16\ninputargs: v10245 v10247\n\nv10246 = int_mul((1), v10245)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10247 v10246", style="solid", color="black", dir="forward", weight="5"]; +_generated____16 -> _generated____17 +_generated____17 [shape="box", label="generated__17\ninputargs: v10248 v10251\n\nv10249 = cast_pointer(v10248)\lv10250 = getfield(v10249, ('inst_value'))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10250 v10251 v10248", style="solid", color="black", dir="forward", weight="5"]; +_generated____17 -> _generated____18 +_generated____18 [shape="box", label="generated__18\ninputargs: v10252 v10254 v10255\n\nv10253 = int_sub(v10252, (1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10253 v10254 v10255", style="solid", color="black", dir="forward", weight="5"]; +_generated____18 -> _generated____19 +_generated____19 [shape="box", label="generated__19\ninputargs: v10256 v10258 v10259\n\nv10257 = int_lt((1), v10256)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10257 v10258 v10256 v10259", style="solid", color="black", dir="forward", weight="5"]; +_generated____19 -> _generated____20 +_generated____20 [shape="octagon", label="generated__20\ninputargs: v10260 v10261 v10262 v10263\n\n\lexitswitch: v10260", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10261", style="dotted", color="red", dir="forward", weight="5"]; +_generated____20 -> _generated____8 +edge [label="True: v10263 v10261 v10262", style="dotted", color="red", dir="forward", weight="5"]; +_generated____20 -> _generated____21 +_generated____21 [shape="box", label="generated__21\ninputargs: v10265 v10266 v10267\n\nv10264 = same_as((0))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10265 v10266 v10267 v10264", style="solid", color="black", dir="forward", weight="5"]; +_generated____21 -> _generated____22 +_generated____22 [shape="box", label="generated__22\ninputargs: v10270 v10271 v10272 v10268\n\nv10269 = int_is_true(v10268)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10269 v10271 v10272 v10270", style="solid", color="black", dir="forward", weight="5"]; +_generated____22 -> _generated____23 +_generated____23 [shape="octagon", label="generated__23\ninputargs: v10273 v10274 v10275 v10276\n\n\lexitswitch: v10273", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v10275 v10274 v10276", style="dotted", color="red", dir="forward", weight="5"]; +_generated____23 -> _generated____24 +edge [label="True: v10274", style="dotted", color="red", dir="forward", weight="5"]; +_generated____23 -> _generated____8 +_generated____24 [shape="box", label="generated__24\ninputargs: v10278 v10277 v10280\n\nv10279 = int_mul(v10277, v10278)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10278 v10279 v10280", style="solid", color="black", dir="forward", weight="5"]; +_generated____24 -> _generated____25 +_generated____25 [shape="box", label="generated__25\ninputargs: v10281 v10283 v10284\n\nv10282 = int_sub(v10281, (1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10282 v10283 v10284", style="solid", color="black", dir="forward", weight="5"]; +_generated____25 -> _generated____19 +_generated____15 [shape="box", label="generated__15\ninputargs: v10286\n\nv10285 = same_as((1))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v10285", style="solid", color="black", dir="forward", weight="5"]; +_generated____15 -> _generated____8 +_generated____5 [shape="box", label="generated__5\ninputargs: v6039 v6035\n\nv6031 = getfield((<* struct ExcData { ex...=... }>), ('exc_type'))\lv6032 = getfield((<* struct ExcData { ex...=... }>), ('exc_value'))\lv6033 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), (<* None>))\lv6034 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), (<* None>))\lv6036 = direct_call((<* fn ll_continue_comp...blePtr>), (<* struct object { typ...=... }>), v6035)\lv6037 = setfield((<* struct ExcData { ex...=... }>), ('exc_type'), v6031)\lv6038 = setfield((<* struct ExcData { ex...=... }>), ('exc_value'), v6032)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v6039 v6035", style="solid", color="black", dir="forward", weight="5"]; +_generated____5 -> _generated____3 +} +} \ No newline at end of file Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 12:16:19 2007 @@ -105,6 +105,8 @@ pypy-c-jit ====================== +.. where to put this? + PyPy 1.0 contains both the dynamic compiler generator and the start of its application to PyPy's Python intepreter. @@ -180,8 +182,6 @@ .. need to save state in a compact form: paths -.. demo? - Promotion (example) ======================== @@ -251,4 +251,32 @@ PyPy proves this a viable approach worth of further exploration. +.. EXTRA MATERIAL: +.. backend material: not for the general talk +.. virtualizables: not for general talk +.. state of gc framework + +Open Issues +============== + +- inlining control + +- promotion switch explosion fallbacks + +- jit only the hot-spots + +- more hints needed in PyPy's Python + +Virtualizable Frames +====================== +- frames need to live in the heap + (tracebacks ...) and be introspectable +- jit code wants local variables to live in registers + and on the stack +- => mark the frame class as "virtualizable" +- jit code uses lazy allocation and stores + some contents (local variables...) in register + and stack +- outside world access gets intercepted + to be able to force lazy virtual data into the heap From pedronis at codespeak.net Thu Nov 1 12:54:05 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 12:54:05 +0100 (CET) Subject: [pypy-svn] r48233 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101115405.3398781B7@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 12:54:04 2007 New Revision: 48233 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: WIP: translation aspects slides Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 12:54:04 2007 @@ -1,5 +1,67 @@ .. include:: + +Translation aspects +======================== + +- The interpreters that we write should be free of low-level details + (this is also required to target platform as different as Posix/C + and the JVM/.NET) + +- Advanced features related to execution should not need wide-spread + changes to the interpreters + +- The interpreters instead should use support offered and inserted by + the framework + +Examples: gc and memory management, stack inspection and manipulation + +Implementation +================== + +- Translation aspects are implemented as transformation of low-level + graphs + +- Calls to library/helper code written in RPython can be inserted + too which will also be analyzed and translated + +GC Framework +=============== + +- RPython has been extended with allocation and adress manipulation + primitives that can be used to express GC in RPython directly + +- GCs are linked by substituting memory allocation operations with calls + into them + +- Right now bookeeping code to keep track of reference counting or + roots is inserted by the GC framework + +- Inlining is used to eliminate call overhead for the fast paths of + allocation + +.. MMTk reference + +Stackless transformation +========================= + +- One translation aspect transformation insertes support code across + around calls such that the stack can be unwind and function asked to + store and reflect their current activation frame state to the heap + +- Chains of saved activation state can then be resumed + +- We have implemented coroutine switching using this + +Resume points +=============== + +- Further this transformation can insert code tha allows to construct + chains of activation states corresponding to labeled points in the + program, we use this to construct resumable low-level activation + chains from information captured as serialized language level frame + chains. + Flexibility vs. Performance ============================= From rxe at codespeak.net Thu Nov 1 13:54:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 1 Nov 2007 13:54:03 +0100 (CET) Subject: [pypy-svn] r48234 - pypy/dist/pypy/translator/llvm Message-ID: <20071101125403.3DB8A814F@code0.codespeak.net> Author: rxe Date: Thu Nov 1 13:54:01 2007 New Revision: 48234 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py Log: make the whole way we make entry names and filenames saner and fix issue with names like propgating through to filenames Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Thu Nov 1 13:54:01 2007 @@ -131,7 +131,7 @@ self.lastdir = py.path.local() self.dirpath.chdir() - return llvmfile.purebasename + return self.genllvm.entry_name def make_module(self): base = self.setup() Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Nov 1 13:54:01 2007 @@ -177,7 +177,18 @@ ptr = getfunctionptr(bk.getdesc(func).getuniquegraph()) c = inputconst(lltype.typeOf(ptr), ptr) self.db.prepare_arg_value(c) - self.entry_func_name = func.func_name + + # ensure unqiue entry node name for testing + entry_node = self.db.obj2node[c.value._obj] + name = entry_node.name + if name in self.function_count: + self.function_count[name] += 1 + name += '_%d' % self.function_count[name] + entry_node.name = name + else: + self.function_count[name] = 1 + + self.entry_name = name[6:] return c.value._obj def generate_ll_externs(self): @@ -201,13 +212,7 @@ def create_codewriter(self): # prevent running the same function twice in a test - if self.entry_func_name in self.function_count: - postfix = '_%d' % self.function_count[self.entry_func_name] - self.function_count[self.entry_func_name] += 1 - else: - postfix = '' - self.function_count[self.entry_func_name] = 1 - filename = udir.join(self.entry_func_name + postfix).new(ext='.ll') + filename = udir.join(self.entry_name).new(ext='.ll') f = open(str(filename), 'w') return CodeWriter(f, self.db), filename Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 1 13:54:01 2007 @@ -132,7 +132,7 @@ """ epilog = """ -__entrypoint__ = _c.__entrypoint__%(name)s +__entrypoint__ = _c.__entrypoint__pypy_%(name)s # %(RT)r to_llargs = %(to_llargs)s @@ -157,7 +157,7 @@ def __init__(self, genllvm, dllname): self.genllvm = genllvm self.dllname = dllname - basename = self.genllvm.filename.purebasename + '_wrapper.py' + basename = self.genllvm.entry_name + '_wrapper.py' self.modfilename = genllvm.filename.new(basename=basename) self.count = 0 @@ -166,7 +166,7 @@ self.file.write(self.prolog % self.dllname) g = self.genllvm.entrynode.graph - name = "pypy_" + g.name + name = self.genllvm.entry_name ARGS = [a.concretetype for a in g.startblock.inputargs] inputargtypes, to_llargs = self.build_args_to_ctypes_to_lltype(ARGS) From antocuni at codespeak.net Thu Nov 1 13:55:44 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 1 Nov 2007 13:55:44 +0100 (CET) Subject: [pypy-svn] r48235 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101125544.630678172@code0.codespeak.net> Author: antocuni Date: Thu Nov 1 13:55:43 2007 New Revision: 48235 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: I guess this should really be y*y. Without the extra ``y``, it shows up as 9+y in firefox. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 13:55:43 2007 @@ -259,7 +259,7 @@ | | | ``def f_(x, y):`` | | | | ``switch x:`` | | | | *case 3:* | -| | | *return 9 + y* | +| | | *return 9 + y*y | | | | ``default:`` | | | | ``compile_more(value=x)`` | | | | | From cfbolz at codespeak.net Thu Nov 1 14:01:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Nov 2007 14:01:02 +0100 (CET) Subject: [pypy-svn] r48236 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101130102.3542281B5@code0.codespeak.net> Author: cfbolz Date: Thu Nov 1 14:01:01 2007 New Revision: 48236 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: some fixes, XXXs Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:01:01 2007 @@ -12,9 +12,9 @@ changes to the interpreters - The interpreters instead should use support offered and inserted by - the framework + the translation framework -Examples: gc and memory management, stack inspection and manipulation +Examples: GC and memory management, stack inspection and manipulation Implementation ================== @@ -23,32 +23,34 @@ graphs - Calls to library/helper code written in RPython can be inserted - too which will also be analyzed and translated + too which will also be analyzed and translated GC Framework =============== -- RPython has been extended with allocation and adress manipulation +- RPython has been extended with allocation and address manipulation primitives that can be used to express GC in RPython directly - GCs are linked by substituting memory allocation operations with calls into them -- Right now bookeeping code to keep track of reference counting or +- Right now bookkeeping code to keep track of reference counting or roots is inserted by the GC framework - Inlining is used to eliminate call overhead for the fast paths of - allocation + allocation (XXX not only of allocation, of the write barrier and the root bookkeeping too. do you want to mention that?) .. MMTk reference Stackless transformation ========================= -- One translation aspect transformation insertes support code across +- One translation aspect transformation inserts support code across around calls such that the stack can be unwind and function asked to store and reflect their current activation frame state to the heap +XXX mention that this works regardless of the backend or something? + - Chains of saved activation state can then be resumed - We have implemented coroutine switching using this @@ -56,11 +58,14 @@ Resume points =============== -- Further this transformation can insert code tha allows to construct - chains of activation states corresponding to labeled points in the - program, we use this to construct resumable low-level activation + +- this transformation can also insert code that allows to construct + artificial chains of activation states corresponding to labeled points in the + program + +- we use this to construct resumable low-level activation chains from information captured as serialized language level frame - chains. + chains. XXX a bit hard to understand Flexibility vs. Performance ============================= From cfbolz at codespeak.net Thu Nov 1 14:03:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Nov 2007 14:03:35 +0100 (CET) Subject: [pypy-svn] r48237 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101130335.C29A4814F@code0.codespeak.net> Author: cfbolz Date: Thu Nov 1 14:03:35 2007 New Revision: 48237 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: I think this is what was intended Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:03:35 2007 @@ -264,7 +264,7 @@ | | | ``def f_(x, y):`` | | | | ``switch x:`` | | | | *case 3:* | -| | | *return 9 + y*y | +| | | *return 9 + y*y* | | | | ``default:`` | | | | ``compile_more(value=x)`` | | | | | From rxe at codespeak.net Thu Nov 1 14:39:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 1 Nov 2007 14:39:56 +0100 (CET) Subject: [pypy-svn] r48238 - pypy/dist/pypy/translator/llvm/module Message-ID: <20071101133956.B011C81A9@code0.codespeak.net> Author: rxe Date: Thu Nov 1 14:39:55 2007 New Revision: 48238 Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py pypy/dist/pypy/translator/llvm/module/support.py Log: well, well... so it is seems that we have been running two independent exception strategies in llvm - one for the backend and the tests, and the other one done by transformation. so upshot is none of the externs could raise a real error and our tests never raised an error. ive know idea how long this has been this way - probably a veeery long time. (and it is probably my fault :-) Modified: pypy/dist/pypy/translator/llvm/module/excsupport.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/excsupport.py (original) +++ pypy/dist/pypy/translator/llvm/module/excsupport.py Thu Nov 1 14:39:55 2007 @@ -1,7 +1,7 @@ entrycode = ''' ccc %(returntype)s %%__entrypoint__%(entrypointname)s { - store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + call fastcc void %%pypy_rpyexc_clear() %%result = call %(cconv)s %(returntype)s%%%(entrypointname)s ret %(returntype)s %%result } @@ -9,18 +9,15 @@ voidentrycode = ''' ccc %(returntype)s %%__entrypoint__%(entrypointname)s { - store %%RPYTHON_EXCEPTION_VTABLE* null, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type + call fastcc void %%pypy_rpyexc_clear() call %(cconv)s %(returntype)s%%%(entrypointname)s ret void } ''' raisedcode = ''' -;XXX this should use the transformation data that has the same purpose -ccc int %%__entrypoint__raised_LLVMException() { - %%tmp = load %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - %%result = cast %%RPYTHON_EXCEPTION_VTABLE* %%tmp to int - ret int %%result +ccc bool %%__entrypoint__raised_LLVMException() { + %%result = call fastcc bool %%pypy_rpyexc_occured() + ret bool %%result } - ''' Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Thu Nov 1 14:39:55 2007 @@ -1,8 +1,4 @@ - extdeclarations = """ -%last_exception_type = internal global %RPYTHON_EXCEPTION_VTABLE* null -%last_exception_value = internal global %RPYTHON_EXCEPTION* null - declare ccc uint %strlen(sbyte*) declare ccc void %llvm.memsetPOSTFIX(sbyte*, ubyte, UWORD, UWORD) declare ccc void %llvm.memcpyPOSTFIX(sbyte*, sbyte*, UWORD, UWORD) @@ -21,13 +17,6 @@ ret WORD %size } -internal fastcc int %RPyExceptionOccurred() { - %tmp.0 = load %RPYTHON_EXCEPTION_VTABLE** %last_exception_type - %bool_res = setne %RPYTHON_EXCEPTION_VTABLE* %tmp.0, null - %res = cast bool %bool_res to int - ret int %res -} - internal fastcc %RPyString* %RPyString_FromString(sbyte* %s) { %lenu = call ccc uint %strlen(sbyte* %s) %lenuword = cast uint %lenu to UWORD @@ -79,25 +68,9 @@ """ -from sys import maxint -if maxint != 2**31-1: - extfunctions += """ -internal fastcc void %pypy_ll_raise_OSError__Signed(int %errno_0) { - %tmp = cast int %errno_0 to long - call fastcc void %pypy_ll_raise_OSError__Signed(long %tmp) - ret void -} - -internal fastcc void %pypy__RPyListOfString_SetItem__listPtr_Signed_rpy_stringPtr(%RPyListOfString* %l_1, int %index_0, %RPyString* %newstring_0) { - %index_0_long = cast int %index_0 to long - call fastcc void %pypy__RPyListOfString_SetItem__listPtr_Signed_rpy_stringPtr(%RPyListOfString* %l_1, long %index_0_long, %RPyString* %newstring_0) - ret void -} - -""" - extfunctions_standalone = """ """ +from sys import maxint if maxint != 2**31-1: extfunctions_standalone += """ internal fastcc int %pypy_entry_point(%RPyListOfString* %argv) { @@ -108,15 +81,12 @@ """ + def write_raise_exc(c_name, exc_repr, codewriter): l = """ internal fastcc void %%raise%s(sbyte* %%msg) { - %%exception_value = cast %s to %%RPYTHON_EXCEPTION* - %%tmp = getelementptr %%RPYTHON_EXCEPTION* %%exception_value, int 0, uint 0 - %%exception_type = load %%RPYTHON_EXCEPTION_VTABLE** %%tmp - store %%RPYTHON_EXCEPTION_VTABLE* %%exception_type, %%RPYTHON_EXCEPTION_VTABLE** %%last_exception_type - store %%RPYTHON_EXCEPTION* %%exception_value, %%RPYTHON_EXCEPTION** %%last_exception_value + ;%%exception_value = cast %s to %%RPYTHON_EXCEPTION* ret void } """ % (c_name, exc_repr) From pedronis at codespeak.net Thu Nov 1 14:44:56 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 14:44:56 +0100 (CET) Subject: [pypy-svn] r48239 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101134456.EDF4281BD@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 14:44:56 2007 New Revision: 48239 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: fixes, shuffling stuff around Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:44:56 2007 @@ -38,35 +38,21 @@ roots is inserted by the GC framework - Inlining is used to eliminate call overhead for the fast paths of - allocation (XXX not only of allocation, of the write barrier and the root bookkeeping too. do you want to mention that?) + allocation and barriers .. MMTk reference Stackless transformation ========================= -- One translation aspect transformation inserts support code across - around calls such that the stack can be unwind and function asked to +- One translation aspect transformation inserts support code + around calls such that the stack can be unwound and functions asked to store and reflect their current activation frame state to the heap -XXX mention that this works regardless of the backend or something? - - Chains of saved activation state can then be resumed - We have implemented coroutine switching using this -Resume points -=============== - - -- this transformation can also insert code that allows to construct - artificial chains of activation states corresponding to labeled points in the - program - -- we use this to construct resumable low-level activation - chains from information captured as serialized language level frame - chains. XXX a bit hard to understand - Flexibility vs. Performance ============================= @@ -318,7 +304,9 @@ PyPy proves this a viable approach worth of further exploration. -.. EXTRA MATERIAL: +EXTRA MATERIAL +================== + .. backend material: not for the general talk .. virtualizables: not for general talk .. state of gc framework @@ -347,3 +335,14 @@ and stack - outside world access gets intercepted to be able to force lazy virtual data into the heap + + +Resume points +=============== + +- this transformation can also insert code that allows to construct + artificial chains of activation states corresponding to labeled points in the + program + +- we use this to support resuming serialized language-level coroutines + From pedronis at codespeak.net Thu Nov 1 14:48:23 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 14:48:23 +0100 (CET) Subject: [pypy-svn] r48240 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101134823.546AB81D7@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 14:48:22 2007 New Revision: 48240 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: connecting title Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 14:48:22 2007 @@ -53,8 +53,10 @@ - We have implemented coroutine switching using this -Flexibility vs. Performance -============================= +A special aspect: JIT generation +================================== + +Flexibility vs. Performance: * Interpreters are easy to write and evolve From arigo at codespeak.net Thu Nov 1 16:29:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Nov 2007 16:29:54 +0100 (CET) Subject: [pypy-svn] r48242 - pypy/extradoc/talk/roadshow-ibm/ui Message-ID: <20071101152954.BDC65816E@code0.codespeak.net> Author: arigo Date: Thu Nov 1 16:29:53 2007 New Revision: 48242 Added: pypy/extradoc/talk/roadshow-ibm/ui/ - copied from r48241, pypy/extradoc/talk/ep2007/ui/ Log: Add this directory for the rst2s5 HTML to display properly. From arigo at codespeak.net Thu Nov 1 16:44:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Nov 2007 16:44:52 +0100 (CET) Subject: [pypy-svn] r48243 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101154452.B6DF98184@code0.codespeak.net> Author: arigo Date: Thu Nov 1 16:44:52 2007 New Revision: 48243 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Details. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 16:44:52 2007 @@ -5,7 +5,7 @@ ======================== - The interpreters that we write should be free of low-level details - (this is also required to target platform as different as Posix/C + (this is also required to target platforms as different as Posix/C and the JVM/.NET) - Advanced features related to execution should not need wide-spread @@ -60,7 +60,7 @@ * Interpreters are easy to write and evolve -* For high performance dynamic compilation is required +* For high performance, dynamic compilation is required Traditional JIT compilers =============================== From pedronis at codespeak.net Thu Nov 1 19:24:07 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 19:24:07 +0100 (CET) Subject: [pypy-svn] r48244 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101182407.BFE528178@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 19:24:06 2007 New Revision: 48244 Added: pypy/extradoc/talk/roadshow-ibm/abstract.txt (contents, props changed) Log: the abstract we sent ibm Added: pypy/extradoc/talk/roadshow-ibm/abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-ibm/abstract.txt Thu Nov 1 19:24:06 2007 @@ -0,0 +1,24 @@ +PyPy - automatic generation of VMs for dynamic languages - JIT included +------------------------------------------------------------------------ + +PyPy is a framework written in Python for generating virtual machines +for dynamic languages. The VMs are flexibly produced from a +high-level "specification" in the form of a simple interpreter for the +dynamic language. The interpreters are written in a high-level static +subset of Python. + +Low-level details like memory allocation and object layout and stack +inspection are not encoded manually, but inserted by the VM generation process. +This allows us to produce VMs that run within a wide range of execution +environments (from C-like to JVM/.NET). + +The framework has the ability to automatically generate a dynamic compiler +from the interpreter too. A pragmatic application of partial evaluation +techniques guided by a few hints is used for the task. + +Crucial for the effectiveness of dynamic compilation is +the use of run-time information to improve compilation results: in our +approach, a powerful primitive called "promotion" that +"promotes" run-time values to compile-time is used to that effect. + + From lac at codespeak.net Thu Nov 1 19:26:02 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Thu, 1 Nov 2007 19:26:02 +0100 (CET) Subject: [pypy-svn] r48245 - pypy/extradoc/talk/roadshow-google Message-ID: <20071101182602.C6C4C8185@code0.codespeak.net> Author: lac Date: Thu Nov 1 19:26:02 2007 New Revision: 48245 Added: pypy/extradoc/talk/roadshow-google/abstract.txt (contents, props changed) Log: This is the abstract we sent to Google Added: pypy/extradoc/talk/roadshow-google/abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-google/abstract.txt Thu Nov 1 19:26:02 2007 @@ -0,0 +1,35 @@ +In Search of Speed and Flexibility. PyPy and the Art of Generating Virtual Machines +------------------------------------------------------------------------------------ +We all want our favourite dynamic language to be faster, to be even more +dynamic and to pick up nifty ideas from its peers. + +But this is so much work! + +There ought to be a better way to implement dynamic languages that +enables growth and necessary change with less effort and more ease. +Writing yet another interpreter in C or even Java is not it. + +PyPy is a framework to implement dynamic languages that embraces this +challenge. It allows us to write such languages in the form of simple +interpreters, expressed in a rich subset of Python. This means that +the resulting implementations can can be evolved with reasonable +effort. + +PyPy then generates a variety of virtual machines (VMs) from a single +source. We can therefore target a wide range of environments, +including C/Posix, Java, or .NET. This avoids the major source of +fragmentation within a dynamic language community -- the need for a +separate language implementation for each virtual machine. It also +means that languages as distinct as Python, Prolog and Smalltalk can +share the bulk of our compiler machinery. + +The flexible toolchain which is necessary for virtual machine +generation turns out to be good for other things as well. Traditional +implementations for dynamic languages have to make some very hard +decisions early, which then become entrenched in the whole codebase, +making it difficult or impossible to change later. It can be nice to +experiment with different garbage collectors, but not if you have to +write a complete new implementation of your favourite language for +each one. With PyPy you don't have to. You can also generate and +tailor a JIT-compiler for the language at hand, putting to rest the +notion that flexibility must come at the expense of speed. \ No newline at end of file From pedronis at codespeak.net Thu Nov 1 19:28:05 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Nov 2007 19:28:05 +0100 (CET) Subject: [pypy-svn] r48246 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071101182805.EAF868185@code0.codespeak.net> Author: pedronis Date: Thu Nov 1 19:28:05 2007 New Revision: 48246 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: checkin the title Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Thu Nov 1 19:28:05 2007 @@ -1,5 +1,8 @@ .. include:: +======================================================================== +PyPy - automatic generation of VMs for dynamic languages - JIT included +======================================================================== Translation aspects ======================== From tismer at codespeak.net Fri Nov 2 06:08:01 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Fri, 2 Nov 2007 06:08:01 +0100 (CET) Subject: [pypy-svn] r48247 - pypy/dist/pypy/translator/goal Message-ID: <20071102050801.9F33F81A2@code0.codespeak.net> Author: tismer Date: Fri Nov 2 06:08:00 2007 New Revision: 48247 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: this is still not completely correct, but way better than before. Nobody will believe that I hacked a week on this, already, because it cannot be debugged without compilation... Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Fri Nov 2 06:08:00 2007 @@ -16,14 +16,14 @@ --info print translation information about this PyPy executable """ -import sys, os +import sys DEBUG = False # dump exceptions before calling the except hook originalexcepthook = sys.__excepthook__ def run_toplevel(f, *fargs, **fkwds): - """Calls f() and handle all OperationErrors. + """Calls f() and handles all OperationErrors. Intended use is to run the main program or one interactive statement. run_protected() handles details like forwarding exceptions to sys.excepthook(), catching SystemExit, printing a newline after @@ -144,16 +144,120 @@ # ____________________________________________________________ # Main entry point +# faking os, until we really are able to import it from source +""" +Why fake_os ? +------------- + +When pypy is starting, the builtin modules are there, but os.path +is a Python module. The current startup code depends on os.py, +which had the side effect that os.py got frozen into the binary. + +Os.py is a wrapper for system specific built-in modules and tries +to unify the interface. One problem is os.environ which looks +very differently per os. +Due to the way os.py initializes its environ variable on Windows, +it s hard to get a correct os.getenv that uses the actual environment +variables. In order to fix that without modifying os.py, we need to +re-import it several times, before and after compilation. + +When compiling the source, we first create a fake_os class instance. +fake_os contains all of os, when compiling.reduction seems to be not +appropriate, since lots of other modules are sucked in, at the same time. +A complete reduction is possible, but would require much more work. + +In the course of creating fake_os, we import the real os. We keep a record +of which modules were imported, before. + +During start of the entry_point, we call os.setup(). +This repeats an import of the specific parts in os.name. +Depending of the underlying os, this might or might not +cause a redefinition of certain os variables, but this happens +exactly like in os.py's initialization. +We then capture the variable environ, which may or may not come from the +specific os. +After that, we re-initialize our dict to what is expected in standard +os. The effect of all this magic is that we have captured the environ +cariable, again, and we now can redefine os.getenv to use this fresh +variable, being accurate and not frozen. + +At this point, we finally can compute the location of our import. + +As a side effect, the involved modules are re-imported, although they had +been compiled in, so PyPy behaves really conformant, making all .py +modules changeable, again. +""" + +from pypy.rlib.objectmodel import we_are_translated + +class fake_os: + def __init__(self): + import sys + self.pre_import = sys.modules.keys() + import os + self.os = os + # make ourselves a clone of os + self.__dict__.update(self.os.__dict__) + + def setup(self): + # we now repeat the os-specific initialization, which + # must be done before importing os, since os hides + # variables after initialization. + specifics = __import__(self.os.name) + self.__dict__.update(specifics.__dict__) + # depending on the os, we now might or might not have + # a new environ variable. However, we can now + # repeat the environ initialisation from os.py + environ = self.os._Environ(self.environ) + # to be safe, reset our dict to be like os + self.__dict__.update(self.os.__dict__) + # but now we insert the fresh environ + self.environ = environ + del self.getenv + # use our method, instead of the os's one's + assert self.getenv + + def teardown(self): + for mod in sys.modules.keys(): + if mod not in self.pre_import: + del sys.modules[mod] + global os + import os + + def getenv(self, key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default.""" + return self.environ.get(key, default) + +os = fake_os() + AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' +# hack to determine if we are on windows. Maybe there is a better way... +DRIVE_LETTER_SEP = '' +try: + here = os.getcwd() + try: + os.chdir('C:') + DRIVE_LETTER_SEP = ':' + finally: + os.chdir(here) +except os.OSError: + DRIVE_LETTER_SEP = None +IS_WINDOWS = DRIVE_LETTER_SEP is not None +# XXX extend here + def entry_point(executable, argv): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH - if os.sep not in executable: + os.setup() # this is the faked one + if os.sep not in executable and DRIVE_LETTER_SEP not in executable: path = os.getenv('PATH') if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) + if we_are_translated and IS_WINDOWS and not fn.lower().endswith('.exe'): + fn += '.exe' if os.path.isfile(fn): executable = fn break @@ -177,7 +281,9 @@ continue sys.path = newpath # found! break - + + os.teardown() # from now on this is the real one + go_interactive = False run_command = False import_site = True From santagada at codespeak.net Fri Nov 2 06:37:13 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 2 Nov 2007 06:37:13 +0100 (CET) Subject: [pypy-svn] r48248 - in pypy/dist/pypy: rpython/raisingops translator/llvm/test Message-ID: <20071102053713.DDE7D80F9@code0.codespeak.net> Author: santagada Date: Fri Nov 2 06:37:12 2007 New Revision: 48248 Added: pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py pypy/dist/pypy/translator/llvm/test/test_exception.py Log: Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py ============================================================================== --- pypy/dist/pypy/rpython/raisingops/raisingops.py (original) +++ pypy/dist/pypy/rpython/raisingops/raisingops.py Fri Nov 2 06:37:12 2007 @@ -1,7 +1,8 @@ import sys from pypy.rlib.rarithmetic import r_longlong, r_uint, intmask from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.lltypesystem.lltype import Signed +from pypy.rpython.lltypesystem.lltype import Signed, SignedLongLong, \ + UnsignedLongLong #XXX original SIGNED_RIGHT_SHIFT_ZERO_FILLS not taken into account #XXX assuming HAVE_LONG_LONG (int_mul_ovf) @@ -33,6 +34,27 @@ else: raise ZeroDivisionError("unsigned integer division") +def llong_floordiv_zer(x, y): + '''#define OP_LLONG_FLOORDIV_ZER(x,y,r) \ + if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("integer division") + ''' + if y: + return llop.llong_floordiv(SignedLongLong, x, y) + else: + raise ZeroDivisionError("integer division") + +def ullong_floordiv_zer(x, y): + '''#define OP_ULLONG_FLOORDIV_ZER(x,y,r) \ + if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("unsigned integer division") + ''' + if y: + return llop.llong_floordiv(UnsignedLongLong, x, y) + else: + raise ZeroDivisionError("unsigned integer division") + + def int_neg_ovf(x): if x == LONG_MIN: raise OverflowError("integer negate") @@ -199,6 +221,16 @@ else: raise ZeroDivisionError("integer modulo") +def llong_mod_zer(x, y): + '''#define OP_LLONG_MOD_ZER(x,y,r) \ + if ((y)) { OP_LLONG_MOD(x,y,r); } \ + else FAIL_ZER("integer modulo") + ''' + if y: + return llop.int_mod(SignedLongLong, x, y) + else: + raise ZeroDivisionError("integer modulo") + # Helpers... def _Py_ARITHMETIC_RIGHT_SHIFT(i, j): Modified: pypy/dist/pypy/translator/llvm/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_exception.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_exception.py Fri Nov 2 06:37:12 2007 @@ -3,9 +3,12 @@ import py from pypy.translator.test.snippet import try_raise_choose from pypy.rlib.rarithmetic import r_uint, ovfcheck, ovfcheck_lshift - +from pypy.rpython.test.test_exception import BaseTestException from pypy.translator.llvm.test.runtest import * +class TestLLVMException(LLVMTest, BaseTestException): + pass + class TestException(Exception): pass Added: pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_rconstantdict.py Fri Nov 2 06:37:12 2007 @@ -0,0 +1,6 @@ +import py +from pypy.rpython.test.test_rconstantdict import BaseTestRconstantdict +from pypy.translator.llvm.test.runtest import * + +class TestLLVMRconstantdict(LLVMTest, BaseTestRconstantdict): + pass From arigo at codespeak.net Fri Nov 2 08:42:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 08:42:17 +0100 (CET) Subject: [pypy-svn] r48249 - pypy/branch/roadshow Message-ID: <20071102074217.5918A8195@code0.codespeak.net> Author: arigo Date: Fri Nov 2 08:42:16 2007 New Revision: 48249 Added: pypy/branch/roadshow/ - copied from r48246, pypy/dist/ Log: A branch (or possibly a tag) that is known to translate and be otherwise reasonably free of blocking issues. We'll stick to this branch during our U.S. trip to avoid bad surprizes. From arigo at codespeak.net Fri Nov 2 10:35:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 10:35:07 +0100 (CET) Subject: [pypy-svn] r48251 - pypy/dist/pypy/translator/c/src Message-ID: <20071102093507.8283581A1@code0.codespeak.net> Author: arigo Date: Fri Nov 2 10:35:06 2007 New Revision: 48251 Modified: pypy/dist/pypy/translator/c/src/int.h Log: An XXX. Modified: pypy/dist/pypy/translator/c/src/int.h ============================================================================== --- pypy/dist/pypy/translator/c/src/int.h (original) +++ pypy/dist/pypy/translator/c/src/int.h Fri Nov 2 10:35:06 2007 @@ -60,6 +60,8 @@ OP_INT_ADD(x,y,r); \ if (r >= (x)); \ else FAIL_OVF("integer addition") +/* XXX can a C compiler be too clever and think it can "prove" that + * r >= x always hold above? */ #define OP_INT_SUB(x,y,r) r = (x) - (y) From cfbolz at codespeak.net Fri Nov 2 11:18:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 11:18:22 +0100 (CET) Subject: [pypy-svn] r48252 - pypy/branch/ooexternals Message-ID: <20071102101822.CE53381AF@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 11:18:21 2007 New Revision: 48252 Added: pypy/branch/ooexternals/ - copied from r48251, pypy/dist/ Log: a branch to explore refactoring ideas for OO externals From antocuni at codespeak.net Fri Nov 2 16:15:45 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Nov 2007 16:15:45 +0100 (CET) Subject: [pypy-svn] r48253 - pypy/dist/pypy/doc Message-ID: <20071102151545.090D580E7@code0.codespeak.net> Author: antocuni Date: Fri Nov 2 16:15:44 2007 New Revision: 48253 Modified: pypy/dist/pypy/doc/cli-backend.txt Log: add instructions about how to run pythonnet Modified: pypy/dist/pypy/doc/cli-backend.txt ============================================================================== --- pypy/dist/pypy/doc/cli-backend.txt (original) +++ pypy/dist/pypy/doc/cli-backend.txt Fri Nov 2 16:15:44 2007 @@ -430,8 +430,31 @@ chars). By contrast, most types are fine for being returned: these include all primitive types, list, tuples and instances. +Installing Python for .NET on Linux +=================================== + +With the CLI backend, you can access .NET libraries from RPython; +programs using .NET libraries will always run when translated, but you +might also want to test them on top of CPython. + +To do so, you can install `Python for .NET`_. Unfortunately, it does +not work out of the box under Linux. + +To make it working, download and unpack the source package of Python +for .NET; the only version tested with PyPy is the 1.0-rc2, but it +might work also with others. Then, you need to create a file named +Python.Runtime.dll.config at the root of the unpacked archive; put the +following lines inside the file:: + + + + + +The installation should be complete now. To run Python for .NET, +simply type ``mono python.exe``. .. _`Standard Ecma 335`: http://www.ecma-international.org/publications/standards/Ecma-335.htm .. _`flow graph`: translation.html#the-flow-model .. _`rtyper`: rtyper.html +.. _`Python for .NET`: http://pythonnet.sourceforge.net/ From pedronis at codespeak.net Fri Nov 2 16:23:32 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 2 Nov 2007 16:23:32 +0100 (CET) Subject: [pypy-svn] r48254 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102152332.0693B81D7@code0.codespeak.net> Author: pedronis Date: Fri Nov 2 16:23:32 2007 New Revision: 48254 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: put in some stub intro slides from europython, put the jit transformation slides in the extra material part Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 16:23:32 2007 @@ -4,6 +4,35 @@ PyPy - automatic generation of VMs for dynamic languages - JIT included ======================================================================== +What is PyPy? +============== + +PyPy is: + +* A fun and challenging open source project with many contributors + +* An open source implementation of Python (written in Python) + +* A tool-chain with a *new paradigm* for building interpreters + +* Dec 2004 - March 2007: a research project funded by the EU + and companies. + +Motivation +============= + +Conventional interpreters face hard trade-offs: +*speed - maintainability - flexibility* + +A lot of decisions are hard-coded pervasively, improvements +and evolution are hampered (e.g. psyco, stackless) + +Lots of duplication of effort to cover many platforms (C, Java, .NET). + +*It should be possible to do better.* + + + Translation aspects ======================== @@ -175,6 +204,9 @@ .. demo f1 +EXTRA MATERIAL +================== + The transformation ================================== @@ -309,8 +341,6 @@ PyPy proves this a viable approach worth of further exploration. -EXTRA MATERIAL -================== .. backend material: not for the general talk .. virtualizables: not for general talk From arigo at codespeak.net Fri Nov 2 18:02:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Nov 2007 18:02:24 +0100 (CET) Subject: [pypy-svn] r48255 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102170224.24BB481E6@code0.codespeak.net> Author: arigo Date: Fri Nov 2 18:02:22 2007 New Revision: 48255 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Shuffle shuffle. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 18:02:22 2007 @@ -4,19 +4,29 @@ PyPy - automatic generation of VMs for dynamic languages - JIT included ======================================================================== + What is PyPy? ============== -PyPy is: +PyPy is a tool-chain with a *new paradigm* for building interpreters: + +* Speed + +* Maintainability + +* Flexibility + +* ...and ease of implementation + +What is PyPy? +============== * A fun and challenging open source project with many contributors * An open source implementation of Python (written in Python) -* A tool-chain with a *new paradigm* for building interpreters - -* Dec 2004 - March 2007: a research project funded by the EU - and companies. +* Dec 2004 - March 2007: a research project funded by the European + Union and companies. Motivation ============= From jacob at codespeak.net Fri Nov 2 18:07:30 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Fri, 2 Nov 2007 18:07:30 +0100 (CET) Subject: [pypy-svn] r48256 - pypy/extradoc/talk/roadshow-google Message-ID: <20071102170730.0B60A81E6@code0.codespeak.net> Author: jacob Date: Fri Nov 2 18:07:29 2007 New Revision: 48256 Added: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp (contents, props changed) Log: Pretty pictures. Added: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp ============================================================================== Binary file. No diff available. From lac at codespeak.net Fri Nov 2 18:29:03 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Fri, 2 Nov 2007 18:29:03 +0100 (CET) Subject: [pypy-svn] r48257 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071102172903.DCABF80D5@code0.codespeak.net> Author: lac Date: Fri Nov 2 18:29:01 2007 New Revision: 48257 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: add better line, work more later Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Fri Nov 2 18:29:01 2007 @@ -10,6 +10,9 @@ PyPy is a tool-chain with a *new paradigm* for building interpreters: +It challenges the following commonly-held assumptions about the +construction of interpreters for dynamic languages. + * Speed * Maintainability From cfbolz at codespeak.net Fri Nov 2 20:14:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 20:14:52 +0100 (CET) Subject: [pypy-svn] r48260 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071102191452.82BF281A4@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 20:14:50 2007 New Revision: 48260 Modified: pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/shadow.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: test and fix for NEW_METHOD primitive Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Fri Nov 2 20:14:50 2007 @@ -35,6 +35,8 @@ # Squeak has primitives all the way up to 575 # So all optional primitives will default to the bytecode implementation prim_table = [make_failing(i) for i in range(576)] +# clean up namespace: +del i prim_table_implemented_only = [] # indicates that what is pushed is an index1, but it is unwrapped and @@ -426,8 +428,8 @@ # XXX not sure this is correct assert isinstance(w_method, model.W_CompiledMethod) w_method.literals[constants.METHOD_HEADER_INDEX] = w_header - for i in range(0,literalcount): - w_method.literals[i+1] = objtable.w_nil + for i0 in range(1, literalcount): + w_method.literals[i0] = objtable.w_nil w_method.bytes = "\x00" * bytecount return w_method Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Fri Nov 2 20:14:50 2007 @@ -129,6 +129,8 @@ return model.W_WordsObject(w_cls, extrasize) elif self.instance_kind == BYTES: return model.W_BytesObject(w_cls, extrasize) + elif self.instance_kind == COMPILED_METHOD: + return model.W_CompiledMethod(extrasize) else: raise NotImplementedError(self.instance_kind) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Fri Nov 2 20:14:50 2007 @@ -411,6 +411,17 @@ w_v = prim(primitives.INST_VAR_AT_0, ["q"]) assert w_v.value == ord("q") +def test_new_method(): + bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) + + shadow = mockclass(0).as_class_get_shadow() + w_method = prim(primitives.NEW_METHOD, [classtable.w_CompiledMethod, len(bytecode), 1025]) + assert w_method.literals[0].value == 1025 + assert len(w_method.literals) == 2 + assert w_method.literals[1] is objtable.w_nil + assert w_method.bytes == "\x00" * len(bytecode) + + # Note: # primitives.NEXT is unimplemented as it is a performance optimization # primitives.NEXT_PUT is unimplemented as it is a performance optimization From cfbolz at codespeak.net Fri Nov 2 21:57:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 21:57:57 +0100 (CET) Subject: [pypy-svn] r48262 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071102205757.CBA3D81DA@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 21:57:57 2007 New Revision: 48262 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: fix test Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Nov 2 21:57:57 2007 @@ -9,6 +9,7 @@ from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk import classtable from pypy.lang.smalltalk import shadow +from pypy.lang.smalltalk import utility # lazy initialization of test data, ie ImageReader and Float class def setup_module(module): @@ -226,7 +227,6 @@ assert w_false is objtable.w_false def test_compile_method(): - py.test.skip("not quite yet") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] @@ -240,15 +240,15 @@ if isinstance(any, str): # assume never have strings of length 1 if len(any) == 1: - return objtable.wrap_chr(any) + return utility.wrap_chr(any) else: - return objtable.wrap_string(any) + return utility.wrap_string(any) if isinstance(any, int): - return objtable.wrap_int(any) + return utility.wrap_int(any) if isinstance(any, bool): - return objtable.wrap_bool(any) + return utility.wrap_bool(any) if isinstance(any, float): - return objtable.wrap_float(any) + return utility.wrap_float(any) else: raise Exception From cfbolz at codespeak.net Fri Nov 2 21:58:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Nov 2007 21:58:23 +0100 (CET) Subject: [pypy-svn] r48263 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071102205823.7E5B481DB@code0.codespeak.net> Author: cfbolz Date: Fri Nov 2 21:58:23 2007 New Revision: 48263 Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: should still be skipped Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Fri Nov 2 21:58:23 2007 @@ -227,6 +227,7 @@ assert w_false is objtable.w_false def test_compile_method(): + py.test.skip("not working") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] From lac at codespeak.net Sat Nov 3 09:06:32 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sat, 3 Nov 2007 09:06:32 +0100 (CET) Subject: [pypy-svn] r48264 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071103080632.8492D8207@code0.codespeak.net> Author: lac Date: Sat Nov 3 09:06:31 2007 New Revision: 48264 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Latest attempt at 3 slides covering 'motivation', except that it doesn't quite do that any more. This may be ok. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sat Nov 3 09:06:31 2007 @@ -10,41 +10,51 @@ PyPy is a tool-chain with a *new paradigm* for building interpreters: -It challenges the following commonly-held assumptions about the +It challenges commonly held assumptions about the construction of interpreters for dynamic languages. -* Speed +What we Reject +============== +* Interpreters for Dynamic Languages must be slow -* Maintainability +* Fast, Maintainable, Flexible -- pick one -* Flexibility +* C is the most appropriate language for implementing interpreters. -* ...and ease of implementation +* C++ is the most appropriate language for implementing interpreters. -What is PyPy? -============== +* Requiring type-hints everywhere, thus turning your dynamic language + into a static one, on the sly -* A fun and challenging open source project with many contributors +What we insist upon +=================== -* An open source implementation of Python (written in Python) +* real source code portability between different target implementations -* Dec 2004 - March 2007: a research project funded by the European - Union and companies. +* support for an actual existing dynamic language, not an academic toy -Motivation -============= +* support for the complete language, not a subset + +* The interpreter we write should be free of low-level details as possible -Conventional interpreters face hard trade-offs: -*speed - maintainability - flexibility* +* the ability to experiement with all aspects of the compiler, including + garbage collection strategies, threading models, security models + +* fun + +Implications? +============= -A lot of decisions are hard-coded pervasively, improvements -and evolution are hampered (e.g. psyco, stackless) +* type analysis must be performed on live code objects, not dead source trees -Lots of duplication of effort to cover many platforms (C, Java, .NET). +* parts of the interpreter must be written in a static manner. -*It should be possible to do better.* +* Generate compilers using partial evaluation techniques. Don't write them + by hand! +* Build a toolchain with pluggable components, for e.g. garbage collection +* The translation framework can be reused by _any_ dynamic language! Translation aspects ======================== From arigo at codespeak.net Sat Nov 3 09:18:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Nov 2007 09:18:11 +0100 (CET) Subject: [pypy-svn] r48265 - pypy/dist/pypy/translator/goal Message-ID: <20071103081811.01E2C8200@code0.codespeak.net> Author: arigo Date: Sat Nov 3 09:18:10 2007 New Revision: 48265 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: This is supposedly the official way to know if we are running on NT or Posix. Althought I cannot find right now the code in PyPy that would expose 'module/posix' under the name 'nt' I guess it must be done somewhere - otherwise tons of things would break on NT. The previous code was also untested (i.e. broken) on non-NT platforms. There are still 3 failures in test2/test_app_main.py but I don't have time to investigate now. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Nov 3 09:18:10 2007 @@ -233,25 +233,19 @@ AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' -# hack to determine if we are on windows. Maybe there is a better way... -DRIVE_LETTER_SEP = '' -try: - here = os.getcwd() - try: - os.chdir('C:') - DRIVE_LETTER_SEP = ':' - finally: - os.chdir(here) -except os.OSError: - DRIVE_LETTER_SEP = None -IS_WINDOWS = DRIVE_LETTER_SEP is not None -# XXX extend here +if 'nt' in sys.builtin_module_names: + IS_WINDOWS = True + DRIVE_LETTER_SEP = ':' +else: + IS_WINDOWS = False def entry_point(executable, argv): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH os.setup() # this is the faked one - if os.sep not in executable and DRIVE_LETTER_SEP not in executable: + if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable): + pass # the path is already more than just an executable name + else: path = os.getenv('PATH') if path: for dir in path.split(os.pathsep): From pedronis at codespeak.net Sat Nov 3 09:31:47 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 3 Nov 2007 09:31:47 +0100 (CET) Subject: [pypy-svn] r48266 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071103083147.37CF38208@code0.codespeak.net> Author: pedronis Date: Sat Nov 3 09:31:46 2007 New Revision: 48266 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: some rough technical meat for the ibm talk before I forget about it (samuele, slitghtly more lucid than yesterday) Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sat Nov 3 09:31:46 2007 @@ -56,6 +56,45 @@ * The translation framework can be reused by _any_ dynamic language! +Translation details +======================= + +- RPython translation starts from loaded and initialized RPython + code as python bytecode in a Python VM. + +- PyPy uses abstract interpretation extensively: to construct flow-graphs, + for type inference, to gather information for some optimisations + +- Obviously in the PE based generated Dynamic Compilers + +- Flow-graph transformation and rewriting is also used + +Representation choice +======================== + +A complex part of RPython translation is choosing implementations +and representations of its still rich built-in types that work for +the target platforms (C/LLVM vs. OO VMs) + +We model the classes of targets through different type systems: + +- low-level: data and function pointers, structures, ... +- object oriented: classes, instances, inheritance and dispatching + +Type systems and helpers +=========================== + +We have emulation of the type systems that can run on top of CPython, +we use them for testing but also for: + +- constructing and representing the static data that our approach involves + (we start from live objects) at translation time + +- the implementation of built-in type for the targets require helper functions: + they are expressed using the emulations which our translation knows about + and can translate too + + Translation aspects ======================== From cfbolz at codespeak.net Sat Nov 3 15:07:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 15:07:23 +0100 (CET) Subject: [pypy-svn] r48267 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103140723.A6FBE8216@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 15:07:22 2007 New Revision: 48267 Added: pypy/dist/pypy/translator/backendopt/coalloc.py (contents, props changed) pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (contents, props changed) Modified: pypy/dist/pypy/translator/backendopt/escape.py Log: a so far very simplistic analysis to find potential coallocation sites Added: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 15:07:22 2007 @@ -0,0 +1,307 @@ +from pypy.annotation.model import setunion +from pypy.objspace.flow.model import Variable, Constant +from pypy.rpython.lltypesystem import lltype +from pypy.translator.simplify import get_graph +from pypy.rpython.rmodel import inputconst +from pypy.translator.backendopt import support +from pypy.tool.uid import uid + +class CreationPoint(object): + def __init__(self, creation_method, TYPE): + self.creation_method = creation_method + self.TYPE = TYPE + + def __repr__(self): + return ("CreationPoint(<0x%x>, %r)" % + (uid(self), self.TYPE)) + +class VarState(object): + def __init__(self, crep=None): + self.creation_points = {} + if crep is not None: + self.creation_points[crep] = True + self.returned = False + + def contains(self, other): + for crep in other.creation_points: + if crep not in self.creation_points: + return False + return True + + def merge(self, other): + creation_points = setunion(self.creation_points, other.creation_points) + newstate = VarState() + newstate.creation_points = creation_points + return newstate + + def __repr__(self): + crepsrepr = (", ".join([repr(crep) for crep in self.creation_points]), ) + return "VarState({%s})" % crepsrepr + +class GraphState(object): + def __init__(self, graph): + self.graph = graph + + +class AbstractDataFlowInterpreter(object): + def __init__(self, translation_context): + self.translation_context = translation_context + self.scheduled = {} # block: graph containing it + self.varstates = {} # var-or-const: state + self.creationpoints = {} # var: creationpoint + self.constant_cps = {} # const: creationpoint + self.dependencies = {} # creationpoint: {block: graph containing it} + self.functionargs = {} # graph: list of state of args + self.flown_blocks = {} # block: True + + def getstate(self, var_or_const): + if not isonheap(var_or_const): + return None + if var_or_const in self.varstates: + return self.varstates[var_or_const] + if isinstance(var_or_const, Variable): + varstate = VarState() + else: + if var_or_const not in self.constant_cps: + crep = CreationPoint("constant", var_or_const.concretetype) + self.constant_cps[var_or_const] = crep + else: + crep = self.constant_cps[var_or_const] + varstate = VarState(crep) + self.varstates[var_or_const] = varstate + return varstate + + def getstates(self, varorconstlist): + return [self.getstate(var) for var in varorconstlist] + + def setstate(self, var, state): + self.varstates[var] = state + + def get_creationpoint(self, var, method="?"): + if var in self.creationpoints: + return self.creationpoints[var] + crep = CreationPoint(method, var.concretetype) + self.creationpoints[var] = crep + return crep + + def schedule_function(self, graph): + #print "scheduling function:", graph.name + startblock = graph.startblock + if graph in self.functionargs: + args = self.functionargs[graph] + else: + args = [] + for var in startblock.inputargs: + if not isonheap(var): + varstate = None + else: + crep = self.get_creationpoint(var, "arg") + varstate = VarState(crep) + self.setstate(var, varstate) + args.append(varstate) + self.scheduled[startblock] = graph + self.functionargs[graph] = args + resultstate = self.getstate(graph.returnblock.inputargs[0]) + return resultstate, args + + def flow_block(self, block, graph): + #print "flowing in block %s of function %s" % (block, graph.name) + self.flown_blocks[block] = True + if block is graph.returnblock: + if isonheap(block.inputargs[0]): + self.getstate(block.inputargs[0]).returned = True + return + if block is graph.exceptblock: + return + self.curr_block = block + self.curr_graph = graph + #print "inputargs", self.getstates(block.inputargs) + + for op in block.operations: + self.flow_operation(op) + #print "checking exits..." + for exit in block.exits: + #print "exit", exit + args = self.getstates(exit.args) + targetargs = self.getstates(exit.target.inputargs) + #print " newargs", args + #print " targetargs", targetargs + # flow every block at least once: + if (multicontains(targetargs, args) and + exit.target in self.flown_blocks): + #print " not necessary" + continue + #else: + #print " scheduling for flowin" + for prevstate, origstate, var in zip(args, targetargs, + exit.target.inputargs): + if not isonheap(var): + continue + newstate = prevstate.merge(origstate) + self.setstate(var, newstate) + #print " args", self.getstates(exit.target.inputargs) + self.scheduled[exit.target] = graph + + def flow_operation(self, op): + #print "handling", op + args = self.getstates(op.args) + #print "args:", args + opimpl = getattr(self, 'op_'+op.opname, None) + if opimpl is not None: + res = opimpl(op, *args) + if res is not NotImplemented: + self.setstate(op.result, res) + return + + if isonheap(op.result) or filter(None, args): + raise NotImplementedError("can't handle %s" % (op.opname, )) + #print "assuming that '%s' is irrelevant" % op + + def complete(self): + while self.scheduled: + block, graph = self.scheduled.popitem() + self.flow_block(block, graph) + + def handle_changed(self, changed): + for crep in changed: + if crep not in self.dependencies: + continue + self.scheduled.update(self.dependencies[crep]) + + def register_block_dependency(self, state, block=None, graph=None): + if block is None: + block = self.curr_block + graph = self.curr_graph + for crep in state.creation_points: + self.dependencies.setdefault(crep, {})[block] = graph + + def register_state_dependency(self, state1, state2): + "state1 depends on state2: if state2 does escape/change, so does state1" + # change state1 according to how state2 is now + #print "registering dependency of %s on %s" % (state1, state2) + if state2.does_escape(): + changed = state1.setescapes() # mark all crep's as escaping + self.handle_changed(changed) + if state2.does_change(): + changed = state1.setchanges() # mark all crep's as changing + self.handle_changed(changed) + # register a dependency of the current block on state2: + # that means that if state2 changes the current block will be reflown + # triggering this function again and thus updating state1 + self.register_block_dependency(state2) + + # _____________________________________________________________________ + # operation implementations + + def op_malloc(self, op, typestate, flagsstate): + assert flagsstate is None + flags = op.args[1].value + if flags != {'flavor': 'gc'}: + return NotImplemented + return VarState(self.get_creationpoint(op.result, "malloc")) + + def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): + assert flagsstate is None + flags = op.args[1].value + if flags != {'flavor': 'gc'}: + return NotImplemented + return VarState(self.get_creationpoint(op.result, "malloc_varsize")) + + def op_keepalive(self, op, state): + return None + + def op_cast_pointer(self, op, state): + return state + + def op_getfield(self, op, objstate, fieldname): + # connectivity-wise the field within is identical to the containing + # structure + return objstate + op_getarrayitem = op_getinteriorfield = op_getfield + + def op_getarraysize(self, op, arraystate): + pass + + def op_setfield(self, op, objstate, fieldname, valuestate): + pass + op_setarrayitem = op_setinteriorfield = op_setfield + + def op_direct_call(self, op, function, *args): +# graph = get_graph(op.args[0], self.translation_context) +# if graph is None: +# for arg in args: +# if arg is None: +# continue +# # an external function can change every parameter: +# changed = arg.setchanges() +# self.handle_changed(changed) +# funcargs = [None] * len(args) +# else: +# result, funcargs = self.schedule_function(graph) +# assert len(args) == len(funcargs) +# for localarg, funcarg in zip(args, funcargs): +# if localarg is None: +# assert funcarg is None +# continue +# if funcarg is not None: +# self.register_state_dependency(localarg, funcarg) + if isonheap(op.result): + # for now assume that a call always creates a new value + return VarState(self.get_creationpoint(op.result, "direct_call")) + + def op_indirect_call(self, op, function, *args): +# graphs = op.args[-1].value +# args = args[:-1] +# if graphs is None: +# for localarg in args: +# if localarg is None: +# continue +# changed = localarg.setescapes() +# self.handle_changed(changed) +# changed = localarg.setchanges() +# self.handle_changed(changed) +# else: +# for graph in graphs: +# result, funcargs = self.schedule_function(graph) +# assert len(args) == len(funcargs) +# for localarg, funcarg in zip(args, funcargs): +# if localarg is None: +# assert funcarg is None +# continue +# self.register_state_dependency(localarg, funcarg) + if isonheap(op.result): + # for now assume that a call always creates a new value + return VarState(self.get_creationpoint(op.result, "indirect_call")) + + def op_ptr_iszero(self, op, ptrstate): + return None + + op_cast_ptr_to_int = op_keepalive = op_ptr_nonzero = op_ptr_iszero + + def op_ptr_eq(self, op, ptr1state, ptr2state): + return None + + op_ptr_ne = op_ptr_eq + + def op_same_as(self, op, objstate): + return objstate + +def isonheap(var_or_const): + return isinstance(var_or_const.concretetype, lltype.Ptr) + +def multicontains(l1, l2): + assert len(l1) == len(l2) + for a, b in zip(l1, l2): + if a is None: + assert b is None + elif not a.contains(b): + return False + return True + +def malloc_to_coalloc(t): + adi = AbstractDataFlowInterpreter(t) + for graph in t.graphs: + if graph.startblock not in adi.flown_blocks: + adi.schedule_function(graph) + adi.complete() + return adi Modified: pypy/dist/pypy/translator/backendopt/escape.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/escape.py (original) +++ pypy/dist/pypy/translator/backendopt/escape.py Sat Nov 3 15:07:22 2007 @@ -386,25 +386,26 @@ for graph in t.graphs: loop_blocks = support.find_loop_blocks(graph) for block, op in graph.iterblockops(): - if op.opname == 'malloc': - STRUCT = op.args[0].value - # must not remove mallocs of structures that have a RTTI with a destructor - try: - destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr - if destr_ptr: - continue - except (ValueError, AttributeError), e: - pass - varstate = adi.getstate(op.result) - assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: - if block not in loop_blocks: - print "moving object from heap to stack %s in %s" % (op, graph.name) - flags = op.args[1].value - assert flags == {'flavor': 'gc'} - op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) - else: - print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) + if op.opname != 'malloc': + continue + STRUCT = op.args[0].value + # must not remove mallocs of structures that have a RTTI with a destructor + try: + destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr + if destr_ptr: + continue + except (ValueError, AttributeError), e: + pass + varstate = adi.getstate(op.result) + assert len(varstate.creation_points) == 1 + crep = varstate.creation_points.keys()[0] + if not crep.escapes: + if block not in loop_blocks: + print "moving object from heap to stack %s in %s" % (op, graph.name) + flags = op.args[1].value + assert flags == {'flavor': 'gc'} + op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) + else: + print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) Added: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 15:07:22 2007 @@ -0,0 +1,114 @@ +from pypy.translator.translator import TranslationContext, graphof +from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter +from pypy.rpython.llinterp import LLInterpreter +from pypy.rlib.objectmodel import instantiate +from pypy import conftest + +import py + +def build_adi(function, types): + t = TranslationContext() + t.buildannotator().build_types(function, types) + t.buildrtyper().specialize() + if conftest.option.view: + t.view() + adi = AbstractDataFlowInterpreter(t) + graph = graphof(t, function) + adi.schedule_function(graph) + adi.complete() + return t, adi, graph + +def test_simple(): + class A(object): + pass + def f(): + a = A() + a.x = 1 + return a.x + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + +def test_branch(): + class T: + pass + def fn2(x, y): + t = T() + t.x = x + t.y = y + if x > 0: + return t.x + t.y + else: + return t.x - t.y + t, adi, graph = build_adi(fn2, [int, int]) + tvar = graph.startblock.operations[0].result + state = adi.getstate(tvar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + +def test_loop(): + class A(object): + pass + def f(): + a = A() + i = 0 + while i < 3: + a.x = i + a = A() + i += 1 + return a.x + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + avarinloop = graph.startblock.exits[0].target.inputargs[1] + state1 = adi.getstate(avarinloop) + assert crep in state1.creation_points + assert len(state1.creation_points) == 2 + +def test_global(): + class A(object): + pass + globala = A() + def f(): + a = A() + a.next = None + globala.next = a + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "malloc" + const = graph.startblock.operations[-1].args[0] + state = adi.getstate(const) + assert len(state.creation_points) == 1 + crep = state.creation_points.keys()[0] + assert crep.creation_method == "constant" + +def test_aliasing(): + class A: + pass + def fn6(n): + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 6 + if n > 0: + a = a1 + else: + a = a2 + a.x = 12 + return a1.x + t, adi, graph = build_adi(fn6, [int]) + avar = graph.startblock.exits[0].target.inputargs[1] + state = adi.getstate(avar) + assert len(state.creation_points) == 2 + for crep in state.creation_points.keys(): + assert crep.creation_method == "malloc" + From cfbolz at codespeak.net Sat Nov 3 16:16:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 16:16:13 +0100 (CET) Subject: [pypy-svn] r48268 - in pypy/dist/pypy/rpython: . lltypesystem Message-ID: <20071103151613.80B6381D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 16:16:11 2007 New Revision: 48268 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py Log: add coalloc support to the llinterp Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Sat Nov 3 16:16:11 2007 @@ -675,6 +675,10 @@ self.llinterpreter.remember_malloc(ptr, self) return ptr + def op_coalloc(self, obj, coallocator, flags): + # it's always safe to ignore the coallocator + return self.op_malloc(obj, flags) + # only after gc transform def op_cpy_malloc(self, obj, cpytype): # xxx return self.heap.malloc(obj, flavor='cpy', extra_args=(cpytype,)) @@ -694,6 +698,10 @@ except MemoryError: self.make_llexception() + def op_coalloc_varsize(self, obj, coallocator, flags, size): + # it's always safe to ignore the coallocator + return self.op_malloc_varsize(obj, flags, size) + def op_free(self, obj, flavor): assert isinstance(flavor, str) if flavor == 'raw' and self.llinterpreter.malloc_check: Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sat Nov 3 16:16:11 2007 @@ -316,6 +316,8 @@ 'malloc': LLOp(canraise=(MemoryError,), canunwindgc=True), 'malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'coalloc': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'coalloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), From cfbolz at codespeak.net Sat Nov 3 16:19:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 16:19:46 +0100 (CET) Subject: [pypy-svn] r48269 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103151946.6750D81D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 16:19:46 2007 New Revision: 48269 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: replace mallocs by coallocs Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 16:19:46 2007 @@ -7,6 +7,7 @@ from pypy.tool.uid import uid class CreationPoint(object): + constant = None def __init__(self, creation_method, TYPE): self.creation_method = creation_method self.TYPE = TYPE @@ -64,6 +65,7 @@ else: if var_or_const not in self.constant_cps: crep = CreationPoint("constant", var_or_const.concretetype) + crep.constant = var_or_const self.constant_cps[var_or_const] = crep else: crep = self.constant_cps[var_or_const] @@ -154,8 +156,7 @@ return if isonheap(op.result) or filter(None, args): - raise NotImplementedError("can't handle %s" % (op.opname, )) - #print "assuming that '%s' is irrelevant" % op + print "assuming that '%s' is irrelevant" % op def complete(self): while self.scheduled: @@ -304,4 +305,59 @@ if graph.startblock not in adi.flown_blocks: adi.schedule_function(graph) adi.complete() - return adi + look_at = t.graphs[:] + total = 0 + while look_at: + graph = look_at.pop() + for block, op in graph.iterblockops(): + if not op.opname.startswith("set"): + continue + if not isonheap(op.args[-1]): + continue + tovarstate = adi.getstate(op.args[-1]) + fromvarstate = adi.getstate(op.args[0]) + if (len(tovarstate.creation_points) != 1 or + len(fromvarstate.creation_points) != 1): + continue + fromcrep = fromvarstate.creation_points.keys()[0] + tocrep = tovarstate.creation_points.keys()[0] + if not tocrep.creation_method.startswith("malloc"): + continue + if fromcrep.creation_method.startswith("malloc"): + continue # also recently malloced + + num = do_coalloc(adi, graph, fromcrep, tocrep) + + if num: + look_at.append(graph) + print "changed %s mallocs to coallocs in %s" % (num, graph.name) + total += num + return total + + +def do_coalloc(adi, graph, fromcrep, tocrep): + result = 0 + for block, op in graph.iterblockops(): + if not op.opname.startswith("malloc"): + continue + # find coallocation var + if fromcrep.creation_method == "constant": + coallocvar = fromcrep.constant + else: + for var in block.inputargs: + varstate = adi.getstate(var) + assert len(varstate.creation_points) == 1 + crep = varstate.creation_points.keys()[0] + if crep is fromcrep: + coallocvar = var + break + else: + continue + op.opname = "coalloc" + op.opname[len("malloc"):] + op.args.insert(1, coallocvar) + mallocvarstate = adi.getstate(op.result) + assert len(mallocvarstate.creation_points) == 1 + malloccrep = mallocvarstate.creation_points.keys()[0] + malloccrep.creation_method = "coalloc" + result += 1 + return result Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 16:19:46 2007 @@ -1,5 +1,5 @@ from pypy.translator.translator import TranslationContext, graphof -from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter +from pypy.translator.backendopt.coalloc import AbstractDataFlowInterpreter, malloc_to_coalloc from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import instantiate from pypy import conftest @@ -18,6 +18,27 @@ adi.complete() return t, adi, graph +def check_malloc_to_coalloc(function, types, args, expected_result, must_remove=-1): + t = TranslationContext() + t.buildannotator().build_types(function, types) + t.buildrtyper().specialize() + interp = LLInterpreter(t.rtyper) + graph = graphof(t, function) + res = interp.eval_graph(graph, args) + assert res == expected_result + num = malloc_to_coalloc(t) + if must_remove == -1: + for block in graph.iterblocks(): + for op in block.operations: + assert op.opname != "malloc" + else: + assert num == must_remove + t.view() + res = interp.eval_graph(graph, args) + assert res == expected_result + return t + + def test_simple(): class A(object): pass @@ -112,3 +133,41 @@ for crep in state.creation_points.keys(): assert crep.creation_method == "malloc" +def test_coalloc_constants(): + class A(object): + pass + a = A() + def f(): + n = A() + a.next = n + return 1 + check_malloc_to_coalloc(f, [], [], 1) + +def test_nocoalloc_aliasing(): + class A: + pass + def fn6(n): + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 6 + if n > 0: + a = a1 + else: + a = a2 + a.x = 12 + return a1.x + t = check_malloc_to_coalloc(fn6, [int], [2], 12, must_remove=0) + +def test_coalloc_with_arg(): + class A(object): + pass + def g(b): + b.x = A() + def f(): + a = A() + g(a) + a.i = 2 + return 4 + t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) + From cfbolz at codespeak.net Sat Nov 3 18:09:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 18:09:47 +0100 (CET) Subject: [pypy-svn] r48271 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103170947.96E8E822B@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 18:09:47 2007 New Revision: 48271 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: another test, tweaks Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 18:09:47 2007 @@ -316,17 +316,18 @@ continue tovarstate = adi.getstate(op.args[-1]) fromvarstate = adi.getstate(op.args[0]) - if (len(tovarstate.creation_points) != 1 or - len(fromvarstate.creation_points) != 1): + if len(tovarstate.creation_points) != 1: continue - fromcrep = fromvarstate.creation_points.keys()[0] + fromcreps = set(fromvarstate.creation_points.keys()) tocrep = tovarstate.creation_points.keys()[0] if not tocrep.creation_method.startswith("malloc"): continue - if fromcrep.creation_method.startswith("malloc"): - continue # also recently malloced + for fromcrep in fromcreps: + if fromcrep.creation_method.startswith("malloc"): + continue # also recently malloced - num = do_coalloc(adi, graph, fromcrep, tocrep) + num = do_coalloc(adi, graph, op.args[0], block, + fromcreps, tocrep) if num: look_at.append(graph) @@ -335,24 +336,28 @@ return total -def do_coalloc(adi, graph, fromcrep, tocrep): - result = 0 - for block, op in graph.iterblockops(): - if not op.opname.startswith("malloc"): - continue - # find coallocation var - if fromcrep.creation_method == "constant": - coallocvar = fromcrep.constant - else: +def do_coalloc(adi, graph, fromvar, setblock, fromcreps, tocrep): + def find_coalloc_var(): + if block is setblock: + return fromvar + for fromcrep in fromcreps: + if fromcrep.creation_method == "constant": + return fromcrep.constant + for fromcrep in fromcreps: for var in block.inputargs: varstate = adi.getstate(var) assert len(varstate.creation_points) == 1 crep = varstate.creation_points.keys()[0] if crep is fromcrep: - coallocvar = var - break - else: - continue + return var + return None + result = 0 + for block, op in graph.iterblockops(): + if not op.opname.startswith("malloc"): + continue + coallocvar = find_coalloc_var() + if coallocvar is None: + continue op.opname = "coalloc" + op.opname[len("malloc"):] op.args.insert(1, coallocvar) mallocvarstate = adi.getstate(op.result) Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 18:09:47 2007 @@ -33,7 +33,8 @@ assert op.opname != "malloc" else: assert num == must_remove - t.view() + if conftest.option.view: + t.view() res = interp.eval_graph(graph, args) assert res == expected_result return t @@ -171,3 +172,17 @@ return 4 t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) +def test_coalloc_with_arg_several_creationpoints(): + class A(object): + pass + a1 = A() + def g(cond, b): + if cond: + b = a1 + b.x = A() + def f(cond): + a2 = A() + g(cond, a2) + return 4 + t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + From cfbolz at codespeak.net Sat Nov 3 18:11:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 18:11:13 +0100 (CET) Subject: [pypy-svn] r48272 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20071103171113.93AED822C@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 18:11:13 2007 New Revision: 48272 Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: another working test Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 18:11:13 2007 @@ -172,7 +172,7 @@ return 4 t = check_malloc_to_coalloc(f, [], [], 4, must_remove=1) -def test_coalloc_with_arg_several_creationpoints(): +def test_coalloc_with_arg_set_in_same_block(): class A(object): pass a1 = A() @@ -186,3 +186,18 @@ return 4 t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) +def test_coalloc_with_arg_several_creationpoints(): + class A(object): + pass + a1 = A() + def g(cond, b): + a = A() + if cond: + b = a1 + b.x = a + def f(cond): + a2 = A() + g(cond, a2) + return 4 + t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + From tismer at codespeak.net Sat Nov 3 18:30:53 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 3 Nov 2007 18:30:53 +0100 (CET) Subject: [pypy-svn] r48273 - pypy/dist/pypy/translator/goal/test2 Message-ID: <20071103173053.C1884823C@code0.codespeak.net> Author: tismer Date: Sat Nov 3 18:30:52 2007 New Revision: 48273 Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py Log: relaxing test that depends on filenames in windows. os.abspath is incompatible with autopath right now. The real fix is to implement _getfullpathname . The provided fallback in ntpath is pretty insufficient, and a patch would probably be suitable for CPython Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/dist/pypy/translator/goal/test2/test_app_main.py Sat Nov 3 18:30:52 2007 @@ -337,5 +337,8 @@ data = self.run('-m pypy.translator.goal.test2.mymodule extra') assert 'mymodule running' in data assert 'Name: __main__' in data - assert ('File: ' + p) in data - assert ('Argv: ' + repr([p, 'extra'])) in data + # ignoring case for windows. abspath behaves different from autopath + # concerning drive letters right now + # XXX fix nt module to implement _getfullpathname + assert ('File: ' + p).lower() in data.lower() + assert ('Argv: ' + repr([p, 'extra'])).lower() in data.lower() From cfbolz at codespeak.net Sat Nov 3 19:11:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:11:55 +0100 (CET) Subject: [pypy-svn] r48274 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103181155.F255E823B@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:11:55 2007 New Revision: 48274 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: woops, check whether the malloc we are trying to replace is the correct one. Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:11:55 2007 @@ -326,6 +326,7 @@ if fromcrep.creation_method.startswith("malloc"): continue # also recently malloced + #import pdb; pdb.set_trace() num = do_coalloc(adi, graph, op.args[0], block, fromcreps, tocrep) @@ -355,6 +356,8 @@ for block, op in graph.iterblockops(): if not op.opname.startswith("malloc"): continue + if adi.getstate(op.result).creation_points.keys()[0] is not tocrep: + continue coallocvar = find_coalloc_var() if coallocvar is None: continue Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:11:55 2007 @@ -201,3 +201,28 @@ return 4 t = check_malloc_to_coalloc(f, [bool], [True], 4, must_remove=1) + +def test_coalloc_list(): + class A(object): + pass + a1 = A() + def f(count): + i = 0 + l = [] + while i < count: + l.append(A()) + i += 1 + return len(l) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=2) + + +def test_nocoalloc_bug(): + class A(object): + pass + a1 = A() + def f(count): + a = A() + a.length = count + a.items = A() + return a.length + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) From tismer at codespeak.net Sat Nov 3 19:15:58 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 3 Nov 2007 19:15:58 +0100 (CET) Subject: [pypy-svn] r48275 - pypy/dist/pypy/translator/goal Message-ID: <20071103181558.0CDB581DC@code0.codespeak.net> Author: tismer Date: Sat Nov 3 19:15:58 2007 New Revision: 48275 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: some of the test cases see to have trouble with imports. "from pypy.rlib.objectmodel import we_are_translated" caused test_options_u_PYTHONINSPECT to fail. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Nov 3 19:15:58 2007 @@ -188,7 +188,11 @@ modules changeable, again. """ -from pypy.rlib.objectmodel import we_are_translated +try: + # for some reason, some tests break with this import + from pypy.rlib.objectmodel import we_are_translated +except ImportError: + we_are_translated = False class fake_os: def __init__(self): From cfbolz at codespeak.net Sat Nov 3 19:19:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:19:28 +0100 (CET) Subject: [pypy-svn] r48276 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103181928.9DF5381DC@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:19:28 2007 New Revision: 48276 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: fix two wrong tests, and the code Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:19:28 2007 @@ -324,16 +324,17 @@ continue for fromcrep in fromcreps: if fromcrep.creation_method.startswith("malloc"): - continue # also recently malloced + break # also recently malloced + else: + #import pdb; pdb.set_trace() + num = do_coalloc(adi, graph, op.args[0], block, + fromcreps, tocrep) - #import pdb; pdb.set_trace() - num = do_coalloc(adi, graph, op.args[0], block, - fromcreps, tocrep) - - if num: - look_at.append(graph) - print "changed %s mallocs to coallocs in %s" % (num, graph.name) - total += num + if num: + look_at.append(graph) + print "changed %s mallocs to coallocs in %s" % ( + num, graph.name) + total += num return total Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:19:28 2007 @@ -213,16 +213,18 @@ l.append(A()) i += 1 return len(l) - t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=2) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) def test_nocoalloc_bug(): class A(object): pass a1 = A() + def g(a): + a.items = A() def f(count): a = A() a.length = count - a.items = A() + g(a) return a.length t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) From cfbolz at codespeak.net Sat Nov 3 19:32:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Nov 2007 19:32:27 +0100 (CET) Subject: [pypy-svn] r48277 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071103183227.258568232@code0.codespeak.net> Author: cfbolz Date: Sat Nov 3 19:32:25 2007 New Revision: 48277 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: some fixes necessary to make coallocation with dictionaries work. Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Sat Nov 3 19:32:25 2007 @@ -197,14 +197,14 @@ def op_malloc(self, op, typestate, flagsstate): assert flagsstate is None flags = op.args[1].value - if flags != {'flavor': 'gc'}: + if flags['flavor'] != 'gc': return NotImplemented return VarState(self.get_creationpoint(op.result, "malloc")) def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): assert flagsstate is None flags = op.args[1].value - if flags != {'flavor': 'gc'}: + if flags['flavor'] != 'gc': return NotImplemented return VarState(self.get_creationpoint(op.result, "malloc_varsize")) @@ -214,7 +214,7 @@ def op_cast_pointer(self, op, state): return state - def op_getfield(self, op, objstate, fieldname): + def op_getfield(self, op, objstate, *rest): # connectivity-wise the field within is identical to the containing # structure return objstate @@ -223,7 +223,7 @@ def op_getarraysize(self, op, arraystate): pass - def op_setfield(self, op, objstate, fieldname, valuestate): + def op_setfield(self, op, objstate, *args): pass op_setarrayitem = op_setinteriorfield = op_setfield Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Sat Nov 3 19:32:25 2007 @@ -215,6 +215,18 @@ return len(l) t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) +def test_coalloc_dict(): + class A(object): + pass + a1 = A() + def f(count): + i = 0 + d = {} + while i < count: + d[i] = A() + i += 1 + return len(d) + t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) def test_nocoalloc_bug(): class A(object): From fijal at codespeak.net Sun Nov 4 12:14:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:14:09 +0100 (CET) Subject: [pypy-svn] r48278 - pypy/dist/pypy/rpython/raisingops Message-ID: <20071104111409.A768C81C7@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:14:07 2007 New Revision: 48278 Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py Log: Kill tabs. Not sure what this stuff is doing here and why I cannot find tests for that. Modified: pypy/dist/pypy/rpython/raisingops/raisingops.py ============================================================================== --- pypy/dist/pypy/rpython/raisingops/raisingops.py (original) +++ pypy/dist/pypy/rpython/raisingops/raisingops.py Sun Nov 4 12:14:07 2007 @@ -36,8 +36,8 @@ def llong_floordiv_zer(x, y): '''#define OP_LLONG_FLOORDIV_ZER(x,y,r) \ - if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ - else FAIL_ZER("integer division") + if ((y)) { OP_LLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("integer division") ''' if y: return llop.llong_floordiv(SignedLongLong, x, y) @@ -46,8 +46,8 @@ def ullong_floordiv_zer(x, y): '''#define OP_ULLONG_FLOORDIV_ZER(x,y,r) \ - if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ - else FAIL_ZER("unsigned integer division") + if ((y)) { OP_ULLONG_FLOORDIV(x,y,r); } \ + else FAIL_ZER("unsigned integer division") ''' if y: return llop.llong_floordiv(UnsignedLongLong, x, y) @@ -223,8 +223,8 @@ def llong_mod_zer(x, y): '''#define OP_LLONG_MOD_ZER(x,y,r) \ - if ((y)) { OP_LLONG_MOD(x,y,r); } \ - else FAIL_ZER("integer modulo") + if ((y)) { OP_LLONG_MOD(x,y,r); } \ + else FAIL_ZER("integer modulo") ''' if y: return llop.int_mod(SignedLongLong, x, y) From fijal at codespeak.net Sun Nov 4 12:57:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:57:49 +0100 (CET) Subject: [pypy-svn] r48279 - pypy/dist/pypy/interpreter Message-ID: <20071104115749.117A481D7@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:57:49 2007 New Revision: 48279 Modified: pypy/dist/pypy/interpreter/argument.py Log: Mute warnings. Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 12:57:49 2007 @@ -145,6 +145,11 @@ """ args_w, kwds_w = self.unpack() return Arguments(self.space, args_w, kwds_w) + + def unpack(self): + """ Purely abstract + """ + raise NotImplementedError() class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): From fijal at codespeak.net Sun Nov 4 12:59:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 12:59:44 +0100 (CET) Subject: [pypy-svn] r48280 - pypy/dist/pypy/interpreter Message-ID: <20071104115944.B3D37819F@code0.codespeak.net> Author: fijal Date: Sun Nov 4 12:59:44 2007 New Revision: 48280 Modified: pypy/dist/pypy/interpreter/argument.py Log: Another purely abstract method Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 12:59:44 2007 @@ -150,6 +150,11 @@ """ Purely abstract """ raise NotImplementedError() + + def firstarg(self): + """ Purely abstract + """ + raise NotImplementedError() class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): From fijal at codespeak.net Sun Nov 4 13:04:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 13:04:27 +0100 (CET) Subject: [pypy-svn] r48281 - pypy/dist/pypy/interpreter Message-ID: <20071104120427.0C52181C3@code0.codespeak.net> Author: fijal Date: Sun Nov 4 13:04:26 2007 New Revision: 48281 Modified: pypy/dist/pypy/interpreter/argument.py Log: Another purely-abstract methods Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sun Nov 4 13:04:26 2007 @@ -155,7 +155,17 @@ """ Purely abstract """ raise NotImplementedError() + + def _match_signature(self, scope_w, argnames, has_vararg=False, has_kwarg=False, defaults_w=[], blindargs=0, extravarargs=None): + """ Purely abstract + """ + raise NotImplementedError() + def fixedunpack(self, argcount): + """ Purely abstract + """ + raise NotImplementedError() + class ArgumentsPrepended(AbstractArguments): def __init__(self, args, w_firstarg): self.space = args.space From lac at codespeak.net Sun Nov 4 13:12:42 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 4 Nov 2007 13:12:42 +0100 (CET) Subject: [pypy-svn] r48282 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104121242.C8CF581C4@code0.codespeak.net> Author: lac Date: Sun Nov 4 13:12:41 2007 New Revision: 48282 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Ok, try the non-challenging, non-threatening version Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 13:12:41 2007 @@ -13,48 +13,43 @@ It challenges commonly held assumptions about the construction of interpreters for dynamic languages. -What we Reject -============== -* Interpreters for Dynamic Languages must be slow - -* Fast, Maintainable, Flexible -- pick one - -* C is the most appropriate language for implementing interpreters. - -* C++ is the most appropriate language for implementing interpreters. - -* Requiring type-hints everywhere, thus turning your dynamic language - into a static one, on the sly - -What we insist upon -=================== +Folk Wisdom about Interpreters for Dynamic Languages +==================================================== +* There are unavoidable tradeoffs between flexibilty, maintainability, + and speed -* real source code portability between different target implementations +* Popular dynamic languages such as Python and Ruby are the best we + can do given these tradeoffs and constraints. -* support for an actual existing dynamic language, not an academic toy - -* support for the complete language, not a subset - -* The interpreter we write should be free of low-level details as possible - -* the ability to experiement with all aspects of the compiler, including - garbage collection strategies, threading models, security models - -* fun +* Fast, Maintainable, Flexible -- pick one -Implications? -============= +What this means in Practice +=========================== -* type analysis must be performed on live code objects, not dead source trees +Dynamic languages are not very flexible. -* parts of the interpreter must be written in a static manner. +- Low-level decisions permeate the entire code base. +- One cannot simply plug-in a new garbage collector, or threading model + when one desires to experiment. +- Early decisions come back to haunt you. + +Dynamic languages are hard to maintain. + +- because they are traditionally written in low-level languages, they are + harder to write than they need be. +- the language community, which generates experts in the dynamic langauge, + can not use this expertise in its own maintenance. Instead, expertise + in C, or C++ is usually needed. +- every time a new VM is needed, there is a fork in the language community. + The maintainers of Jython and IronPython, for instance, are lost to the + C Python community. They have enough to do just to keep up with C Python. -* Generate compilers using partial evaluation techniques. Don't write them - by hand! +Dynamic Languages are Slow. -* Build a toolchain with pluggable components, for e.g. garbage collection +Thus it looks like what we really have is: +* Fast, Maintainable, Flexible -- you get NONE. -* The translation framework can be reused by _any_ dynamic language! +There has got to be a better way. Translation details ======================= From fijal at codespeak.net Sun Nov 4 14:31:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 14:31:12 +0100 (CET) Subject: [pypy-svn] r48283 - pypy/dist/pypy/translator/goal Message-ID: <20071104133112.7D43A820A@code0.codespeak.net> Author: fijal Date: Sun Nov 4 14:31:10 2007 New Revision: 48283 Added: pypy/dist/pypy/translator/goal/targetwarningalone.py (contents, props changed) Log: Strange target for rpbc experiments. Added: pypy/dist/pypy/translator/goal/targetwarningalone.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetwarningalone.py Sun Nov 4 14:31:10 2007 @@ -0,0 +1,38 @@ + +class Base(object): + pass + +class A(Base): + a = 1 + b = 2 + c = 3 + +class B(Base): + a = 1 + b = 2 + +class C(Base): + b = 8 + c = 6 + +def f(n): + if n > 3: + x = A + elif n > 1: + x = B + else: + x = C + if n > 0: + return x.a + return 8 + +# __________ Entry point __________ + +def entry_point(argv): + f(int(argv[0])) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From fijal at codespeak.net Sun Nov 4 14:40:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 14:40:04 +0100 (CET) Subject: [pypy-svn] r48284 - pypy/dist/pypy/rpython/test Message-ID: <20071104134004.B6A398215@code0.codespeak.net> Author: fijal Date: Sun Nov 4 14:40:04 2007 New Revision: 48284 Modified: pypy/dist/pypy/rpython/test/test_rpbc.py Log: Add (skipped) test for possible missing attribute. should at least give some warning or explode Modified: pypy/dist/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rpbc.py (original) +++ pypy/dist/pypy/rpython/test/test_rpbc.py Sun Nov 4 14:40:04 2007 @@ -1565,6 +1565,34 @@ self.interpret_raises(NotImplementedError, f, [7]) + def test_possible_missing_attribute_access(self): + py.test.skip("Should explode or give some warning") + class Base(object): + pass + + class A(Base): + a = 1 + b = 2 + + class B(Base): + a = 2 + b = 2 + + class C(Base): + b = 8 + + def f(n): + if n > 3: + x = A + elif n > 1: + x = B + else: + x = C + if n > 0: + return x.a + return 9 + + self.interpret(f, [int]) class TestLLtype(BaseTestRPBC, LLRtypeMixin): pass From fijal at codespeak.net Sun Nov 4 14:44:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 14:44:49 +0100 (CET) Subject: [pypy-svn] r48285 - pypy/dist/pypy/translator/goal Message-ID: <20071104134449.AF6F78216@code0.codespeak.net> Author: fijal Date: Sun Nov 4 14:44:49 2007 New Revision: 48285 Modified: pypy/dist/pypy/translator/goal/targetwarningalone.py Log: Add few things. Modified: pypy/dist/pypy/translator/goal/targetwarningalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetwarningalone.py (original) +++ pypy/dist/pypy/translator/goal/targetwarningalone.py Sun Nov 4 14:44:49 2007 @@ -8,7 +8,7 @@ c = 3 class B(Base): - a = 1 + a = 2 b = 2 class C(Base): @@ -24,15 +24,19 @@ x = C if n > 0: return x.a - return 8 + return 9 # __________ Entry point __________ def entry_point(argv): - f(int(argv[0])) + print f(int(argv[1])) return 0 # _____ Define and setup target ___ def target(*args): return entry_point, None + +if __name__ == '__main__': + import sys + entry_point(sys.argv) From fijal at codespeak.net Sun Nov 4 15:16:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 15:16:33 +0100 (CET) Subject: [pypy-svn] r48286 - pypy/dist/pypy/rpython Message-ID: <20071104141633.DE4BF81DC@code0.codespeak.net> Author: fijal Date: Sun Nov 4 15:16:32 2007 New Revision: 48286 Modified: pypy/dist/pypy/rpython/rpbc.py Log: This effectively mutes some rpbc rtyper warnings Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Sun Nov 4 15:16:32 2007 @@ -481,8 +481,8 @@ if r_value.lowleveltype is Void: continue try: - thisattrvalue = frozendesc.attrcache[attr] - except KeyError: + thisattrvalue = frozendesc.read_attribute(attr) + except AttributeError: warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) From fijal at codespeak.net Sun Nov 4 15:18:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 15:18:12 +0100 (CET) Subject: [pypy-svn] r48287 - in pypy/dist/pypy: module/_sre objspace/std Message-ID: <20071104141812.4464781DC@code0.codespeak.net> Author: fijal Date: Sun Nov 4 15:18:11 2007 New Revision: 48287 Modified: pypy/dist/pypy/module/_sre/interp_sre.py pypy/dist/pypy/objspace/std/formatting.py Log: Mute some more warnings about harmless demotion Modified: pypy/dist/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/dist/pypy/module/_sre/interp_sre.py (original) +++ pypy/dist/pypy/module/_sre/interp_sre.py Sun Nov 4 15:18:11 2007 @@ -114,6 +114,9 @@ def fset_string_position(space, self, w_value): self.start = space.int_w(w_value) + def get_char_ord(self, p): + raise NotImplementedError + getset_start = GetSetProperty(W_State.fget_start, W_State.fset_start, cls=W_State) getset_string_position = GetSetProperty(W_State.fget_string_position, W_State.fset_string_position, cls=W_State) Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Sun Nov 4 15:18:11 2007 @@ -125,6 +125,8 @@ "formatted float is too long (precision too large?)")) self.std_wp_number(r) + def std_wp_number(self, r, prefix=''): + raise NotImplementedError def make_formatter_subclass(do_unicode): # to build two subclasses of the BaseStringFormatter class, From fijal at codespeak.net Sun Nov 4 15:18:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 15:18:28 +0100 (CET) Subject: [pypy-svn] r48288 - pypy/dist/pypy/interpreter Message-ID: <20071104141828.587188219@code0.codespeak.net> Author: fijal Date: Sun Nov 4 15:18:27 2007 New Revision: 48288 Modified: pypy/dist/pypy/interpreter/pyopcode.py Log: Another harmless warning Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sun Nov 4 15:18:27 2007 @@ -1073,6 +1073,11 @@ next_instr = self.really_handle(frame, unroller) # JIT hack return hint(next_instr, promote=True) + def really_handle(self, frame, unroller): + """ Purely abstract method + """ + raise NotImplementedError + class LoopBlock(FrameBlock): """A loop block. Stores the end-of-loop pointer in case of 'break'.""" From fijal at codespeak.net Sun Nov 4 15:38:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 15:38:13 +0100 (CET) Subject: [pypy-svn] r48289 - in pypy/dist/pypy/interpreter: astcompiler pyparser Message-ID: <20071104143813.BF956823F@code0.codespeak.net> Author: fijal Date: Sun Nov 4 15:38:11 2007 New Revision: 48289 Modified: pypy/dist/pypy/interpreter/astcompiler/ast.py pypy/dist/pypy/interpreter/pyparser/grammar.py Log: Make annotator happy. Modified: pypy/dist/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/dist/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/dist/pypy/interpreter/astcompiler/ast.py Sun Nov 4 15:38:11 2007 @@ -73,6 +73,9 @@ lst = self.getChildNodes() return space.newlist( [ space.wrap( it ) for it in lst ] ) + def get_value(self): + pass + def descr_node_accept( space, w_self, w_visitor ): return space.call_method( w_visitor, 'visitNode', w_self ) Modified: pypy/dist/pypy/interpreter/pyparser/grammar.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/grammar.py (original) +++ pypy/dist/pypy/interpreter/pyparser/grammar.py Sun Nov 4 15:38:11 2007 @@ -68,6 +68,8 @@ """Returns the source text between two scanner positions""" return "" + def peek(self): + pass ###################################################################### From pedronis at codespeak.net Sun Nov 4 15:48:18 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 4 Nov 2007 15:48:18 +0100 (CET) Subject: [pypy-svn] r48290 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104144818.0051681EC@code0.codespeak.net> Author: pedronis Date: Sun Nov 4 15:48:18 2007 New Revision: 48290 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: check in my what is PyPy slides Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 15:48:18 2007 @@ -5,6 +5,42 @@ ======================================================================== +What is PyPy +================== + +PyPy is a tool-chain for constructing dynamic languages. + +Interpreters in a relatively high-level language, without low-level +details are the easiest, most evolvable and portable way to implement +such languages. + +What PyPy does is setup enough infrastructure such that speed is regained +and features requiring low-level manipulations are (re-)added as aspects +without cluttering the intepreter. + +Targets as different as C and the industry OO VMs (JVM, CLR) are supported. + + +What is PyPy +============== + +PyPy is a reaction to the frustrations, resource problems and +duplicated efforts of how mainstream open-source languages (like +Python) are implemented now. + +We want the tool-chain itself to be as simple as possible. + +We operate both as an open source with production usage aspirations and +research project. + +We focus on the whole system. + +Some of what we do is relatively straight-forward, some is challenging +(generating dynamic compilers!). + + + + What is PyPy? ============== @@ -51,6 +87,20 @@ There has got to be a better way. +Translation: Going from interpreters to VMs +============================================== + +In PyPy interpreteres are written in RPython: +a subset of Python amenable to static analysis. +RPython itself still has garbage collection support +and rich built-in types. + +The tool-chain implements good static compilation +of RPython to multiple targets. + +It has pluggable backends, and implements so called +translation aspects. + Translation details ======================= From fijal at codespeak.net Sun Nov 4 16:12:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 16:12:36 +0100 (CET) Subject: [pypy-svn] r48291 - pypy/dist/pypy/rpython Message-ID: <20071104151236.0CFBD8217@code0.codespeak.net> Author: fijal Date: Sun Nov 4 16:12:34 2007 New Revision: 48291 Modified: pypy/dist/pypy/rpython/rpbc.py Log: Be on the safe side. Attribute when non-existant is not needed on the result desc, but should not generate a warning anyway. Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Sun Nov 4 16:12:34 2007 @@ -481,9 +481,12 @@ if r_value.lowleveltype is Void: continue try: - thisattrvalue = frozendesc.read_attribute(attr) - except AttributeError: - warning("Desc %r has no attribute %r" % (frozendesc, attr)) + thisattrvalue = frozendesc.attrcache[attr] + except KeyError: + # don't issue warning if this attribute can be read, but + # is not used + if not getattr(frozendesc.pyobj, attr, None): + warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) setattr(result, mangled_name, llvalue) From pedronis at codespeak.net Sun Nov 4 16:24:38 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 4 Nov 2007 16:24:38 +0100 (CET) Subject: [pypy-svn] r48292 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104152438.DCC6281BF@code0.codespeak.net> Author: pedronis Date: Sun Nov 4 16:24:38 2007 New Revision: 48292 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: tweak and shuffle Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 16:24:38 2007 @@ -20,7 +20,6 @@ Targets as different as C and the industry OO VMs (JVM, CLR) are supported. - What is PyPy ============== @@ -30,49 +29,36 @@ We want the tool-chain itself to be as simple as possible. -We operate both as an open source with production usage aspirations and -research project. - -We focus on the whole system. - -Some of what we do is relatively straight-forward, some is challenging -(generating dynamic compilers!). - - - - -What is PyPy? -============== - -PyPy is a tool-chain with a *new paradigm* for building interpreters: - -It challenges commonly held assumptions about the -construction of interpreters for dynamic languages. Folk Wisdom about Interpreters for Dynamic Languages ==================================================== * There are unavoidable tradeoffs between flexibilty, maintainability, and speed -* Popular dynamic languages such as Python and Ruby are the best we - can do given these tradeoffs and constraints. - * Fast, Maintainable, Flexible -- pick one What this means in Practice =========================== -Dynamic languages are not very flexible. +Current popular open source dynamic language implemtations: + +* are relatively slow + +* are not very flexible: - Low-level decisions permeate the entire code base. - One cannot simply plug-in a new garbage collector, or threading model when one desires to experiment. - Early decisions come back to haunt you. -Dynamic languages are hard to maintain. +What this means in Practice (2) +================================== -- because they are traditionally written in low-level languages, they are - harder to write than they need be. +.. XXX too big + +* are harder to maintain than we would like them to be + +- because they are traditionally written in low-level languages - the language community, which generates experts in the dynamic langauge, can not use this expertise in its own maintenance. Instead, expertise in C, or C++ is usually needed. @@ -80,12 +66,18 @@ The maintainers of Jython and IronPython, for instance, are lost to the C Python community. They have enough to do just to keep up with C Python. -Dynamic Languages are Slow. +PyPy as a project +=================== + +We operate both as an open source with production usage aspirations and +research project. + +We focus on the whole system. + +Some of what we do is relatively straight-forward, some is challenging +(generating dynamic compilers!). -Thus it looks like what we really have is: -* Fast, Maintainable, Flexible -- you get NONE. -There has got to be a better way. Translation: Going from interpreters to VMs ============================================== From pedronis at codespeak.net Sun Nov 4 16:31:43 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 4 Nov 2007 16:31:43 +0100 (CET) Subject: [pypy-svn] r48293 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104153143.8B2658232@code0.codespeak.net> Author: pedronis Date: Sun Nov 4 16:31:41 2007 New Revision: 48293 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: fix some typos Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 16:31:41 2007 @@ -16,7 +16,7 @@ What PyPy does is setup enough infrastructure such that speed is regained and features requiring low-level manipulations are (re-)added as aspects -without cluttering the intepreter. +without cluttering the interpreter. Targets as different as C and the industry OO VMs (JVM, CLR) are supported. @@ -32,7 +32,7 @@ Folk Wisdom about Interpreters for Dynamic Languages ==================================================== -* There are unavoidable tradeoffs between flexibilty, maintainability, +* There are unavoidable tradeoffs between flexibility, maintainability, and speed * Fast, Maintainable, Flexible -- pick one @@ -40,7 +40,7 @@ What this means in Practice =========================== -Current popular open source dynamic language implemtations: +Current popular open source dynamic language implementations: * are relatively slow @@ -59,7 +59,7 @@ * are harder to maintain than we would like them to be - because they are traditionally written in low-level languages -- the language community, which generates experts in the dynamic langauge, +- the language community, which generates experts in the dynamic language, can not use this expertise in its own maintenance. Instead, expertise in C, or C++ is usually needed. - every time a new VM is needed, there is a fork in the language community. @@ -82,7 +82,7 @@ Translation: Going from interpreters to VMs ============================================== -In PyPy interpreteres are written in RPython: +In PyPy interpreters are written in RPython: a subset of Python amenable to static analysis. RPython itself still has garbage collection support and rich built-in types. From arigo at codespeak.net Sun Nov 4 16:44:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Nov 2007 16:44:09 +0100 (CET) Subject: [pypy-svn] r48294 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104154409.B89528242@code0.codespeak.net> Author: arigo Date: Sun Nov 4 16:44:09 2007 New Revision: 48294 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Title slide. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 16:44:09 2007 @@ -1,9 +1,14 @@ .. include:: ======================================================================== -PyPy - automatic generation of VMs for dynamic languages - JIT included + PyPy ======================================================================== +Automatic generation of VMs for dynamic languages - JIT included +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:authors: Samuele Pedroni, Armin Rigo, Laura Creighton, Jacob Hall?n + What is PyPy ================== From jacob at codespeak.net Sun Nov 4 16:46:46 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Sun, 4 Nov 2007 16:46:46 +0100 (CET) Subject: [pypy-svn] r48295 - pypy/extradoc/talk/roadshow-google Message-ID: <20071104154646.759AB8246@code0.codespeak.net> Author: jacob Date: Sun Nov 4 16:46:45 2007 New Revision: 48295 Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp Log: Added an extra picture, to replace 4 others, in shorter talks. Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp ============================================================================== Binary files. No diff available. From jacob at codespeak.net Sun Nov 4 16:47:12 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Sun, 4 Nov 2007 16:47:12 +0100 (CET) Subject: [pypy-svn] r48296 - pypy/extradoc/talk/roadshow-google Message-ID: <20071104154712.8DE40824D@code0.codespeak.net> Author: jacob Date: Sun Nov 4 16:47:09 2007 New Revision: 48296 Added: pypy/extradoc/talk/roadshow-google/future.txt Log: A slide about the future. Added: pypy/extradoc/talk/roadshow-google/future.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/roadshow-google/future.txt Sun Nov 4 16:47:09 2007 @@ -0,0 +1,12 @@ +Where do we go from here? +========================= + +Focus on reaching production quality in a single field: + +- Viable alternative to CPython + +- Massive parallelism + +- High security environments + +- JVM From arigo at codespeak.net Sun Nov 4 19:31:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Nov 2007 19:31:41 +0100 (CET) Subject: [pypy-svn] r48297 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104183141.BA45A818D@code0.codespeak.net> Author: arigo Date: Sun Nov 4 19:31:39 2007 New Revision: 48297 Added: pypy/extradoc/talk/roadshow-ibm/flowgraph.png (contents, props changed) pypy/extradoc/talk/roadshow-ibm/overview1.png (contents, props changed) Modified: pypy/extradoc/talk/roadshow-ibm/overview2.png pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Review the slides. Slidify pages. Add a couple of diagrams. Added: pypy/extradoc/talk/roadshow-ibm/flowgraph.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/roadshow-ibm/overview1.png ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/roadshow-ibm/overview2.png ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 19:31:39 2007 @@ -7,36 +7,86 @@ Automatic generation of VMs for dynamic languages - JIT included ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:authors: Samuele Pedroni, Armin Rigo, Laura Creighton, Jacob Hall?n +.. raw:: html + +
+
+ + + + + +
Samuele Pedroni   Laura Creighton
Armin RigoJacob Hall?n
+
+ + +PyPy +================== + +.. raw:: html +
+
+
+ +**PyPy is a tool-chain for constructing dynamic languages.** + +.. raw:: html + +
-What is PyPy +Interpreters ================== -PyPy is a tool-chain for constructing dynamic languages. +...are good to implement dynamic languages: + +* Easy to write + +* Portable -Interpreters in a relatively high-level language, without low-level -details are the easiest, most evolvable and portable way to implement -such languages. - -What PyPy does is setup enough infrastructure such that speed is regained -and features requiring low-level manipulations are (re-)added as aspects -without cluttering the interpreter. +* Flexible and easy to evolve, if written in high-level language + (without low-level details) + +The PyPy Project +================== + +We built enough infrastructure such that: + +* speed is regained + +* features requiring low-level manipulations are (re-)added as *aspects* + +* interpreters are kept simple and uncluttered Targets as different as C and the industry OO VMs (JVM, CLR) are supported. -What is PyPy -============== +PyPy as a project +=================== + +We operate both as an open source with production usage aspirations and +research project. + +We focus on the whole system. + +We want the tool-chain itself to be as simple as possible +(but not simpler). + +Some of what we do is relatively straight-forward, some is challenging +(generating dynamic compilers!). + +The Origin of PyPy +===================== PyPy is a reaction to the frustrations, resource problems and duplicated efforts of how mainstream open-source languages (like Python) are implemented now. -We want the tool-chain itself to be as simple as possible. - -Folk Wisdom about Interpreters for Dynamic Languages +Folk Wisdom ==================================================== + +...about interpreters for Dynamic Languages: + * There are unavoidable tradeoffs between flexibility, maintainability, and speed @@ -49,108 +99,175 @@ * are relatively slow -* are not very flexible: +* are not very flexible + +* are harder to maintain than we would like them to be + +Not very flexible +================= - Low-level decisions permeate the entire code base. -- One cannot simply plug-in a new garbage collector, or threading model - when one desires to experiment. +- Not ideal to experiment - cannot simply plug-in a new garbage collector, + memory model, or threading model - Early decisions come back to haunt you. -What this means in Practice (2) -================================== +Hard to maintain +================ -.. XXX too big +because they are traditionally written in low-level languages: -* are harder to maintain than we would like them to be +- the community generates experts in the dynamic language but + requires experts in C or C++ for its own maintenance +- every time a new VM is needed, the language's community forks + (CPython - Jython - IronPython) -- because they are traditionally written in low-level languages -- the language community, which generates experts in the dynamic language, - can not use this expertise in its own maintenance. Instead, expertise - in C, or C++ is usually needed. -- every time a new VM is needed, there is a fork in the language community. - The maintainers of Jython and IronPython, for instance, are lost to the - C Python community. They have enough to do just to keep up with C Python. -PyPy as a project -=================== +PyPy Approach +============================= -We operate both as an open source with production usage aspirations and -research project. +.. raw:: html -We focus on the whole system. +
-Some of what we do is relatively straight-forward, some is challenging -(generating dynamic compilers!). +.. image:: overview1.png + :align: center +Translation +============================================== +.. raw:: html -Translation: Going from interpreters to VMs -============================================== +
+ +Going from interpreters to VMs +------------------------------ In PyPy interpreters are written in RPython: -a subset of Python amenable to static analysis. -RPython itself still has garbage collection support -and rich built-in types. -The tool-chain implements good static compilation +* A subset of Python amenable to static analysis + +* Still fully garbage collected + +* Rich built-in types + +RPython is still close to Python. + +Translation +============================================== + +.. raw:: html + +
+ +Going from interpreters to VMs (2) +---------------------------------- + +The translation tool-chain implements good static compilation of RPython to multiple targets. -It has pluggable backends, and implements so called -translation aspects. +It has pluggable backends, and inserts low-level details +as needed (*translation aspects*). Translation details ======================= -- RPython translation starts from loaded and initialized RPython - code as python bytecode in a Python VM. +.. raw:: html + +
-- PyPy uses abstract interpretation extensively: to construct flow-graphs, - for type inference, to gather information for some optimisations +- First, load and initialize RPython code inside a normal Python VM -- Obviously in the PE based generated Dynamic Compilers +- RPython translation starts from the resulting "live" bytecode -- Flow-graph transformation and rewriting is also used +- Unified "intermediate code" representation: + a forest of *Control Flow Graphs* + +.. raw:: html + +  
+ +Translation details (2) +======================= + +PyPy uses abstract interpretation extensively: + +- to construct Flow Graphs +- for type inference +- to gather info for some optimisations +- for Partial Evaluation in the generated Dynamic Compilers... + +also uses Flow Graph transformation and rewriting. Representation choice ======================== -A complex part of RPython translation is choosing implementations -and representations of its still rich built-in types that work for -the target platforms (C/LLVM vs. OO VMs) +A complex part of RPython translation: + +* RPython types are still rich + +* we have to choose implementations and representations + that work for the target platforms (C vs. OO VMs) + +Type Systems +========================= + +We model the different targets through different type systems: + +- LL (low-level C-like targets): data and function pointers, structures, + arrays... + +- OO (object oriented targets): classes and instances + with inheritance and dispatching + +Type systems (2) +=========================== -We model the classes of targets through different type systems: +Translation: -- low-level: data and function pointers, structures, ... -- object oriented: classes, instances, inheritance and dispatching +* starts from *RPython Flow Graphs* + +* turns them into *LL Flow Graphs* or *OO Flow Graphs* + +* which are then sent to the backends. Type systems and helpers =========================== -We have emulation of the type systems that can run on top of CPython, -we use them for testing but also for: +We have emulation of the type systems that can run on top of CPython +for testing but also for: -- constructing and representing the static data that our approach involves - (we start from live objects) at translation time +- constructing and representing the prebuilt data that our approach involves + (we start from live objects) -- the implementation of built-in type for the targets require helper functions: - they are expressed using the emulations which our translation knows about - and can translate too +- helper functions (e.g. implementations of RPython types) + use the emulations which our translation knows about too Translation aspects ======================== -- The interpreters that we write should be free of low-level details - (this is also required to target platforms as different as Posix/C - and the JVM/.NET) +The interpreters in RPython are free of low-level details +(as required to target platforms as different as Posix/C +and the JVM/.NET). - Advanced features related to execution should not need wide-spread changes to the interpreters -- The interpreters instead should use support offered and inserted by +- Instead, the interpreters should use support from the translation framework -Examples: GC and memory management, stack inspection and manipulation +Translation aspects (2) +======================== + +Examples: + +- GC and memory management + +- memory layout + +- stack inspection and manipulation + +- unboxed integers as tagged pointers Implementation ================== @@ -158,38 +275,52 @@ - Translation aspects are implemented as transformation of low-level graphs -- Calls to library/helper code written in RPython can be inserted - too which will also be analyzed and translated +- Calls to library/helper code can be inserted too + +- The helper code is also written in RPython and analyzed and translated GC Framework =============== -- RPython has been extended with allocation and address manipulation - primitives that can be used to express GC in RPython directly +The LL Type System is extended with allocation and address manipulation +primitives, used to express GC in RPython directly. - GCs are linked by substituting memory allocation operations with calls into them -- Right now bookkeeping code to keep track of reference counting or - roots is inserted by the GC framework +- Transformation inserts bookkeeping code, e.g. to keep track of roots -- Inlining is used to eliminate call overhead for the fast paths of - allocation and barriers +- Inline fast paths of allocation and barriers .. MMTk reference Stackless transformation ========================= -- One translation aspect transformation inserts support code - around calls such that the stack can be unwound and functions asked to - store and reflect their current activation frame state to the heap +Inserts support code around calls such that the stack can be unwound. + +- Functions can store their current activation frame state to the heap + +- Chains of saved activation state can be resumed + +We have implemented coroutine switching using this. + +A special aspect +================================== + +.. raw:: html + +
+
+
-- Chains of saved activation state can then be resumed +**Generating JIT compilers** -- We have implemented coroutine switching using this +.. raw:: html + +
-A special aspect: JIT generation +JIT motivation ================================== Flexibility vs. Performance: @@ -277,8 +408,7 @@ - a few hints in the Python interpreter to guide the JIT generator - *promotion* -- lazy allocation of objects - only when they escape - ("virtuals") +- lazy allocation of objects (only on escape) - use CPU stack and registers for the contents of the Python frame .. ("virtualizables") @@ -308,9 +438,24 @@ .. demo f1 + EXTRA MATERIAL ================== +* More about the JIT Generation: + + - The *Timeshifting* transformation + - *Virtuals* and *Promotion* + +* More on the Stackless transformation + + - *Resume points* + +* More on any other part that you are interested in + +* More demos + + The transformation ================================== @@ -320,7 +465,7 @@ * Guided by a binding time analysis ("color" of the graphs) -*"timeshifting"* +* *"timeshifting"* Coloring ================= @@ -479,6 +624,8 @@ Resume points =============== +Based on the Stackless Transformation: + - this transformation can also insert code that allows to construct artificial chains of activation states corresponding to labeled points in the program From arigo at codespeak.net Sun Nov 4 19:37:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Nov 2007 19:37:40 +0100 (CET) Subject: [pypy-svn] r48298 - pypy/extradoc/talk/roadshow-ibm Message-ID: <20071104183740.61C2E8239@code0.codespeak.net> Author: arigo Date: Sun Nov 4 19:37:40 2007 New Revision: 48298 Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt Log: Add url. Modified: pypy/extradoc/talk/roadshow-ibm/talk.txt ============================================================================== --- pypy/extradoc/talk/roadshow-ibm/talk.txt (original) +++ pypy/extradoc/talk/roadshow-ibm/talk.txt Sun Nov 4 19:37:40 2007 @@ -17,6 +17,7 @@ Armin Rigo Jacob Hall?n + http://codespeak.net/pypy/ From fijal at codespeak.net Sun Nov 4 19:39:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Nov 2007 19:39:38 +0100 (CET) Subject: [pypy-svn] r48299 - pypy/dist/pypy/rpython Message-ID: <20071104183938.9CA4881A6@code0.codespeak.net> Author: fijal Date: Sun Nov 4 19:39:37 2007 New Revision: 48299 Modified: pypy/dist/pypy/rpython/rpbc.py Log: check for attribute existance, rather than value Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Sun Nov 4 19:39:37 2007 @@ -485,7 +485,7 @@ except KeyError: # don't issue warning if this attribute can be read, but # is not used - if not getattr(frozendesc.pyobj, attr, None): + if not hasattr(frozendesc.pyobj, attr): warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) From akuhn at codespeak.net Sun Nov 4 20:39:25 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Sun, 4 Nov 2007 20:39:25 +0100 (CET) Subject: [pypy-svn] r48301 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071104193925.586218187@code0.codespeak.net> Author: akuhn Date: Sun Nov 4 20:39:23 2007 New Revision: 48301 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py pypy/dist/pypy/lang/smalltalk/model.py Log: read interpreter.py and added comments where stuff was unclear. Checked in the blue book, but did not find where doesNotUnderstand: is looked up for super sends. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Sun Nov 4 20:39:23 2007 @@ -42,11 +42,13 @@ def step(self): next = self.w_active_context.getNextBytecode() + # AK please explain this cond if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] if self._w_last_active_context != self.w_active_context: cnt = 0 p = self.w_active_context + # AK make method while p is not None: cnt += 1 p = p.w_sender @@ -63,6 +65,7 @@ next, bytecodeimpl.__name__,) bytecodeimpl(self.w_active_context, self) else: + # AK does not understand below for code, bytecodeimpl in unrolling_bytecode_table: if code == next: bytecodeimpl(self.w_active_context, self) @@ -78,6 +81,7 @@ # # "self" is always a W_ContextPart instance. +# AK does not understand __extend__ class __extend__(W_ContextPart): # push bytecodes def pushReceiverVariableBytecode(self, interp): @@ -99,6 +103,7 @@ index = self.currentBytecode & 31 association = self.w_method().getliteral(index) assert isinstance(association, model.W_PointersObject) + assert association.size() == 2 self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) def storeAndPopReceiverVariableBytecode(self, interp): @@ -166,6 +171,7 @@ assert argcount >= 0 method = receiverclassshadow.lookup(selector) # XXX catch MethodNotFound here and send doesNotUnderstand: + # AK shouln't that be done in lookup itself, please check what spec says about DNU in case of super sends. if method.primitive: # the primitive pushes the result (if any) onto the stack itself code = method.primitive @@ -224,6 +230,7 @@ raise MissingBytecode("unknownBytecode") def extendedVariableTypeAndIndex(self): + # AK please explain this method (a helper, I guess) descriptor = self.getbytecode() return ((descriptor >> 6) & 3), (descriptor & 63) @@ -239,7 +246,9 @@ association = self.w_method().getliteral(variableIndex) assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) - + else: + assert 0 + def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Sun Nov 4 20:39:23 2007 @@ -302,8 +302,9 @@ self.tempsize = tempsize self.primitive = primitive - def compiledin(self): + def compiledin(self): if self.w_compiledin is None: + # (Blue book, p 607) All CompiledMethods that contain extended-super bytecodes have the clain which they are found as their last literal variable. # Last of the literals is an association with compiledin # as a class association = self.literals[-1] From cfbolz at codespeak.net Sun Nov 4 22:46:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 4 Nov 2007 22:46:45 +0100 (CET) Subject: [pypy-svn] r48302 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071104214645.8DFF881C0@code0.codespeak.net> Author: cfbolz Date: Sun Nov 4 22:46:44 2007 New Revision: 48302 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: try to replace some of adrian's questions by explanations what is going on. Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Sun Nov 4 22:46:44 2007 @@ -42,7 +42,8 @@ def step(self): next = self.w_active_context.getNextBytecode() - # AK please explain this cond + # we_are_translated returns false on top of CPython and true when + # translating the interpreter if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] if self._w_last_active_context != self.w_active_context: @@ -65,7 +66,10 @@ next, bytecodeimpl.__name__,) bytecodeimpl(self.w_active_context, self) else: - # AK does not understand below + # this is a performance optimization: when translating the + # interpreter, the bytecode dispatching is not implemented as a + # list lookup and an indirect call but as a switch. The for loop + # below produces the switch (by being unrolled). for code, bytecodeimpl in unrolling_bytecode_table: if code == next: bytecodeimpl(self.w_active_context, self) @@ -81,7 +85,7 @@ # # "self" is always a W_ContextPart instance. -# AK does not understand __extend__ +# __extend__ adds new methods to the W_ContextPart class class __extend__(W_ContextPart): # push bytecodes def pushReceiverVariableBytecode(self, interp): From fijal at codespeak.net Mon Nov 5 09:15:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 09:15:58 +0100 (CET) Subject: [pypy-svn] r48305 - pypy/dist/pypy/tool Message-ID: <20071105081558.2133281AA@code0.codespeak.net> Author: fijal Date: Mon Nov 5 09:15:57 2007 New Revision: 48305 Modified: pypy/dist/pypy/tool/gcc_cache.py Log: Make sure that _cache exists as well. I thought it's enough to make sure about child dir. Modified: pypy/dist/pypy/tool/gcc_cache.py ============================================================================== --- pypy/dist/pypy/tool/gcc_cache.py (original) +++ pypy/dist/pypy/tool/gcc_cache.py Mon Nov 5 09:15:57 2007 @@ -5,6 +5,7 @@ import py import distutils +py.path.local(pypydir).join('_cache').ensure(dir=1) cache_dir = py.path.local(pypydir).join('_cache', 'gcc') cache_dir.ensure(dir=1) From tismer at codespeak.net Mon Nov 5 10:02:52 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 5 Nov 2007 10:02:52 +0100 (CET) Subject: [pypy-svn] r48306 - pypy/dist/pypy/translator/goal Message-ID: <20071105090252.0CB2F81E9@code0.codespeak.net> Author: tismer Date: Mon Nov 5 10:02:52 2007 New Revision: 48306 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: I guess there is a library bug somewhere in python that gives __exitfunc problems when removing modules from sys.modules. I changed this to happen in compiled mode, only. Still, we should investigate this. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Mon Nov 5 10:02:52 2007 @@ -192,7 +192,7 @@ # for some reason, some tests break with this import from pypy.rlib.objectmodel import we_are_translated except ImportError: - we_are_translated = False + we_are_translated = lambda: False class fake_os: def __init__(self): @@ -222,9 +222,14 @@ assert self.getenv def teardown(self): - for mod in sys.modules.keys(): - if mod not in self.pre_import: - del sys.modules[mod] + # re-load modules instead of using the pre-compiled ones + # note that this gives trouble if we are not translated, + # since some exithandler in threading.py complains + # XXX check if this is a bug to be fixed for cpython + if we_are_translated(): + for mod in sys.modules.keys(): + if mod not in self.pre_import: + del sys.modules[mod] global os import os @@ -254,7 +259,7 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if we_are_translated and IS_WINDOWS and not fn.lower().endswith('.exe'): + if we_are_translated() and IS_WINDOWS and not fn.lower().endswith('.exe'): fn += '.exe' if os.path.isfile(fn): executable = fn From tismer at codespeak.net Mon Nov 5 10:56:50 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Mon, 5 Nov 2007 10:56:50 +0100 (CET) Subject: [pypy-svn] r48307 - pypy/dist/pypy/translator/goal Message-ID: <20071105095650.D9BF081A5@code0.codespeak.net> Author: tismer Date: Mon Nov 5 10:56:49 2007 New Revision: 48307 Modified: pypy/dist/pypy/translator/goal/app_main.py Log: handle .exe correctly, regardless if we search the path or not Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Mon Nov 5 10:56:49 2007 @@ -252,6 +252,8 @@ # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH os.setup() # this is the faked one + if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): + executable += '.exe' if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable): pass # the path is already more than just an executable name else: @@ -259,8 +261,6 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if we_are_translated() and IS_WINDOWS and not fn.lower().endswith('.exe'): - fn += '.exe' if os.path.isfile(fn): executable = fn break From fijal at codespeak.net Mon Nov 5 11:25:47 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 11:25:47 +0100 (CET) Subject: [pypy-svn] r48311 - in pypy/dist/pypy/annotation: . test Message-ID: <20071105102547.D3BAA81DC@code0.codespeak.net> Author: fijal Date: Mon Nov 5 11:25:47 2007 New Revision: 48311 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/classdef.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: Some support for annotating unicode strings in rpython. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Mon Nov 5 11:25:47 2007 @@ -19,7 +19,7 @@ from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import lltype_to_annotation from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.model import SomeExternalInstance +from pypy.annotation.model import SomeExternalInstance, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.annotation.listdef import ListDef @@ -636,12 +636,39 @@ getbookkeeper().count("str_mul", str1, int2) return SomeString() +class __extend__(pairtype(SomeUnicodeString, SomeInteger)): + def getitem((str1, int2)): + getbookkeeper().count("str_getitem", int2) + return SomeUnicodeCodePoint() + getitem.can_only_throw = [] + + getitem_key = getitem + + def getitem_idx((str1, int2)): + getbookkeeper().count("str_getitem", int2) + return SomeUnicodeCodePoint() + getitem_idx.can_only_throw = [IndexError] + + getitem_idx_key = getitem_idx + + # uncomment if we really want to support that + #def mul((str1, int2)): # xxx do we want to support this + # getbookkeeper().count("str_mul", str1, int2) + # return SomeString() + class __extend__(pairtype(SomeInteger, SomeString)): def mul((int1, str2)): # xxx do we want to support this getbookkeeper().count("str_mul", str2, int1) return SomeString() +class __extend__(pairtype(SomeString, SomeUnicodeString), + pairtype(SomeUnicodeString, SomeString), + pairtype(SomeUnicodeString, SomeUnicodeString)): + def union((str1, str2)): + return SomeUnicodeString(can_be_None=str1.can_be_None or + str2.can_be_None) + class __extend__(pairtype(SomeInteger, SomeList)): def mul((int1, lst2)): @@ -781,6 +808,7 @@ _make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') _make_none_union('SomeString', 'can_be_None=True') +_make_none_union('SomeUnicodeString', 'can_be_None=True') _make_none_union('SomeList', 'obj.listdef') _make_none_union('SomeDict', 'obj.dictdef') _make_none_union('SomeExternalObject', 'obj.knowntype') Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Mon Nov 5 11:25:47 2007 @@ -11,7 +11,8 @@ SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ - SomeList, SomeObject, HarmlesslyBlocked, SomeWeakRef, lltype_to_annotation + SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ + SomeWeakRef, lltype_to_annotation from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -322,8 +323,11 @@ result = SomeChar() else: result = SomeString() - elif tp is unicode and len(x) == 1: - result = SomeUnicodeCodePoint() + elif tp is unicode: + if len(x) == 1: + result = SomeUnicodeCodePoint() + else: + result = SomeUnicodeString() elif tp is tuple: result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x]) elif tp is float: Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Mon Nov 5 11:25:47 2007 @@ -6,7 +6,7 @@ from pypy.annotation.model import SomeInteger, SomeObject, SomeChar, SomeBool from pypy.annotation.model import SomeString, SomeTuple, SomeSlice, s_Bool from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress -from pypy.annotation.model import SomeFloat, unionof +from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict from pypy.annotation.model import SomeExternalObject from pypy.annotation.model import SomeWeakRef @@ -107,8 +107,8 @@ def builtin_unichr(s_int): return constpropagate(unichr, [s_int], SomeUnicodeCodePoint()) -##def builtin_unicode(s_obj): -## raise TypeError, "unicode() calls should not happen at interp-level" +def builtin_unicode(s_unicode): + return constpropagate(unicode, [s_unicode], SomeUnicodeString()) def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" Modified: pypy/dist/pypy/annotation/classdef.py ============================================================================== --- pypy/dist/pypy/annotation/classdef.py (original) +++ pypy/dist/pypy/annotation/classdef.py Mon Nov 5 11:25:47 2007 @@ -144,7 +144,7 @@ raise NoSuchAttrError(classdef, self.name) -class ClassDef: +class ClassDef(object): "Wraps a user class." def __init__(self, bookkeeper, classdesc): Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Mon Nov 5 11:25:47 2007 @@ -204,6 +204,19 @@ def nonnoneify(self): return SomeString(can_be_None=False) +class SomeUnicodeString(SomeObject): + "Stands for an object which is known to be an unicode string" + knowntype = unicode + immutable = True + def __init__(self, can_be_None=False): + self.can_be_None = can_be_None + + def can_be_none(self): + return self.can_be_None + + def nonnoneify(self): + return SomeUnicodeString(can_be_None=False) + class SomeChar(SomeString): "Stands for an object known to be a string of length 1." Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Mon Nov 5 11:25:47 2007 @@ -2872,6 +2872,39 @@ s = a.build_types(g, [int]) assert isinstance(s, annmodel.SomeSingleFloat) + def test_unicode_simple(self): + def f(): + return u'xxx' + + a = self.RPythonAnnotator() + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_unicode(self): + def g(n): + if n > 0: + return "xxx" + else: + return u"x\xe4x" + + def f(n): + x = g(0) + return x[n] + + a = self.RPythonAnnotator() + s = a.build_types(g, [int]) + assert isinstance(s, annmodel.SomeUnicodeString) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeUnicodeCodePoint) + + def test_unicode_from_string(self): + def f(x): + return unicode(x) + + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeUnicodeString) def g(n): return [0,1,2,n] From fijal at codespeak.net Mon Nov 5 11:26:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 11:26:41 +0100 (CET) Subject: [pypy-svn] r48312 - in pypy/dist/pypy/rpython: . lltypesystem ootypesystem test Message-ID: <20071105102641.B1AEA81DC@code0.codespeak.net> Author: fijal Date: Mon Nov 5 11:26:41 2007 New Revision: 48312 Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/test/test_llinterp.py Log: Some unicode support (constants & getitem notably). More support needs fixes here and there. Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Mon Nov 5 11:26:41 2007 @@ -12,7 +12,7 @@ from pypy.rpython.lltypesystem import ll_str from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, UniChar, Ptr, malloc, \ - Bool, Void, GcArray, nullptr, pyobjectptr + Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive # ____________________________________________________________ @@ -27,29 +27,28 @@ STR = GcStruct('rpy_string', ('hash', Signed), ('chars', Array(Char, hints={'immutable': True, 'isrpystring': True}))) +UNICODE = GcStruct('rpy_unicode', ('hash', Signed), + ('chars', Array(UniChar, hints={'immutable': True}))) SIGNED_ARRAY = GcArray(Signed) CONST_STR_CACHE = WeakValueDictionary() -class StringRepr(AbstractStringRepr): - - lowleveltype = Ptr(STR) - +class BaseStringRepr(AbstractStringRepr): def __init__(self, *args): AbstractStringRepr.__init__(self, *args) self.ll = LLHelpers def convert_const(self, value): if value is None: - return nullptr(STR) + return nullptr(self.lowleveltype.TO) #value = getattr(value, '__self__', value) # for bound string methods - if not isinstance(value, str): + if not isinstance(value, self.basetype): raise TyperError("not a str: %r" % (value,)) try: return CONST_STR_CACHE[value] except KeyError: - p = mallocstr(len(value)) + p = self.malloc(len(value)) for i in range(len(value)): - p.chars[i] = value[i] + p.chars[i] = cast_primitive(self.base, value[i]) p.hash = 0 self.ll.ll_strhash(p) # precompute the hash CONST_STR_CACHE[value] = p @@ -72,6 +71,23 @@ v_items = hop.gendirectcall(LIST.ll_items, v_lst) return v_length, v_items +class StringRepr(BaseStringRepr): + lowleveltype = Ptr(STR) + basetype = str + base = Char + + def __init__(self, *args): + BaseStringRepr.__init__(self, *args) + self.malloc = mallocstr + +class UnicodeRepr(BaseStringRepr): + lowleveltype = Ptr(UNICODE) + basetype = basestring + base = UniChar + + def __init__(self, *args): + BaseStringRepr.__init__(self, *args) + self.malloc = mallocunicode class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char @@ -79,7 +95,6 @@ class UniCharRepr(AbstractUniCharRepr): lowleveltype = UniChar - class __extend__(pairtype(PyObjRepr, AbstractStringRepr)): def convert_from_to((r_from, r_to), v, llops): v_len = llops.gencapicall('PyString_Size', [v], resulttype=Signed) @@ -104,13 +119,18 @@ resulttype=pyobj_repr, _callable= lambda v: pyobjectptr(''.join(v.chars))) -def mallocstr(length): - debug_assert(length >= 0, "negative string length") - r = malloc(STR, length) - if not we_are_translated() or not malloc_zero_filled: - r.hash = 0 - return r -mallocstr._annspecialcase_ = 'specialize:semierased' +def new_malloc(TP): + def mallocstr(length): + debug_assert(length >= 0, "negative string length") + r = malloc(TP, length) + if not we_are_translated() or not malloc_zero_filled: + r.hash = 0 + return r + mallocstr._annspecialcase_ = 'specialize:semierased' + return mallocstr + +mallocstr = new_malloc(STR) +mallocunicode = new_malloc(UNICODE) # ____________________________________________________________ # @@ -760,8 +780,11 @@ unichar_repr = UniCharRepr() char_repr.ll = LLHelpers unichar_repr.ll = LLHelpers +unicode_repr = UnicodeRepr() emptystr = string_repr.convert_const("") +StringRepr.repr = string_repr +UnicodeRepr.repr = unicode_repr class StringIteratorRepr(AbstractStringIteratorRepr): Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Mon Nov 5 11:26:41 2007 @@ -300,6 +300,7 @@ char_repr.ll = LLHelpers unichar_repr.ll = LLHelpers emptystr = string_repr.convert_const("") +StringRepr.repr = string_repr class StringIteratorRepr(AbstractStringIteratorRepr): lowleveltype = ootype.Record({'string': string_repr.lowleveltype, Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Mon Nov 5 11:26:41 2007 @@ -7,7 +7,8 @@ from pypy.rpython.rtuple import AbstractTupleRepr from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython import rint -from pypy.rpython.lltypesystem.lltype import Signed, Bool, Void, UniChar +from pypy.rpython.lltypesystem.lltype import Signed, Bool, Void, UniChar,\ + cast_primitive class AbstractStringRepr(Repr): pass @@ -18,13 +19,19 @@ class AbstractUniCharRepr(Repr): pass - class __extend__(annmodel.SomeString): def rtyper_makerepr(self, rtyper): return rtyper.type_system.rstr.string_repr def rtyper_makekey(self): return self.__class__, +class __extend__(annmodel.SomeUnicodeString): + def rtyper_makerepr(self, rtyper): + return rtyper.type_system.rstr.unicode_repr + + def rtyper_makekey(self): + return self.__class__, + class __extend__(annmodel.SomeChar): def rtyper_makerepr(self, rtyper): return rtyper.type_system.rstr.char_repr @@ -50,7 +57,7 @@ return self.ll.ll_strfasthash def rtype_len(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = self.repr v_str, = hop.inputargs(string_repr) return hop.gendirectcall(self.ll.ll_strlen, v_str) @@ -244,7 +251,7 @@ class __extend__(pairtype(AbstractStringRepr, IntegerRepr)): def rtype_getitem((r_str, r_int), hop, checkidx=False): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = r_str.repr v_str, v_index = hop.inputargs(string_repr, Signed) if checkidx: if hop.args_s[1].nonneg: Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Mon Nov 5 11:26:41 2007 @@ -257,12 +257,12 @@ res = interpret(f, [3]) assert res == 3 -def test_unicode(): - def f(): - return u'Hello world' - res = interpret(f, [], someobjects=True) +##def test_unicode(): +## def f(): +## return u'Hello world' +## res = interpret(f, [], someobjects=True) - assert res._obj.value == u'Hello world' +## assert res._obj.value == u'Hello world' ##def test_unicode_split(): ## def f(): From fijal at codespeak.net Mon Nov 5 12:17:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 12:17:51 +0100 (CET) Subject: [pypy-svn] r48313 - pypy/dist/pypy/rpython/test Message-ID: <20071105111751.C17F28248@code0.codespeak.net> Author: fijal Date: Mon Nov 5 12:17:50 2007 New Revision: 48313 Added: pypy/dist/pypy/rpython/test/test_runicode.py (contents, props changed) Log: Add missing unicode test file (not very huge by now) Added: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/test/test_runicode.py Mon Nov 5 12:17:50 2007 @@ -0,0 +1,19 @@ + + +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin + +class BaseTestRUnicode(BaseRtypingTest): + def test_simple(self): + def f(n): + if n % 2 == 0: + x = 'xxx' + else: + x = u'x\u221Ex' + return x[n] + + for i in range(0, 3): + res = self.interpret(f, [i]) + assert res == f(i) + +class TestLLtype(BaseTestRUnicode, LLRtypeMixin): + pass From fijal at codespeak.net Mon Nov 5 12:25:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 12:25:55 +0100 (CET) Subject: [pypy-svn] r48315 - pypy/branch/pypy-rpython-unicode Message-ID: <20071105112555.0D76B81B4@code0.codespeak.net> Author: fijal Date: Mon Nov 5 12:25:54 2007 New Revision: 48315 Added: pypy/branch/pypy-rpython-unicode/ - copied from r48314, pypy/dist/pypy/ Log: Create new branch for rpython unicode support From cfbolz at codespeak.net Mon Nov 5 16:25:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Nov 2007 16:25:51 +0100 (CET) Subject: [pypy-svn] r48317 - pypy/branch/clr-module-improvements Message-ID: <20071105152551.8C8BA824A@code0.codespeak.net> Author: cfbolz Date: Mon Nov 5 16:25:50 2007 New Revision: 48317 Added: pypy/branch/clr-module-improvements/ - copied from r48316, pypy/dist/ Log: create a new branch for amit to experiment in. He is planning to improve the CLR module. From fijal at codespeak.net Mon Nov 5 21:05:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 21:05:57 +0100 (CET) Subject: [pypy-svn] r48325 - in pypy/branch/pypy-rpython-unicode: interpreter objspace/flow objspace/flow/test Message-ID: <20071105200557.8E41681BA@code0.codespeak.net> Author: fijal Date: Mon Nov 5 21:05:56 2007 New Revision: 48325 Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: Support for constant-folding of unichr Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py (original) +++ pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py Mon Nov 5 21:05:56 2007 @@ -910,6 +910,7 @@ ('issubtype', 'issubtype', 2, []), # not for old-style classes ('repr', 'repr', 1, ['__repr__']), ('str', 'str', 1, ['__str__']), + ('unichr', 'unichr', 1, []), ('len', 'len', 1, ['__len__']), ('hash', 'hash', 1, ['__hash__']), ('getattr', 'getattr', 2, ['__getattribute__']), Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Mon Nov 5 21:05:56 2007 @@ -149,6 +149,7 @@ ('issubtype', issubclass), ('repr', repr), ('str', str), + ('unichr', unichr), ('len', len), ('hash', hash), ('getattr', getattr), Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Mon Nov 5 21:05:56 2007 @@ -712,6 +712,12 @@ return x[s] graph = self.codetest(myfunc) + def test_unichr_constfold(self): + def myfunc(): + return unichr(1234) + graph = self.codetest(myfunc) + assert graph.startblock.exits[0].target is graph.returnblock + def test_getitem(self): def f(c, x): try: From fijal at codespeak.net Mon Nov 5 21:07:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Nov 2007 21:07:58 +0100 (CET) Subject: [pypy-svn] r48326 - in pypy/branch/pypy-rpython-unicode: interpreter objspace/flow objspace/flow/test Message-ID: <20071105200758.182F881BE@code0.codespeak.net> Author: fijal Date: Mon Nov 5 21:07:56 2007 New Revision: 48326 Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: Constant folding for unicode as well Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py (original) +++ pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py Mon Nov 5 21:07:56 2007 @@ -911,6 +911,7 @@ ('repr', 'repr', 1, ['__repr__']), ('str', 'str', 1, ['__str__']), ('unichr', 'unichr', 1, []), + ('unicode', 'unicode', 1, ['__unicode__']), ('len', 'len', 1, ['__len__']), ('hash', 'hash', 1, ['__hash__']), ('getattr', 'getattr', 2, ['__getattribute__']), Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Mon Nov 5 21:07:56 2007 @@ -150,6 +150,7 @@ ('repr', repr), ('str', str), ('unichr', unichr), + ('unicode', unicode), ('len', len), ('hash', hash), ('getattr', getattr), Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Mon Nov 5 21:07:56 2007 @@ -718,6 +718,13 @@ graph = self.codetest(myfunc) assert graph.startblock.exits[0].target is graph.returnblock + def test_unicode_constfold(self): + def myfunc(): + return unicode(1234) + graph = self.codetest(myfunc) + assert graph.startblock.exits[0].target is graph.returnblock + + def test_getitem(self): def f(c, x): try: From fijal at codespeak.net Tue Nov 6 00:54:50 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 00:54:50 +0100 (CET) Subject: [pypy-svn] r48327 - pypy/branch/pypy-rpython-unicode/objspace Message-ID: <20071105235450.2FC72824D@code0.codespeak.net> Author: fijal Date: Tue Nov 6 00:54:49 2007 New Revision: 48327 Modified: pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Log: Dummy descroperations, because otherwise something complains about lack of those Modified: pypy/branch/pypy-rpython-unicode/objspace/descroperation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/descroperation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Tue Nov 6 00:54:49 2007 @@ -291,6 +291,12 @@ if w_del is not None: space.get_and_call_function(w_del, w_obj) + def unichr(space, w_obj): + raise NotImplementedError + + def unicode(space, w_obj): + raise NotImplementedError + def cmp(space, w_v, w_w): if space.is_w(w_v, w_w): From fijal at codespeak.net Tue Nov 6 01:08:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 01:08:35 +0100 (CET) Subject: [pypy-svn] r48328 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071106000835.59A908254@code0.codespeak.net> Author: fijal Date: Tue Nov 6 01:08:34 2007 New Revision: 48328 Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Log: Proper support for unicode() and unichr() operations Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/binaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Tue Nov 6 01:08:34 2007 @@ -433,6 +433,9 @@ def union((uchr1, uchr2)): return SomeUnicodeCodePoint() + def add((chr1, chr2)): + return SomeUnicodeString() + class __extend__(pairtype(SomeString, SomeObject)): def mod((str, args)): @@ -664,10 +667,25 @@ class __extend__(pairtype(SomeString, SomeUnicodeString), pairtype(SomeUnicodeString, SomeString), + pairtype(SomeUnicodeCodePoint, SomeString), + pairtype(SomeString, SomeUnicodeCodePoint), + pairtype(SomeUnicodeCodePoint, SomeUnicodeString), + pairtype(SomeUnicodeString, SomeUnicodeCodePoint), + pairtype(SomeUnicodeString, SomeChar), + pairtype(SomeChar, SomeUnicodeString), + pairtype(SomeUnicodeCodePoint, SomeChar), + pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeString, SomeUnicodeString)): def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_None or - str2.can_be_None) + return SomeUnicodeString(can_be_None=str1.can_be_none() or + str2.can_be_none()) + + def add((str1, str2)): + # propagate const-ness to help getattr(obj, 'prefix' + const_name) + result = SomeUnicodeString() + if str1.is_immutable_constant() and str2.is_immutable_constant(): + result.const = str1.const + str2.const + return result class __extend__(pairtype(SomeInteger, SomeList)): Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Tue Nov 6 01:08:34 2007 @@ -2906,6 +2906,28 @@ s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeUnicodeString) + def test_unicode_add(self): + def f(x): + return unicode(x) + unichr(1234) + + def g(x): + return unichr(x) + unichr(2) + + def h(x): + return x + u'a' + + def i(x): + return unicode(x) + 'xxx' + + for func in f, h, i: + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeUnicodeString) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def g(n): return [0,1,2,n] Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Tue Nov 6 01:08:34 2007 @@ -9,7 +9,7 @@ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ SomeCTypesObject, s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable, SomeWeakRef + SomeGenericCallable, SomeWeakRef, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -25,7 +25,7 @@ 'iter', 'next', 'invert', 'type', 'issubtype', 'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct', 'ord', 'int', 'float', 'long', 'id', - 'neg_ovf', 'abs_ovf', 'hint']) + 'neg_ovf', 'abs_ovf', 'hint', 'unicode', 'unichr']) for opname in UNARY_OPERATIONS: missing_operation(SomeObject, opname) @@ -103,6 +103,10 @@ getbookkeeper().count('str', obj) return SomeString() + def unicode(obj): + getbookkeeper().count('unicode', obj) + return SomeUnicodeString() + def repr(obj): getbookkeeper().count('repr', obj) return SomeString() @@ -202,6 +206,10 @@ def invert(self): return SomeInteger(knowntype=self.knowntype) + def unichr(obj): + getbookkeeper().count('unichr', obj) + return SomeUnicodeCodePoint() + invert.can_only_throw = [] def pos(self): From fijal at codespeak.net Tue Nov 6 10:51:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 10:51:37 +0100 (CET) Subject: [pypy-svn] r48329 - pypy/branch/pypy-rpython-unicode/annotation Message-ID: <20071106095137.82ECA8258@code0.codespeak.net> Author: fijal Date: Tue Nov 6 10:51:36 2007 New Revision: 48329 Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Log: Ooops, one annotation too far Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/binaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Tue Nov 6 10:51:36 2007 @@ -673,8 +673,6 @@ pairtype(SomeUnicodeString, SomeUnicodeCodePoint), pairtype(SomeUnicodeString, SomeChar), pairtype(SomeChar, SomeUnicodeString), - pairtype(SomeUnicodeCodePoint, SomeChar), - pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeString, SomeUnicodeString)): def union((str1, str2)): return SomeUnicodeString(can_be_None=str1.can_be_none() or From santagada at codespeak.net Tue Nov 6 10:59:17 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 6 Nov 2007 10:59:17 +0100 (CET) Subject: [pypy-svn] r48330 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071106095917.DB09F8263@code0.codespeak.net> Author: santagada Date: Tue Nov 6 10:59:17 2007 New Revision: 48330 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: All rint tests pass, turning off isolation for now to make it return more stuff. Probably exceptions are the next step Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Nov 6 10:59:17 2007 @@ -35,6 +35,7 @@ prolog = """ import ctypes +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from os.path import join, dirname, realpath _c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) @@ -142,6 +143,7 @@ ll_to_res = %(ll_to_res)s __entrypoint__.restype = %(returntype)s """ + TO_CTYPES = {lltype.Bool: "ctypes.c_byte", lltype.SingleFloat: "ctypes.c_float", lltype.Float: "ctypes.c_double", @@ -216,6 +218,15 @@ elif T is lltype.UniChar: action = 'to_unichar' + elif T is lltype.Unsigned: + action = 'r_uint' + + elif T is lltype.SignedLongLong: + action = 'r_longlong' + + elif T is lltype.UnsignedLongLong: + action = 'r_ulonglong' + elif isinstance(T, lltype.Ptr) and T.TO is STR: action = 'to_str' Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Tue Nov 6 10:59:17 2007 @@ -15,10 +15,10 @@ ext_modules = [] # prevents resource leaking -use_isolate = True +use_isolate = False # if test can't be run using isolate, skip the test (useful for buildbots) -run_isolated_only = True +run_isolated_only = False from pypy import conftest From cfbolz at codespeak.net Tue Nov 6 11:04:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Nov 2007 11:04:16 +0100 (CET) Subject: [pypy-svn] r48331 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071106100416.750758269@code0.codespeak.net> Author: cfbolz Date: Tue Nov 6 11:04:15 2007 New Revision: 48331 Modified: pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/__builtin__/test/test_import.py Log: a (passing) test for the meta_path hook. clean up indentation in one piece of importing code. Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Tue Nov 6 11:04:15 2007 @@ -606,15 +606,15 @@ import sys meta_path = sys.meta_path for hook in meta_path: - loader = hook.find_module(fullname, path) - if loader: - return loader + loader = hook.find_module(fullname, path) + if loader: + return loader if path != None and type(path) == str: - pass - # XXX Check for frozen modules ? + pass + # XXX Check for frozen modules ? if path == None: - # XXX Check frozen - path = sys.path + # XXX Check frozen + path = sys.path path_hooks = sys.path_hooks importer_cache = sys.path_importer_cache importer = None Modified: pypy/dist/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_import.py Tue Nov 6 11:04:15 2007 @@ -520,3 +520,19 @@ finally: if old: os.environ['PYTHONPATH'] = old + +class AppTestImportHooks(object): + def test_meta_path(self): + tried_imports = [] + class Importer(object): + def find_module(self, fullname, path=None): + tried_imports.append((fullname, path)) + + import sys + try: + sys.meta_path.append(Importer()) + import datetime + assert len(tried_imports) == 1 + tried_imports[0][0] == "datetime" + finally: + sys.meta_path.pop() From fijal at codespeak.net Tue Nov 6 12:38:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 12:38:03 +0100 (CET) Subject: [pypy-svn] r48332 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071106113803.E760D81B8@code0.codespeak.net> Author: fijal Date: Tue Nov 6 12:38:02 2007 New Revision: 48332 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Some support for unicode strings. This is work in progress, not all tests are really testing what they should. Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Tue Nov 6 12:38:02 2007 @@ -12,7 +12,7 @@ from pypy.rpython.lltypesystem import ll_str from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, UniChar, Ptr, malloc, \ - Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive + Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive, typeOf # ____________________________________________________________ @@ -31,6 +31,7 @@ ('chars', Array(UniChar, hints={'immutable': True}))) SIGNED_ARRAY = GcArray(Signed) CONST_STR_CACHE = WeakValueDictionary() +CONST_UNICODE_CACHE = WeakValueDictionary() class BaseStringRepr(AbstractStringRepr): def __init__(self, *args): @@ -44,20 +45,21 @@ if not isinstance(value, self.basetype): raise TyperError("not a str: %r" % (value,)) try: - return CONST_STR_CACHE[value] + return self.CACHE[value] except KeyError: p = self.malloc(len(value)) for i in range(len(value)): p.chars[i] = cast_primitive(self.base, value[i]) p.hash = 0 self.ll.ll_strhash(p) # precompute the hash - CONST_STR_CACHE[value] = p + self.CACHE[value] = p return p def make_iterator_repr(self): return string_iterator_repr def can_ll_be_null(self, s_value): + # XXX unicode if self is string_repr: return s_value.can_be_none() else: @@ -75,6 +77,7 @@ lowleveltype = Ptr(STR) basetype = str base = Char + CACHE = CONST_STR_CACHE def __init__(self, *args): BaseStringRepr.__init__(self, *args) @@ -84,6 +87,7 @@ lowleveltype = Ptr(UNICODE) basetype = basestring base = UniChar + CACHE = CONST_UNICODE_CACHE def __init__(self, *args): BaseStringRepr.__init__(self, *args) @@ -163,9 +167,13 @@ class LLHelpers(AbstractLLHelpers): def ll_char_mul(ch, times): + if typeOf(ch) is Char: + malloc = mallocstr + else: + malloc = mallocunicode if times < 0: times = 0 - newstr = mallocstr(times) + newstr = malloc(times) j = 0 while j < times: newstr.chars[j] = ch @@ -183,7 +191,11 @@ ll_stritem_nonneg._annenforceargs_ = [None, int] def ll_chr2str(ch): - s = mallocstr(1) + if typeOf(ch) is Char: + malloc = mallocstr + else: + malloc = mallocunicode + s = malloc(1) s.chars[0] = ch return s @@ -204,7 +216,12 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - newstr = mallocstr(len1 + len2) + if typeOf(s1).TO.chars.OF is Char and typeOf(s2).TO.chars.OF is Char: + malloc = mallocstr + else: + malloc = mallocunicode + + newstr = malloc(len1 + len2) j = 0 while j < len1: newstr.chars[j] = s1.chars[j] Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Tue Nov 6 12:38:02 2007 @@ -47,6 +47,9 @@ class __extend__(AbstractStringRepr): + def _str_reprs(self, hop): + return hop.args_r[0].repr, hop.args_r[1].repr + def get_ll_eq_function(self): return self.ll.ll_streq @@ -64,7 +67,7 @@ def rtype_is_true(self, hop): s_str = hop.args_s[0] if s_str.can_be_None: - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) return hop.gendirectcall(self.ll.ll_str_is_true, v_str) else: @@ -72,25 +75,26 @@ return super(AbstractStringRepr, self).rtype_is_true(hop) def rtype_ord(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) c_zero = inputconst(Signed, 0) v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) def rtype_method_startswith(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str, v_value = hop.inputargs(string_repr, string_repr) + str1_repr, str2_repr = self._str_reprs(hop) + v_str, v_value = hop.inputargs(str1_repr, str2_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_startswith, v_str, v_value) def rtype_method_endswith(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str, v_value = hop.inputargs(string_repr, string_repr) + str1_repr, str2_repr = self._str_reprs(hop) + v_str, v_value = hop.inputargs(str1_repr, str2_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_endswith, v_str, v_value) def rtype_method_find(self, hop, reverse=False): + # XXX binaryop rstr = hop.rtyper.type_system.rstr v_str = hop.inputarg(rstr.string_repr, arg=0) if hop.args_r[1] == rstr.char_repr: @@ -294,13 +298,20 @@ return hop.gendirectcall(r_str.ll.ll_stringslice_minusone, v_str) raise TyperError(r_slic) +def most_general_strrepr(repr1, repr2, rtyper): + string_repr = rtyper.type_system.rstr.string_repr + if repr1 is string_repr and repr2 is string_repr: + return string_repr + return rtyper_type_system.rstr.unicode_repr class __extend__(pairtype(AbstractStringRepr, AbstractStringRepr)): def rtype_add((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + str1_repr = r_str1.repr + str2_repr = r_str2.repr if hop.s_result.is_constant(): - return hop.inputconst(string_repr, hop.s_result.const) - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + res_repr = most_general_strrepr(str1_repr, str2_repr, hop.rtyper) + return hop.inputconst(res_repr, hop.s_result.const) + v_str1, v_str2 = hop.inputargs(str1_repr, str2_repr) return hop.gendirectcall(r_str1.ll.ll_strconcat, v_str1, v_str2) rtype_inplace_add = rtype_add Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Tue Nov 6 12:38:02 2007 @@ -14,26 +14,31 @@ assert parse('%s') == [('s',)] assert parse("name '%s' is not defined") == ["name '", ("s",), "' is not defined"] -class BaseTestRstr(BaseRtypingTest): - +class AbstractTestRstr(BaseRtypingTest): def test_simple(self): + const = self.const def fn(i): - s = 'hello' + s = const('hello') return s[i] for i in range(5): res = self.interpret(fn, [i]) - assert res == 'hello'[i] + expected = fn(i) + assert res == expected + assert res.__class__ is expected.__class__ def test_implicit_index_error(self): + const = self.const def fn(i): - s = 'hello' + s = const('hello') try: return s[i] except IndexError: - return '*' + return const('*') for i in range(-5, 5): res = self.interpret(fn, [i]) - assert res == 'hello'[i] + expected = fn(i) + assert res == expected + assert res.__class__ is expected.__class__ res = self.interpret(fn, [5]) assert res == '*' res = self.interpret(fn, [6]) @@ -42,8 +47,9 @@ assert res == '*' def test_nonzero(self): + const = self.const def fn(i, j): - s = ['', 'xx'][j] + s = [const(''), const('xx')][j] if i < 0: s = None if i > -2: @@ -56,9 +62,10 @@ assert res is fn(i, j) def test_concat(self): + const = self.const def fn(i, j): - s1 = ['', 'a', 'ab'] - s2 = ['', 'x', 'xy'] + s1 = [const(''), const('a'), const('ab')] + s2 = [const(''), const('x'), const('xy')] return s1[i] + s2[j] for i in range(3): for j in range(3): @@ -779,6 +786,9 @@ res = interpret(g, [-2]) assert res._obj.value == 42 +class BaseTestRstr(AbstractTestRstr): + const = str + class TestLLtype(BaseTestRstr, LLRtypeMixin): EMPTY_STRING_HASH = -1 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Tue Nov 6 12:38:02 2007 @@ -1,19 +1,12 @@ -from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin +from pypy.rpython.test.tool import LLRtypeMixin +from pypy.rpython.test.test_rstr import AbstractTestRstr -class BaseTestRUnicode(BaseRtypingTest): - def test_simple(self): - def f(n): - if n % 2 == 0: - x = 'xxx' - else: - x = u'x\u221Ex' - return x[n] +# ====> test_rstr.py - for i in range(0, 3): - res = self.interpret(f, [i]) - assert res == f(i) +class BaseTestRUnicode(AbstractTestRstr): + const = unicode class TestLLtype(BaseTestRUnicode, LLRtypeMixin): pass From fijal at codespeak.net Tue Nov 6 12:39:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 12:39:38 +0100 (CET) Subject: [pypy-svn] r48333 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071106113938.4C71A8221@code0.codespeak.net> Author: fijal Date: Tue Nov 6 12:39:37 2007 New Revision: 48333 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Log: Make test more mix unicode and str Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Tue Nov 6 12:39:37 2007 @@ -64,8 +64,8 @@ def test_concat(self): const = self.const def fn(i, j): - s1 = [const(''), const('a'), const('ab')] - s2 = [const(''), const('x'), const('xy')] + s1 = [const(''), 'a', const('ab')] + s2 = [const(''), const('x'), 'xy'] return s1[i] + s2[j] for i in range(3): for j in range(3): From fijal at codespeak.net Tue Nov 6 12:42:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 12:42:27 +0100 (CET) Subject: [pypy-svn] r48334 - pypy/branch/pypy-rpython-unicode/rpython/lltypesystem Message-ID: <20071106114227.361C18221@code0.codespeak.net> Author: fijal Date: Tue Nov 6 12:42:26 2007 New Revision: 48334 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Log: Simple version works as well Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Tue Nov 6 12:42:26 2007 @@ -216,7 +216,7 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - if typeOf(s1).TO.chars.OF is Char and typeOf(s2).TO.chars.OF is Char: + if typeOf(s1).TO == STR and typeOf(s2).TO == STR: malloc = mallocstr else: malloc = mallocunicode From fijal at codespeak.net Tue Nov 6 13:19:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 13:19:18 +0100 (CET) Subject: [pypy-svn] r48335 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071106121918.AFBDA8145@code0.codespeak.net> Author: fijal Date: Tue Nov 6 13:19:17 2007 New Revision: 48335 Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Log: Forbid mixing unicode and string (this leaves some desgin flaw around as mixing of unichr and chr is ok) Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/binaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Tue Nov 6 13:19:17 2007 @@ -426,10 +426,12 @@ return SomeChar() -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint), - pairtype(SomeChar, SomeUnicodeCodePoint), +class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeCodePoint, SomeChar)): + def union((uchr1, uchr2)): + return SomeUnicodeCodePoint() +class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): def union((uchr1, uchr2)): return SomeUnicodeCodePoint() @@ -665,14 +667,8 @@ getbookkeeper().count("str_mul", str2, int1) return SomeString() -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString), - pairtype(SomeUnicodeCodePoint, SomeString), - pairtype(SomeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeUnicodeString), +class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeChar), - pairtype(SomeChar, SomeUnicodeString), pairtype(SomeUnicodeString, SomeUnicodeString)): def union((str1, str2)): return SomeUnicodeString(can_be_None=str1.can_be_none() or Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Tue Nov 6 13:19:17 2007 @@ -2883,7 +2883,7 @@ def test_unicode(self): def g(n): if n > 0: - return "xxx" + return unichr(1234) else: return u"x\xe4x" @@ -2913,19 +2913,12 @@ def g(x): return unichr(x) + unichr(2) - def h(x): - return x + u'a' - - def i(x): - return unicode(x) + 'xxx' - - for func in f, h, i: - a = self.RPythonAnnotator() - s = a.build_types(f, [str]) - assert isinstance(s, annmodel.SomeUnicodeString) a = self.RPythonAnnotator() - s = a.build_types(f, [int]) + s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeUnicodeString) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeUnicodeString) def g(n): From fijal at codespeak.net Tue Nov 6 13:24:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 13:24:09 +0100 (CET) Subject: [pypy-svn] r48336 - in pypy/branch/pypy-rpython-unicode/rpython: . test Message-ID: <20071106122409.6BE4C8272@code0.codespeak.net> Author: fijal Date: Tue Nov 6 13:24:09 2007 New Revision: 48336 Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Log: simplify code, as mixing unicode and strings are forbidden Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Tue Nov 6 13:24:09 2007 @@ -298,19 +298,12 @@ return hop.gendirectcall(r_str.ll.ll_stringslice_minusone, v_str) raise TyperError(r_slic) -def most_general_strrepr(repr1, repr2, rtyper): - string_repr = rtyper.type_system.rstr.string_repr - if repr1 is string_repr and repr2 is string_repr: - return string_repr - return rtyper_type_system.rstr.unicode_repr - class __extend__(pairtype(AbstractStringRepr, AbstractStringRepr)): def rtype_add((r_str1, r_str2), hop): str1_repr = r_str1.repr str2_repr = r_str2.repr if hop.s_result.is_constant(): - res_repr = most_general_strrepr(str1_repr, str2_repr, hop.rtyper) - return hop.inputconst(res_repr, hop.s_result.const) + return hop.inputconst(str1_repr, hop.s_result.const) v_str1, v_str2 = hop.inputargs(str1_repr, str2_repr) return hop.gendirectcall(r_str1.ll.ll_strconcat, v_str1, v_str2) rtype_inplace_add = rtype_add Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Tue Nov 6 13:24:09 2007 @@ -64,8 +64,8 @@ def test_concat(self): const = self.const def fn(i, j): - s1 = [const(''), 'a', const('ab')] - s2 = [const(''), const('x'), 'xy'] + s1 = [const(''), const('a'), const('ab')] + s2 = [const(''), const('x'), const('xy')] return s1[i] + s2[j] for i in range(3): for j in range(3): From cfbolz at codespeak.net Tue Nov 6 14:20:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Nov 2007 14:20:28 +0100 (CET) Subject: [pypy-svn] r48337 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071106132028.8DD6C8271@code0.codespeak.net> Author: cfbolz Date: Tue Nov 6 14:20:27 2007 New Revision: 48337 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: don't coallocate things with a __del__ Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Tue Nov 6 14:20:27 2007 @@ -8,6 +8,7 @@ class CreationPoint(object): constant = None + coallocator = None def __init__(self, creation_method, TYPE): self.creation_method = creation_method self.TYPE = TYPE @@ -39,6 +40,12 @@ crepsrepr = (", ".join([repr(crep) for crep in self.creation_points]), ) return "VarState({%s})" % crepsrepr + def get_crep(self, checksingle=False): + if checksingle: + assert len(self.creation_points) == 1 + for crep in self.creation_points.iterkeys(): + return crep + class GraphState(object): def __init__(self, graph): self.graph = graph @@ -319,7 +326,7 @@ if len(tovarstate.creation_points) != 1: continue fromcreps = set(fromvarstate.creation_points.keys()) - tocrep = tovarstate.creation_points.keys()[0] + tocrep = tovarstate.get_crep() if not tocrep.creation_method.startswith("malloc"): continue for fromcrep in fromcreps: @@ -327,7 +334,7 @@ break # also recently malloced else: #import pdb; pdb.set_trace() - num = do_coalloc(adi, graph, op.args[0], block, + num = do_coalloc(adi, graph, block, op, fromcreps, tocrep) if num: @@ -338,9 +345,9 @@ return total -def do_coalloc(adi, graph, fromvar, setblock, fromcreps, tocrep): +def do_coalloc(adi, graph, setblock, setop, fromcreps, tocrep): def find_coalloc_var(): - if block is setblock: + if block is setblock and seen_setvar: return fromvar for fromcrep in fromcreps: if fromcrep.creation_method == "constant": @@ -348,16 +355,24 @@ for fromcrep in fromcreps: for var in block.inputargs: varstate = adi.getstate(var) - assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] + if varstate is None: + continue + crep = varstate.get_crep(checksingle=True) if crep is fromcrep: return var return None result = 0 + seen_setvar = False for block, op in graph.iterblockops(): + if op.result is setop.args[0]: + seen_setvar = True if not op.opname.startswith("malloc"): continue - if adi.getstate(op.result).creation_points.keys()[0] is not tocrep: + if adi.getstate(op.result).get_crep(checksingle=True) is not tocrep: + continue + TYPE = op.result.concretetype.TO + # must not remove mallocs of structures that a destructor + if hasdestructor(TYPE): continue coallocvar = find_coalloc_var() if coallocvar is None: @@ -365,8 +380,18 @@ op.opname = "coalloc" + op.opname[len("malloc"):] op.args.insert(1, coallocvar) mallocvarstate = adi.getstate(op.result) - assert len(mallocvarstate.creation_points) == 1 - malloccrep = mallocvarstate.creation_points.keys()[0] + malloccrep = mallocvarstate.get_crep(checksingle=True) malloccrep.creation_method = "coalloc" result += 1 return result + +def hasdestructor(STRUCT): + if not isinstance(STRUCT, lltype.Struct): + return False + try: + destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr + if destr_ptr: + return True + except (ValueError, AttributeError), e: + pass + return False Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Tue Nov 6 14:20:27 2007 @@ -240,3 +240,27 @@ g(a) return a.length t = check_malloc_to_coalloc(f, [int], [8], 8, must_remove=1) + +def test_coalloc_in_setblock(): + class A(object): + pass + a3 = A() + def f(): + a1 = A() + a2 = A() + a2.a = a1 + a3.a = a2 + return 1 + # this should really be mustremove=2, but for now I am happy + t = check_malloc_to_coalloc(f, [], [], 1, must_remove=1) + +def test_nocoalloc_finalizer(): + class A(object): + def __del__(self): + pass + a = A() + def f(): + n = A() + a.next = n + return 1 + check_malloc_to_coalloc(f, [], [], 1, must_remove=0) From cfbolz at codespeak.net Tue Nov 6 14:28:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Nov 2007 14:28:13 +0100 (CET) Subject: [pypy-svn] r48338 - pypy/dist/pypy/rpython/tool Message-ID: <20071106132813.C4110826D@code0.codespeak.net> Author: cfbolz Date: Tue Nov 6 14:28:12 2007 New Revision: 48338 Modified: pypy/dist/pypy/rpython/tool/gcstat.py Log: some changes in my WC: use psyco, print progess Modified: pypy/dist/pypy/rpython/tool/gcstat.py ============================================================================== --- pypy/dist/pypy/rpython/tool/gcstat.py (original) +++ pypy/dist/pypy/rpython/tool/gcstat.py Tue Nov 6 14:28:12 2007 @@ -1,3 +1,8 @@ +try: + import psyco + psyco.full() +except ImportError: + pass class LifeTime(object): __slots__ = "typeid address size varsize birth death".split() @@ -12,7 +17,9 @@ def parse_file(f, callback): unknown_lifetime = {} current = 0 - for line in f: + for i, line in enumerate(f): + if i % 100000 == 0: + print i line = line.split() if line[0] == "free": _, typeid, address = line From pypy-svn at codespeak.net Tue Nov 6 14:39:48 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Tue, 6 Nov 2007 14:39:48 +0100 (CET) Subject: [pypy-svn] November 73% OFF Message-ID: <20071106153944.3183.qmail@ppp-124.120.166.237.revip2.asianet.co.th> An HTML attachment was scrubbed... URL: From rxe at codespeak.net Tue Nov 6 14:42:34 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 6 Nov 2007 14:42:34 +0100 (CET) Subject: [pypy-svn] r48339 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071106134234.4CFF28150@code0.codespeak.net> Author: rxe Date: Tue Nov 6 14:42:31 2007 New Revision: 48339 Removed: pypy/dist/pypy/translator/llvm/module/excsupport.py Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py Log: refactor to one place (almost) where we specify default tail call flag, calling convention and linkage options Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Tue Nov 6 14:42:31 2007 @@ -2,18 +2,20 @@ log = log.codewriter -DEFAULT_TAIL = '' #/tail -DEFAULT_CCONV = 'fastcc' #ccc/fastcc -DEFAULT_LINKAGE = 'internal ' #/internal (disabled for now because of the JIT) - class CodeWriter(object): - def __init__(self, file, db, tail=DEFAULT_TAIL, cconv=DEFAULT_CCONV, - linkage=DEFAULT_LINKAGE): + tail = '' #/tail + cconv = 'fastcc' #ccc/fastcc + linkage = 'internal ' #/internal (disabled for now because of the JIT) + + def __init__(self, file, db, tail=None, cconv=None, linkage=None): self.file = file self.word_repr = db.get_machine_word() - self.tail = tail - self.cconv = cconv - self.linkage = linkage + if tail is not None: + self.tail = tail + if cconv is not None: + self.cconv = cconv + if linkage is not None: + self.linkage = linkage def close(self): self.file.close() Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Tue Nov 6 14:42:31 2007 @@ -6,7 +6,6 @@ from pypy.objspace.flow.model import FunctionGraph from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype -from pypy.translator.llvm.codewriter import DEFAULT_CCONV from pypy.translator.llvm.buildllvm import llvm_gcc_version from pypy.tool.udir import udir @@ -28,7 +27,7 @@ def get_module_file(name): return os.path.join(get_llvm_cpath(), name) -def get_ll(ccode, function_names): +def get_ll(ccode, function_names, default_cconv): function_names += support_functions filename = str(udir.join("ccode.c")) f = open(filename, "w") @@ -82,10 +81,10 @@ if line.find("internal") == -1: if funcname not in ["%main", "%ctypes_RPython_StartupCode"]: internal = 'internal ' - line = '%s%s %s' % (internal, DEFAULT_CCONV, line,) + line = '%s%s %s' % (internal, default_cconv, line,) ll_lines.append(line) - # patch calls to function that we just declared fastcc + # patch calls to function that we just declared with differnet cconv ll_lines2, calltag, declaretag = [], 'call ', 'declare ' for line in ll_lines: i = line.find(calltag) @@ -93,14 +92,14 @@ cconv = 'ccc' for funcname in funcnames.iterkeys(): if line.find(funcname) >= 0: - cconv = DEFAULT_CCONV + cconv = default_cconv break line = "%scall %s %s" % (line[:i], cconv, line[i+len(calltag):]) if line[:len(declaretag)] == declaretag: cconv = 'ccc' for funcname in funcnames.keys(): if line.find(funcname) >= 0: - cconv = DEFAULT_CCONV + cconv = default_cconv break line = "declare %s %s" % (cconv, line[len(declaretag):]) ll_lines2.append(line) @@ -167,7 +166,7 @@ includestr += "-I %s " % ii return includestr -def generate_llfile(db, extern_decls, entrynode, c_includes, c_sources, standalone): +def generate_llfile(db, extern_decls, entrynode, c_includes, c_sources, standalone, default_cconv): ccode = [] function_names = [] @@ -223,5 +222,4 @@ # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) - - return get_ll("".join(ccode), function_names) + return get_ll("".join(ccode), function_names, default_cconv) Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Tue Nov 6 14:42:31 2007 @@ -80,6 +80,7 @@ # malloc_size is unsigned right now codewriter.malloc(targetvar, "sbyte", size) + # XXX uses own cconv codewriter.call(None, 'void', '%llvm.memset' + postfix(), ['sbyte*', 'ubyte', uword, uword], [targetvar, 0, size, boundary_size], @@ -125,6 +126,7 @@ codewriter.call(targetvar, 'sbyte*', fnname, [word], [size]) if atomic: + # XXX uses own cconv codewriter.call(None, 'void', '%llvm.memset' + postfix(), ['sbyte*', 'ubyte', uword, uword], [targetvar, 0, sizeu, boundary_size], Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Tue Nov 6 14:42:31 2007 @@ -45,8 +45,10 @@ # write bottom part of llvm file self.write_implementations(codewriter) - self._checkpoint('done') + # write entry point if there is one + codewriter.comment("End of file") codewriter.close() + self._checkpoint('done') return self.filename @@ -88,13 +90,14 @@ self._print_node_stats() - # create ll file from c code - self.generate_ll_externs() - self._checkpoint('setup_externs') - # open file & create codewriter codewriter, self.filename = self.create_codewriter() self._checkpoint('open file and create codewriter') + + # create ll file from c code + self.generate_ll_externs(codewriter) + self._checkpoint('setup_externs') + return codewriter def _set_wordsize(self, s): @@ -154,20 +157,12 @@ self._checkpoint('write support implentations') - # write wrapper code - if not self.standalone: - from pypy.translator.llvm.modwrapper import llvm_implcode - codewriter.write_lines(llvm_implcode(self.entrynode)) - # write all node implementations for node in self.db.getnodes(): if hasattr(node, 'writeimpl'): node.writeimpl(codewriter) self._checkpoint('write node implementations') - - # write entry point if there is one - codewriter.comment("End of file") def get_entry_point(self, func): assert func is not None @@ -191,7 +186,7 @@ self.entry_name = name[6:] return c.value._obj - def generate_ll_externs(self): + def generate_ll_externs(self, codewriter): c_includes = {} c_sources = {} @@ -208,13 +203,17 @@ self.entrynode, c_includes, c_sources, - self.standalone) + self.standalone, + codewriter.cconv) def create_codewriter(self): # prevent running the same function twice in a test filename = udir.join(self.entry_name).new(ext='.ll') f = open(str(filename), 'w') - return CodeWriter(f, self.db), filename + if self.standalone: + return CodeWriter(f, self.db), filename + else: + return CodeWriter(f, self.db, cconv='ccc', linkage=''), filename def write_extern_decls(self, codewriter): for c_name, obj in self.extern_decls: Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Nov 6 14:42:31 2007 @@ -3,33 +3,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.rstr import STR -def _noresult(returntype): - r = returntype.strip() - if r == 'void': - return 'void' - elif r == 'bool': - return 'bool false' - elif r in 'float double'.split(): - return r + ' 0.0' - elif r in 'ubyte sbyte ushort short uint int ulong long'.split(): - return r + ' 0' - return r + ' null' - -def llvm_implcode(entrynode): - from pypy.translator.llvm.codewriter import DEFAULT_CCONV as cconv - from pypy.translator.llvm.module.excsupport import entrycode, voidentrycode, raisedcode - returntype, entrypointname = entrynode.getdecl().split(' %', 1) - noresult = _noresult(returntype) - - code = raisedcode % locals() - if returntype == "void": - code += voidentrycode % locals() - else: - code += entrycode % locals() - - return code - - class CtypesModule: """ use ctypes to create a temporary module """ @@ -40,9 +13,13 @@ _c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) -raised = _c.__entrypoint__raised_LLVMException -raised.argtypes = [] -raised.restype = ctypes.c_int +rpyexc_occured = _c.pypy__rpyexc_occured +rpyexc_occured.argtypes = [] +rpyexc_occured.restype = ctypes.c_int + +rpyexc_fetch_type = _c.pypy_rpyexc_fetch_type +rpyexc_fetch_type.argtypes = [] +rpyexc_fetch_type.restype = ctypes.c_void_p GC_get_heap_size_wrapper = _c.GC_get_heap_size GC_get_heap_size_wrapper.argtypes = [] @@ -65,7 +42,7 @@ _setup = True args = [f(a) for a, f in zip(args, to_llargs)] result = __entrypoint__(*args) - if raised(): + if rpyexc_occured(): raise LLVMException("Exception raised") return ll_to_res(result) @@ -133,11 +110,11 @@ """ epilog = """ -__entrypoint__ = _c.__entrypoint__pypy_%(name)s +__entrypoint__ = _c.pypy_%(name)s # %(RT)r to_llargs = %(to_llargs)s -__entrypoint__.argtypes = %(args)s +__entrypoint__ = _c.pypy_%(name)s # %(ARGS)r ll_to_res = %(ll_to_res)s From fijal at codespeak.net Tue Nov 6 17:10:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Nov 2007 17:10:38 +0100 (CET) Subject: [pypy-svn] r48341 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071106161038.0E4988184@code0.codespeak.net> Author: fijal Date: Tue Nov 6 17:10:38 2007 New Revision: 48341 Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Log: Expose string methods to unicode strings Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Tue Nov 6 17:10:38 2007 @@ -2918,7 +2918,15 @@ assert isinstance(s, annmodel.SomeUnicodeString) a = self.RPythonAnnotator() s = a.build_types(f, [int]) - assert isinstance(s, annmodel.SomeUnicodeString) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_unicode_startswith(self): + def f(x): + return u'xxxx'.replace(x, u'z') + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) def g(n): Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Tue Nov 6 17:10:38 2007 @@ -410,7 +410,8 @@ return s_Bool -class __extend__(SomeString): +class __extend__(SomeString, + SomeUnicodeString): def method_startswith(str, frag): return s_Bool @@ -441,7 +442,7 @@ s_item = s_list.listdef.read_item() if isinstance(s_item, SomeImpossibleValue): return immutablevalue("") - return SomeString() + return str.__class__() def iter(str): return SomeIterator(str) @@ -458,16 +459,16 @@ def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) - return getbookkeeper().newlist(SomeString()) + return getbookkeeper().newlist(str.__class__()) def method_replace(str, s1, s2): - return SomeString() + return str.__class__() def method_lower(str): - return SomeString() + return str.__class__() def method_upper(str): - return SomeString() + return str.__class__() class __extend__(SomeChar): From cfbolz at codespeak.net Wed Nov 7 01:21:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 01:21:20 +0100 (CET) Subject: [pypy-svn] r48349 - in pypy/dist/pypy: config doc/config rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test rpython/test translator/backendopt Message-ID: <20071107002120.7BBBF8160@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 01:21:18 2007 New Revision: 48349 Added: pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt (contents, props changed) Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llheap.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/rpython/test/test_llinterp.py pypy/dist/pypy/translator/backendopt/all.py Log: add coallocation support to the GCs. needs changes in various places. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Wed Nov 7 01:21:18 2007 @@ -160,6 +160,9 @@ BoolOption("heap2stack", "Escape analysis and stack allocation", default=False, requires=[("translation.stackless", False)]), + BoolOption("coalloc", "Try to replace mallocs by coallocation", + default=False, + suggests=[("translation.gc", "generation")]), # control profile based inlining StrOption("profile_based_inline", "Use call count profiling to drive inlining" Added: pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt Wed Nov 7 01:21:18 2007 @@ -0,0 +1,4 @@ +turn allocations into coallocations when appropriate. Requires the use of a +generational GC. See the paper `Finding your Cronies`_. + +.. _`Finding your Cronies`:: http://www.cs.utexas.edu/~sammy/cronies-oopsla-2004.pdf Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Wed Nov 7 01:21:18 2007 @@ -676,8 +676,11 @@ return ptr def op_coalloc(self, obj, coallocator, flags): - # it's always safe to ignore the coallocator - return self.op_malloc(obj, flags) + flavor = flags['flavor'] + assert flavor == "gc" + zero = flags.get('zero', False) + ptr = self.heap.coalloc(obj, coallocator, zero=zero) + return ptr # only after gc transform def op_cpy_malloc(self, obj, cpytype): # xxx @@ -699,8 +702,12 @@ self.make_llexception() def op_coalloc_varsize(self, obj, coallocator, flags, size): - # it's always safe to ignore the coallocator - return self.op_malloc_varsize(obj, flags, size) + flavor = flags['flavor'] + zero = flags.get('zero', False) + assert flavor == "gc" + zero = flags.get('zero', False) + ptr = self.heap.coalloc(obj, coallocator, size, zero=zero) + return ptr def op_free(self, obj, flavor): assert isinstance(flavor, str) Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Wed Nov 7 01:21:18 2007 @@ -19,3 +19,7 @@ def weakref_create_getlazy(objgetter): return weakref_create(objgetter()) + +def coalloc(T, coallocator, n=None, zero=True): + # ignore the coallocator + return malloc(T, n, zero=zero) Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Nov 7 01:21:18 2007 @@ -36,9 +36,10 @@ def size_gc_header(self, typeid=0): return self.gcheaderbuilder.size_gc_header - def malloc(self, typeid, length=0, zero=False): + def malloc(self, typeid, length=0, zero=False, coallocator=None): """For testing. The interface used by the gctransformer is the four malloc_[fixed,var]size[_clear]() functions. + And (if they exist) to the coalloc_[fixed,var]size functios """ size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) @@ -59,15 +60,26 @@ malloc_varsize = self.malloc_varsize_clear else: malloc_varsize = self.malloc_varsize - ref = malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True, needs_finalizer) + if coallocator is not None and hasattr(self, "coalloc_varsize"): + assert not needs_finalizer + coallocator = llmemory.cast_ptr_to_adr(coallocator) + ref = self.coalloc_varsize(coallocator, typeid, length, size, + itemsize, offset_to_length) + else: + ref = malloc_varsize(typeid, length, size, itemsize, + offset_to_length, True, needs_finalizer) else: if zero: malloc_fixedsize = self.malloc_fixedsize_clear else: malloc_fixedsize = self.malloc_fixedsize - ref = malloc_fixedsize(typeid, size, True, needs_finalizer, - contains_weakptr) + if coallocator is not None and hasattr(self, "coalloc_fixedsize"): + assert not needs_finalizer + coallocator = llmemory.cast_ptr_to_adr(coallocator) + ref = self.coalloc_fixedsize(coallocator, typeid, size) + else: + ref = malloc_fixedsize(typeid, size, True, needs_finalizer, + contains_weakptr) # lots of cast and reverse-cast around... return llmemory.cast_ptr_to_adr(ref) Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Nov 7 01:21:18 2007 @@ -77,6 +77,17 @@ self.young_objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + def coalloc_fixedsize(self, coallocator, typeid, size): + # note: a coallocated object can never return a weakref, since the + # coallocation analysis is done at a time where weakrefs are + # represented as opaque objects which aren't allocated using malloc but + # with weakref_create + if self.is_in_nursery(coallocator): + return self.malloc_fixedsize(typeid, size, True, False, False) + else: + return SemiSpaceGC.malloc_fixedsize(self, typeid, size, True, + False, False) + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): # only use the nursery if there are not too many items @@ -101,6 +112,16 @@ self.nursery_free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + def coalloc_varsize(self, coallocator, typeid, length, size, itemsize, + offset_to_length): + if self.is_in_nursery(coallocator): + return self.malloc_varsize(typeid, length, size, itemsize, + offset_to_length, True, False) + else: + return SemiSpaceGC.malloc_varsize(self, typeid, length, size, + itemsize, offset_to_length, + True, False) + # override the init_gc_object methods to change the default value of 'flags', # used by objects that are directly created outside the nursery by the SemiSpaceGC. # These objects must have the GCFLAG_NO_YOUNG_PTRS flag set immediately. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Nov 7 01:21:18 2007 @@ -27,6 +27,8 @@ if op.opname in ('malloc', 'malloc_varsize'): flags = op.args[1].value return flags['flavor'] == 'gc' and not flags.get('nocollect', False) + if op.opname in ('coalloc', 'coalloc_varsize'): + return True def find_initializing_stores(collect_analyzer, graph): from pypy.objspace.flow.model import mkentrymap @@ -323,6 +325,20 @@ inline=True) else: self.write_barrier_ptr = None + if hasattr(GCClass, "coalloc_fixedsize"): + self.coalloc_clear_ptr = getfn( + GCClass.coalloc_fixedsize.im_func, + [s_gc, annmodel.SomeAddress(), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True)], + s_gcref, inline=True) + self.coalloc_varsize_clear_ptr = getfn( + GCClass.coalloc_varsize.im_func, + [s_gc, annmodel.SomeAddress()] + + [annmodel.SomeInteger(nonneg=True) for i in range(5)], + s_gcref, inline=True) + else: + self.coalloc_clear_ptr = self.coalloc_varsize_clear_ptr = None self.statistics_ptr = getfn(GCClass.statistics.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeInteger()) @@ -563,6 +579,43 @@ gct_fv_gc_malloc_varsize = gct_fv_gc_malloc + def gct_fv_gc_coalloc(self, hop, coallocator, flags, TYPE, *args): + if self.coalloc_clear_ptr is None: + return self.gct_fv_gc_malloc( + hop, flags, TYPE, *args) + op = hop.spaceop + flavor = flags['flavor'] + assert not flags.get("nocollect", False) + + PTRTYPE = op.result.concretetype + assert PTRTYPE.TO == TYPE + type_id = self.get_type_id(TYPE) + + c_type_id = rmodel.inputconst(lltype.Signed, type_id) + info = self.layoutbuilder.type_info_list[type_id] + c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) + assert not has_finalizer + + v_coallocator = gen_cast(hop.llops, llmemory.Address, coallocator) + + if not op.opname.endswith('_varsize'): + malloc_ptr = self.coalloc_clear_ptr + args = [self.c_const_gc, v_coallocator, c_type_id, c_size] + else: + v_length = op.args[-1] + c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) + c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) + malloc_ptr = self.coalloc_varsize_clear_ptr + args = [self.c_const_gc, v_coallocator, c_type_id, v_length, c_size, + c_varitemsize, c_ofstolength] + livevars = self.push_roots(hop) + v_result = hop.genop("direct_call", [malloc_ptr] + args, + resulttype=llmemory.GCREF) + self.pop_roots(hop, livevars) + return v_result + gct_fv_gc_coalloc_varsize = gct_fv_gc_coalloc + def gct_gc__collect(self, hop): op = hop.spaceop livevars = self.push_roots(hop) Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Wed Nov 7 01:21:18 2007 @@ -491,6 +491,21 @@ c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(TYPE)) v_raw = meth(hop, flags, TYPE, c_size) hop.cast_result(v_raw) + + def gct_coalloc(self, hop): + TYPE = hop.spaceop.result.concretetype.TO + assert not TYPE._is_varsize() + flags = hop.spaceop.args[2].value + flavor = flags['flavor'] + c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(TYPE)) + meth = getattr(self, 'gct_fv_%s_comalloc' % flavor, None) + if meth is None: + meth = getattr(self, 'gct_fv_%s_malloc' % flavor, None) + assert meth, "%s has no support for comalloc with flavor %r" % (self, flavor) + v_raw = meth(hop, flags, TYPE, c_size) + else: + v_raw = meth(hop, hop.spaceop.args[1], flags, TYPE, c_size) + hop.cast_result(v_raw) def gct_fv_raw_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.raw_malloc_fixedsize_ptr, c_size], @@ -514,16 +529,32 @@ return hop.genop('cpy_malloc', args, resulttype=op.result.concretetype) def gct_malloc_varsize(self, hop): - def intconst(c): return rmodel.inputconst(lltype.Signed, c) - op = hop.spaceop - TYPE = op.result.concretetype.TO - assert TYPE._is_varsize() flags = hop.spaceop.args[1].value flavor = flags['flavor'] meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None) assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) + return self.varsize_malloc_helper(hop, flags, meth, []) + + def gct_coalloc_varsize(self, hop): + flags = hop.spaceop.args[2].value + flavor = flags['flavor'] + meth = getattr(self, 'gct_fv_%s_coalloc_varsize' % flavor, None) + if meth is None: + meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None) + assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) + return self.varsize_malloc_helper(hop, flags, meth, []) + else: + return self.varsize_malloc_helper(hop, flags, meth, + [hop.spaceop.args[1]]) + + + def varsize_malloc_helper(self, hop, flags, meth, extraargs): + def intconst(c): return rmodel.inputconst(lltype.Signed, c) + op = hop.spaceop + TYPE = op.result.concretetype.TO + assert TYPE._is_varsize() if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] else: @@ -545,11 +576,13 @@ offset_to_length = llmemory.ArrayLengthOffset(ARRAY) c_offset_to_length = intconst(offset_to_length) - v_raw = meth(hop, flags, TYPE, op.args[-1], c_const_size, c_item_size, - c_offset_to_length) + args = [hop] + extraargs + [flags, TYPE, + op.args[-1], c_const_size, c_item_size, c_offset_to_length] + v_raw = meth(*args) hop.cast_result(v_raw) + def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): if c_offset_to_length is None: Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Nov 7 01:21:18 2007 @@ -62,6 +62,17 @@ assert flavor != 'gc' return lltype.free(TYPE, flavor=flavor) + def coalloc(self, TYPE, coallocator, size=None, zero=False): + if hasattr(self.gc, "coalloc_fixedsize"): + typeid = self.get_type_id(TYPE) + addr = self.gc.malloc(typeid, size, zero=zero, + coallocator=coallocator) + result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + if self.gc.needs_zero_gc_pointers: + gctypelayout.zero_gc_pointers(result) + return result + return self.malloc(TYPE, size, 'gc', zero) + def setfield(self, obj, fieldname, fieldvalue): STRUCT = lltype.typeOf(obj).TO addr = llmemory.cast_ptr_to_adr(obj) @@ -104,6 +115,7 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) + # ____________________________________________________________ class RootLinkedList(object): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Nov 7 01:21:18 2007 @@ -368,3 +368,17 @@ class TestGenerationalGC(GCTest): from pypy.rpython.memory.gc.generation import GenerationGC as GCClass + + def test_coalloc(self): + def malloc_a_lot(): + i = 0 + while i < 10: + i += 1 + a = [1] * 10 + j = 0 + while j < 30: + j += 1 + a.append(j) + return 0 + res = self.interpret(malloc_a_lot, [], backendopt=True, coalloc=True) + assert res == 0 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Wed Nov 7 01:21:18 2007 @@ -13,24 +13,29 @@ INT_SIZE = struct.calcsize("i") # only for estimates -def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False): +def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False, + backendopt=False, **extraconfigopts): from pypy.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname t.config.translation.stacklessgc = stacklessgc + t.config.set(**extraconfigopts) t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() + if backendopt: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t) if conftest.option.view: - t.view() + t.viewcg() return t class GCTest(object): gcpolicy = None stacklessgc = False - def runner(self, f, nbargs=0, statistics=False): + def runner(self, f, nbargs=0, statistics=False, **extraconfigopts): if nbargs == 2: def entrypoint(args): x = args[0] @@ -49,7 +54,8 @@ ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs) s_args = annmodel.SomePtr(lltype.Ptr(ARGS)) t = rtype(entrypoint, [s_args], gcname=self.gcname, - stacklessgc=self.stacklessgc) + stacklessgc=self.stacklessgc, + **extraconfigopts) cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, gcpolicy=self.gcpolicy) db = cbuild.generate_graphs_for_llinterp() @@ -688,6 +694,21 @@ res = run([3, 0]) assert res == 1 + def test_coalloc(self): + def malloc_a_lot(): + i = 0 + while i < 10: + i += 1 + a = [1] * 10 + j = 0 + while j < 30: + j += 1 + a.append(j) + return 0 + run, statistics = self.runner(malloc_a_lot, statistics=True, + backendopt=True, coalloc=True) + run([]) + class TestStacklessMarkSweepGC(TestMarkSweepGC): @@ -838,3 +859,4 @@ i += 1 run = self.runner(f, nbargs=0) run([]) + Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Wed Nov 7 01:21:18 2007 @@ -37,8 +37,10 @@ return res def gengraph(func, argtypes=[], viewbefore='auto', policy=None, - type_system="lltype", backendopt=False, config=None): + type_system="lltype", backendopt=False, config=None, + **extraconfigopts): t = TranslationContext(config=config) + t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) timelog("annotating", a.build_types, func, argtypes) if viewbefore == 'auto': @@ -69,9 +71,12 @@ def get_interpreter(func, values, view='auto', viewbefore='auto', policy=None, someobjects=False, type_system="lltype", backendopt=False, - config=None, malloc_check=True): - key = (func,) + tuple([typeOf(x) for x in values])+ (someobjects, - backendopt) + config=None, malloc_check=True, **extraconfigopts): + extra_key = [(key, value) for key, value in extraconfigopts.iteritems()] + extra_key.sort() + extra_key = tuple(extra_key) + key = ((func,) + tuple([typeOf(x) for x in values]) + + (someobjects, backendopt, extra_key)) try: (t, interp, graph) = _tcache[key] except KeyError: @@ -91,7 +96,8 @@ t, typer, graph = gengraph(func, [annotation(x) for x in values], viewbefore, policy, type_system=type_system, - backendopt=backendopt, config=config) + backendopt=backendopt, config=config, + **extraconfigopts) interp = LLInterpreter(typer, malloc_check=malloc_check) _tcache[key] = (t, interp, graph) # keep the cache small Modified: pypy/dist/pypy/translator/backendopt/all.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/all.py (original) +++ pypy/dist/pypy/translator/backendopt/all.py Wed Nov 7 01:21:18 2007 @@ -125,6 +125,10 @@ print "after if-to-switch:" print_statistics(translator.graphs[0], translator) + if config.coalloc and not secondary: + from pypy.translator.backendopt import coalloc + coalloc.malloc_to_coalloc(translator) + for graph in graphs: checkgraph(graph) From tismer at codespeak.net Wed Nov 7 05:18:58 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 7 Nov 2007 05:18:58 +0100 (CET) Subject: [pypy-svn] r48350 - pypy/dist/pypy/objspace/flow Message-ID: <20071107041858.EE8DC81AB@code0.codespeak.net> Author: tismer Date: Wed Nov 7 05:18:57 2007 New Revision: 48350 Modified: pypy/dist/pypy/objspace/flow/specialcase.py Log: some over-due relaxing to __import__ annotation Modified: pypy/dist/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/dist/pypy/objspace/flow/specialcase.py (original) +++ pypy/dist/pypy/objspace/flow/specialcase.py Wed Nov 7 05:18:57 2007 @@ -6,7 +6,18 @@ from pypy.tool.cache import Cache def sc_import(space, fn, args): - w_name, w_glob, w_loc, w_frm = args.fixedunpack(4) + args_w, kwds_w = args.unpack() + assert kwds_w == {}, "should not call %r with keyword arguments" % (fn,) + assert len(args_w) > 0 and len(args_w) <= 4, 'import needs 1 to 4 arguments' + w_name = args_w[0] + w_None = space.wrap(None) + w_glob, w_loc, w_frm = w_None, w_None, w_None + if len(args_w) > 1: + w_glob = args_w[1] + if len(args_w) > 2: + w_loc = args_w[2] + if len(args_w) > 3: + w_frm = args_w[3] if not isinstance(w_loc, Constant): # import * in a function gives us the locals as Variable # we always forbid it as a SyntaxError From cfbolz at codespeak.net Wed Nov 7 11:21:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 11:21:50 +0100 (CET) Subject: [pypy-svn] r48351 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071107102150.5DFF081BD@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 11:21:49 2007 New Revision: 48351 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: test + fix Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Wed Nov 7 11:21:49 2007 @@ -348,7 +348,7 @@ def do_coalloc(adi, graph, setblock, setop, fromcreps, tocrep): def find_coalloc_var(): if block is setblock and seen_setvar: - return fromvar + return setop.args[0] for fromcrep in fromcreps: if fromcrep.creation_method == "constant": return fromcrep.constant @@ -362,27 +362,28 @@ return var return None result = 0 - seen_setvar = False - for block, op in graph.iterblockops(): - if op.result is setop.args[0]: - seen_setvar = True - if not op.opname.startswith("malloc"): - continue - if adi.getstate(op.result).get_crep(checksingle=True) is not tocrep: - continue - TYPE = op.result.concretetype.TO - # must not remove mallocs of structures that a destructor - if hasdestructor(TYPE): - continue - coallocvar = find_coalloc_var() - if coallocvar is None: - continue - op.opname = "coalloc" + op.opname[len("malloc"):] - op.args.insert(1, coallocvar) - mallocvarstate = adi.getstate(op.result) - malloccrep = mallocvarstate.get_crep(checksingle=True) - malloccrep.creation_method = "coalloc" - result += 1 + for block in graph.iterblocks(): + seen_setvar = False + for op in block.operations: + if block is setblock and op.result is setop.args[0]: + seen_setvar = True + if not op.opname.startswith("malloc"): + continue + if adi.getstate(op.result).get_crep(checksingle=True) is not tocrep: + continue + TYPE = op.result.concretetype.TO + # must not remove mallocs of structures that a destructor + if hasdestructor(TYPE): + continue + coallocvar = find_coalloc_var() + if coallocvar is None: + continue + op.opname = "coalloc" + op.opname[len("malloc"):] + op.args.insert(1, coallocvar) + mallocvarstate = adi.getstate(op.result) + malloccrep = mallocvarstate.get_crep(checksingle=True) + malloccrep.creation_method = "coalloc" + result += 1 return result def hasdestructor(STRUCT): Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Wed Nov 7 11:21:49 2007 @@ -254,6 +254,19 @@ # this should really be mustremove=2, but for now I am happy t = check_malloc_to_coalloc(f, [], [], 1, must_remove=1) +def test_coalloc_in_setblock_old(): + class A(object): + pass + def g(): + return A() + def f(): + a = g() + a1 = A() + a.a = a1 + return 1 + t = check_malloc_to_coalloc(f, [], [], 1, must_remove=1) + + def test_nocoalloc_finalizer(): class A(object): def __del__(self): From cfbolz at codespeak.net Wed Nov 7 11:30:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 11:30:00 +0100 (CET) Subject: [pypy-svn] r48352 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071107103000.28FAA823E@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 11:29:59 2007 New Revision: 48352 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: test + fix: try to use the setvar when it is available in more cases. Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Wed Nov 7 11:29:59 2007 @@ -363,7 +363,7 @@ return None result = 0 for block in graph.iterblocks(): - seen_setvar = False + seen_setvar = setop.args[0] in block.inputargs for op in block.operations: if block is setblock and op.result is setop.args[0]: seen_setvar = True Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Wed Nov 7 11:29:59 2007 @@ -266,6 +266,28 @@ return 1 t = check_malloc_to_coalloc(f, [], [], 1, must_remove=1) +def test_coalloc_in_setblock_args(): + class A(object): + pass + def g(): + return A() + globala = A() + def f(x): + if x: + a = g() + else: + a = globala + a1 = A() + a.a = a1 + return 1 + t = check_malloc_to_coalloc(f, [int], [1], 1, must_remove=1) + fgraph = t.graphs[0] + # make sure that the coallocator is not globala but the variable a + block = fgraph.startblock.exits[0].target + op = block.operations[0] + assert op.opname == "coalloc" + assert op.args[1] is block.inputargs[0] + def test_nocoalloc_finalizer(): class A(object): From cfbolz at codespeak.net Wed Nov 7 12:32:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 12:32:50 +0100 (CET) Subject: [pypy-svn] r48353 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071107113250.40AC18197@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 12:32:48 2007 New Revision: 48353 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Log: more exact heuristics Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Wed Nov 7 12:32:48 2007 @@ -347,19 +347,48 @@ def do_coalloc(adi, graph, setblock, setop, fromcreps, tocrep): def find_coalloc_var(): + # if the setting happens in the same block as the malloc, use the + # variable directly if block is setblock and seen_setvar: return setop.args[0] - for fromcrep in fromcreps: + + # if the setting _always_ happens on a constant, use the constant + if len(fromcreps) == 1: + fromcrep, = fromcreps if fromcrep.creation_method == "constant": return fromcrep.constant + + # lots of heuristics ahead + subsets = [] + intersections = [] + for var in block.inputargs: + varstate = adi.getstate(var) + if varstate is None: + continue + creps = set(varstate.creation_points) + if creps == fromcreps: + return var + if creps.issubset(fromcreps): + subsets.append((len(creps), var)) + intersection = creps.intersection(fromcreps) + if intersection: + intersections.append((len(intersection), var)) + + # first, try to use the biggest subset of creps + if subsets: + subsets.sort() + return subsets[-1][1] + # then, get desparate, and use any inputarg that has the biggest + # intersection + if intersections: + intersections.sort() + return intersections[-1][1] + + # if there is still nothing, check the fromcreps again and look for + # a constant for fromcrep in fromcreps: - for var in block.inputargs: - varstate = adi.getstate(var) - if varstate is None: - continue - crep = varstate.get_crep(checksingle=True) - if crep is fromcrep: - return var + if fromcrep.creation_method == "constant": + return fromcrep.constant return None result = 0 for block in graph.iterblocks(): Modified: pypy/dist/pypy/translator/backendopt/test/test_coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_coalloc.py Wed Nov 7 12:32:48 2007 @@ -299,3 +299,29 @@ a.next = n return 1 check_malloc_to_coalloc(f, [], [], 1, must_remove=0) + +def test_coalloc_set_further_down(): + class A(object): + pass + def g(): + return A() + globala = A() + globala.x = 2 + def f(x): + if x: + a = g() + a.x = 42 + else: + a = g() + a.x = 43 + a1 = A() + a.x = 1 + # this if is only there to force the set into a new block + if not x: + b = a + else: + b = globala + a.a = a1 + return b.x + t = check_malloc_to_coalloc(f, [int], [1], 2, must_remove=1) + From cfbolz at codespeak.net Wed Nov 7 13:17:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 13:17:50 +0100 (CET) Subject: [pypy-svn] r48354 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071107121750.2A66D81B8@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 13:17:48 2007 New Revision: 48354 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: a better debug print for initializing stores Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Nov 7 13:17:48 2007 @@ -87,7 +87,8 @@ mallocvars = {target.inputargs[index]: True} mallocnum += 1 find_in_block(target, mallocvars) - print graph.name, mallocnum, len(result) + if result: + print "found %s initializing stores in %s" % (len(result), graph.name) return result ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) From cfbolz at codespeak.net Wed Nov 7 13:44:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 13:44:12 +0100 (CET) Subject: [pypy-svn] r48355 - pypy/dist/pypy/translator/backendopt Message-ID: <20071107124412.8CE7E8139@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 13:44:11 2007 New Revision: 48355 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py Log: remove this debug print Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Wed Nov 7 13:44:11 2007 @@ -162,8 +162,8 @@ self.setstate(op.result, res) return - if isonheap(op.result) or filter(None, args): - print "assuming that '%s' is irrelevant" % op + #if isonheap(op.result) or filter(None, args): + # print "assuming that '%s' is irrelevant" % op def complete(self): while self.scheduled: From fijal at codespeak.net Wed Nov 7 14:02:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 14:02:12 +0100 (CET) Subject: [pypy-svn] r48356 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071107130212.768C380E7@code0.codespeak.net> Author: fijal Date: Wed Nov 7 14:02:11 2007 New Revision: 48356 Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py pypy/branch/pypy-rpython-unicode/annotation/signature.py pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Log: * Relax somewhat return values from method_xxx of SomeString, as SomeChar should return string, not self * Try to make the same working on SomeUnicode, does not work Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/model.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/model.py Wed Nov 7 14:02:11 2007 @@ -217,6 +217,9 @@ def nonnoneify(self): return SomeUnicodeString(can_be_None=False) +SomeString.basestringclass = SomeString +SomeUnicodeString.basestringclass = SomeUnicodeString + class SomeChar(SomeString): "Stands for an object known to be a string of length 1." Modified: pypy/branch/pypy-rpython-unicode/annotation/signature.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/signature.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/signature.py Wed Nov 7 14:02:11 2007 @@ -3,7 +3,7 @@ from pypy.annotation.model import SomeBool, SomeInteger, SomeString,\ SomeFloat, SomeList, SomeDict, s_None, SomeExternalObject,\ SomeObject, SomeInstance, SomeTuple, lltype_to_annotation,\ - unionof + unionof, SomeUnicodeString from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -80,6 +80,8 @@ return SomeFloat() elif issubclass(t, str): # py.lib uses annotated str subclasses return SomeString() + elif t is unicode: + return SomeUnicodeString() elif t is list: return SomeList(MOST_GENERAL_LISTDEF) elif t is dict: Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Wed Nov 7 14:02:11 2007 @@ -2928,6 +2928,30 @@ s = a.build_types(f, [unicode]) assert isinstance(s, annmodel.SomeUnicodeString) + def test_unicode_buildtypes(self): + def f(x): + return x + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_replace_annotations(self): + py.test.skip("FIXME") + def f(x): + return 'a'.replace(x, 'b') + + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeString) + + def f(x): + return u'a'.replace(x, u'b') + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) + def g(n): return [0,1,2,n] Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Wed Nov 7 14:02:11 2007 @@ -442,7 +442,7 @@ s_item = s_list.listdef.read_item() if isinstance(s_item, SomeImpossibleValue): return immutablevalue("") - return str.__class__() + return str.basestringclass() def iter(str): return SomeIterator(str) @@ -459,17 +459,16 @@ def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) - return getbookkeeper().newlist(str.__class__()) + return getbookkeeper().newlist(str.basestringclass()) def method_replace(str, s1, s2): - return str.__class__() + return str.basestringclass() def method_lower(str): - return str.__class__() + return str.basestringclass() def method_upper(str): - return str.__class__() - + return str.basestringclass() class __extend__(SomeChar): From fijal at codespeak.net Wed Nov 7 14:22:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 14:22:07 +0100 (CET) Subject: [pypy-svn] r48357 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071107132207.9AEFB81A9@code0.codespeak.net> Author: fijal Date: Wed Nov 7 14:22:06 2007 New Revision: 48357 Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Log: Make test pass (thanks carl, that was stupid) Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/model.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/model.py Wed Nov 7 14:22:06 2007 @@ -223,11 +223,8 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." -class SomeUnicodeCodePoint(SomeObject): +class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." - knowntype = unicode - immutable = True - def can_be_none(self): return False Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Wed Nov 7 14:22:06 2007 @@ -2937,7 +2937,6 @@ assert isinstance(s, annmodel.SomeUnicodeString) def test_replace_annotations(self): - py.test.skip("FIXME") def f(x): return 'a'.replace(x, 'b') From pypy-svn at codespeak.net Wed Nov 7 14:36:48 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Wed, 7 Nov 2007 14:36:48 +0100 (CET) Subject: [pypy-svn] November 75% OFF Message-ID: <20071107033642.5970.qmail@host92-74-dynamic.1-79-r.retail.telecomitalia.it> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Wed Nov 7 14:43:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 14:43:18 +0100 (CET) Subject: [pypy-svn] r48358 - in pypy/dist/pypy: rpython/memory/gc translator/c/test Message-ID: <20071107134318.CB00B8194@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 14:43:17 2007 New Revision: 48358 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: make allocation of void lists work on top of the generational GC. "how could that ever have worked"? Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Nov 7 14:43:17 2007 @@ -92,7 +92,8 @@ can_collect, has_finalizer=False): # only use the nursery if there are not too many items if (has_finalizer or not can_collect or - length > self.nursery_size // 4 // raw_malloc_usage(itemsize) or + (raw_malloc_usage(itemsize) and + length > self.nursery_size // 4 // raw_malloc_usage(itemsize)) or raw_malloc_usage(size) > self.nursery_size // 4): return SemiSpaceGC.malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Wed Nov 7 14:43:17 2007 @@ -799,6 +799,17 @@ "sizeof(double) not a power of two") assert (res & (expected_alignment-1)) == 0 + def test_void_list(self): + class E: + def __init__(self): + self.l = [] + def f(): + e = E() + return len(e.l) + c_fn = self.getcompiled(f) + assert c_fn() == 0 + + class TestUsingStacklessFramework(TestUsingFramework): def getcompiled(self, f): From fijal at codespeak.net Wed Nov 7 14:46:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 14:46:16 +0100 (CET) Subject: [pypy-svn] r48359 - in pypy/branch/pypy-rpython-unicode/annotation: . test Message-ID: <20071107134616.F061D81A2@code0.codespeak.net> Author: fijal Date: Wed Nov 7 14:46:16 2007 New Revision: 48359 Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Log: Fix thing that iteration over unicode returned chars Modified: pypy/branch/pypy-rpython-unicode/annotation/model.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/model.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/model.py Wed Nov 7 14:46:16 2007 @@ -217,9 +217,6 @@ def nonnoneify(self): return SomeUnicodeString(can_be_None=False) -SomeString.basestringclass = SomeString -SomeUnicodeString.basestringclass = SomeUnicodeString - class SomeChar(SomeString): "Stands for an object known to be a string of length 1." @@ -228,6 +225,11 @@ def can_be_none(self): return False +SomeString.basestringclass = SomeString +SomeString.basecharclass = SomeChar +SomeUnicodeString.basestringclass = SomeUnicodeString +SomeUnicodeString.basecharclass = SomeUnicodeCodePoint + class SomeList(SomeObject): "Stands for a homogenous list of any length." knowntype = list Modified: pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Wed Nov 7 14:46:16 2007 @@ -2951,6 +2951,16 @@ s = a.build_types(f, [unicode]) assert isinstance(s, annmodel.SomeUnicodeString) + def test_unicode_char(self): + def f(x, i): + for c in x: + if c == i: + return c + return 'x' + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode, str]) + assert isinstance(s, annmodel.SomeUnicodeCodePoint) def g(n): return [0,1,2,n] Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Wed Nov 7 14:46:16 2007 @@ -449,7 +449,7 @@ iter.can_only_throw = [] def getanyitem(str): - return SomeChar() + return str.basecharclass() def ord(str): return SomeInteger(nonneg=True) From fijal at codespeak.net Wed Nov 7 14:48:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 14:48:26 +0100 (CET) Subject: [pypy-svn] r48360 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071107134826.A2E9D81A9@code0.codespeak.net> Author: fijal Date: Wed Nov 7 14:48:26 2007 New Revision: 48360 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Log: Iteration over unicode strings works. Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 14:48:26 2007 @@ -56,7 +56,7 @@ return p def make_iterator_repr(self): - return string_iterator_repr + return self.iterator_repr def can_ll_be_null(self, s_value): # XXX unicode @@ -803,18 +803,32 @@ StringRepr.repr = string_repr UnicodeRepr.repr = unicode_repr -class StringIteratorRepr(AbstractStringIteratorRepr): +class BaseStringIteratorRepr(AbstractStringIteratorRepr): + def __init__(self): + self.ll_striter = ll_striter + self.ll_strnext = ll_strnext + +class StringIteratorRepr(BaseStringIteratorRepr): + lowleveltype = Ptr(GcStruct('stringiter', ('string', string_repr.lowleveltype), ('index', Signed))) - def __init__(self): - self.ll_striter = ll_striter - self.ll_strnext = ll_strnext +class UnicodeIteratorRepr(BaseStringIteratorRepr): + + lowleveltype = Ptr(GcStruct('unicodeiter', + ('string', unicode_repr.lowleveltype), + ('index', Signed))) def ll_striter(string): - iter = malloc(string_iterator_repr.lowleveltype.TO) + if typeOf(string) == string_repr.lowleveltype: + TP = string_repr.iterator_repr.lowleveltype.TO + elif typeOf(string) == unicode_repr.lowleveltype: + TP = unicode_repr.iterator_repr.lowleveltype.TO + else: + raise TypeError("Unknown string type %s" % (typeOf(string),)) + iter = malloc(TP) iter.string = string iter.index = 0 return iter @@ -827,8 +841,8 @@ iter.index = index + 1 return chars[index] -string_iterator_repr = StringIteratorRepr() - +string_repr.iterator_repr = StringIteratorRepr() +unicode_repr.iterator_repr = UnicodeIteratorRepr() # these should be in rclass, but circular imports prevent (also it's # not that insane that a string constant is built in this file). Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 14:48:26 2007 @@ -523,7 +523,7 @@ class AbstractStringIteratorRepr(IteratorRepr): def newiter(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) return hop.gendirectcall(self.ll_striter, v_str) Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 14:48:26 2007 @@ -73,8 +73,9 @@ assert self.ll_to_string(res) == fn(i, j) def test_iter(self): + const = self.const def fn(i): - s = ['', 'a', 'hello'][i] + s = [const(''), const('a'), const('hello')][i] i = 0 for c in s: if c != s[i]: From rxe at codespeak.net Wed Nov 7 15:08:17 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 7 Nov 2007 15:08:17 +0100 (CET) Subject: [pypy-svn] r48361 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071107140817.8BEC181C1@code0.codespeak.net> Author: rxe Date: Wed Nov 7 15:08:15 2007 New Revision: 48361 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/support.py Log: forgot these calling conventions. move the patching function to codewriter to save abusing _set_wordsize() even more Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 7 15:08:15 2007 @@ -1,4 +1,5 @@ from pypy.translator.llvm.log import log +from pypy.translator.llvm.buildllvm import postfix log = log.codewriter @@ -10,6 +11,7 @@ def __init__(self, file, db, tail=None, cconv=None, linkage=None): self.file = file self.word_repr = db.get_machine_word() + self.uword_repr = db.get_machine_uword() if tail is not None: self.tail = tail if cconv is not None: @@ -43,8 +45,13 @@ def _indent(self, line): self._append(" " + line) - def write_lines(self, lines): + def write_lines(self, lines, patch=False): for l in lines.split("\n"): + if patch: + l = l.replace('UWORD', self.uword_repr) + l = l.replace('WORD', self.word_repr) + l = l.replace('POSTFIX', postfix()) + l = l.replace('CC', self.cconv) self._append(l) def comment(self, line, indent=True): Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 7 15:08:15 2007 @@ -16,7 +16,6 @@ from pypy.translator.llvm.externs2ll import setup_externs, generate_llfile from pypy.translator.llvm.gc import GcPolicy from pypy.translator.llvm.log import log -from pypy.translator.llvm.buildllvm import llvm_is_on_path, postfix class GenLLVM(object): # see create_codewriter() below @@ -100,12 +99,6 @@ return codewriter - def _set_wordsize(self, s): - s = s.replace('UWORD', self.db.get_machine_uword()) - s = s.replace( 'WORD', self.db.get_machine_word()) - s = s.replace('POSTFIX', postfix()) - return s - def write_headers(self, codewriter): # write external function headers codewriter.header_comment('External Function Headers') @@ -134,7 +127,7 @@ codewriter.header_comment("Function Prototypes") # write external protos - codewriter.write_lines(self._set_wordsize(extdeclarations)) + codewriter.write_lines(extdeclarations, patch=True) # write node protos for node in self.db.getnodes(): @@ -149,9 +142,9 @@ # write external function implementations codewriter.header_comment('External Function Implementation') codewriter.write_lines(self.llexterns_functions) - codewriter.write_lines(self._set_wordsize(extfunctions)) + codewriter.write_lines(extfunctions, patch=True) if self.standalone: - codewriter.write_lines(self._set_wordsize(extfunctions_standalone)) + codewriter.write_lines(extfunctions_standalone, patch=True) self.write_extern_impls(codewriter) self.write_setup_impl(codewriter) Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Wed Nov 7 15:08:15 2007 @@ -5,23 +5,23 @@ """ extfunctions = """ -internal fastcc sbyte* %RPyString_AsString(%RPyString* %structstring) { +internal CC sbyte* %RPyString_AsString(%RPyString* %structstring) { %source1ptr = getelementptr %RPyString* %structstring, int 0, uint 1, uint 1 %source1 = cast [0 x sbyte]* %source1ptr to sbyte* ret sbyte* %source1 } -internal fastcc WORD %RPyString_Size(%RPyString* %structstring) { +internal CC WORD %RPyString_Size(%RPyString* %structstring) { %sizeptr = getelementptr %RPyString* %structstring, int 0, uint 1, uint 0 %size = load WORD* %sizeptr ret WORD %size } -internal fastcc %RPyString* %RPyString_FromString(sbyte* %s) { +internal CC %RPyString* %RPyString_FromString(sbyte* %s) { %lenu = call ccc uint %strlen(sbyte* %s) %lenuword = cast uint %lenu to UWORD %lenword = cast uint %lenu to WORD - %rpy = call fastcc %RPyString* %pypy_RPyString_New__Signed(WORD %lenword) + %rpy = call CC %RPyString* %pypy_RPyString_New__Signed(WORD %lenword) %rpystrptr = getelementptr %RPyString* %rpy, int 0, uint 1, uint 1 %rpystr = cast [0 x sbyte]* %rpystrptr to sbyte* @@ -30,7 +30,7 @@ ret %RPyString* %rpy } -internal fastcc WORD %pypyop_int_abs(WORD %x) { +internal CC WORD %pypyop_int_abs(WORD %x) { block0: %cond1 = setge WORD %x, 0 br bool %cond1, label %return_block, label %block1 @@ -42,7 +42,7 @@ ret WORD %result } -internal fastcc long %pypyop_llong_abs(long %x) { +internal CC long %pypyop_llong_abs(long %x) { block0: %cond1 = setge long %x, 0 br bool %cond1, label %return_block, label %block1 @@ -54,7 +54,7 @@ ret long %result } -internal fastcc double %pypyop_float_abs(double %x) { +internal CC double %pypyop_float_abs(double %x) { block0: %cond1 = setge double %x, 0.0 br bool %cond1, label %return_block, label %block1 @@ -73,8 +73,8 @@ from sys import maxint if maxint != 2**31-1: extfunctions_standalone += """ -internal fastcc int %pypy_entry_point(%RPyListOfString* %argv) { - %result = call fastcc long %pypy_entry_point(%RPyListOfString* %argv) +internal CC int %pypy_entry_point(%RPyListOfString* %argv) { + %result = call CC long %pypy_entry_point(%RPyListOfString* %argv) %tmp = cast long %result to int ret int %tmp } @@ -83,12 +83,11 @@ def write_raise_exc(c_name, exc_repr, codewriter): - l = """ -internal fastcc void %%raise%s(sbyte* %%msg) { +internal CC void %%raise%s(sbyte* %%msg) { ;%%exception_value = cast %s to %%RPYTHON_EXCEPTION* ret void } """ % (c_name, exc_repr) - codewriter.write_lines(l) + codewriter.write_lines(l, patch=True) From fijal at codespeak.net Wed Nov 7 16:04:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 16:04:58 +0100 (CET) Subject: [pypy-svn] r48362 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071107150458.B33978187@code0.codespeak.net> Author: fijal Date: Wed Nov 7 16:04:57 2007 New Revision: 48362 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Log: unichr -> unicode translation Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 16:04:57 2007 @@ -8,12 +8,12 @@ from pypy.rpython.rmodel import inputconst, IntegerRepr from pypy.rpython.rstr import AbstractStringRepr,AbstractCharRepr,\ AbstractUniCharRepr, AbstractStringIteratorRepr,\ - AbstractLLHelpers + AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.lltypesystem import ll_str from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, UniChar, Ptr, malloc, \ Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive, typeOf - +from pypy.rpython.rmodel import Repr # ____________________________________________________________ # @@ -33,11 +33,7 @@ CONST_STR_CACHE = WeakValueDictionary() CONST_UNICODE_CACHE = WeakValueDictionary() -class BaseStringRepr(AbstractStringRepr): - def __init__(self, *args): - AbstractStringRepr.__init__(self, *args) - self.ll = LLHelpers - +class BaseLLStringRepr(Repr): def convert_const(self, value): if value is None: return nullptr(self.lowleveltype.TO) @@ -73,24 +69,26 @@ v_items = hop.gendirectcall(LIST.ll_items, v_lst) return v_length, v_items -class StringRepr(BaseStringRepr): +class StringRepr(BaseLLStringRepr, AbstractStringRepr): lowleveltype = Ptr(STR) basetype = str base = Char CACHE = CONST_STR_CACHE def __init__(self, *args): - BaseStringRepr.__init__(self, *args) + AbstractStringRepr.__init__(self, *args) + self.ll = LLHelpers self.malloc = mallocstr -class UnicodeRepr(BaseStringRepr): +class UnicodeRepr(BaseLLStringRepr, AbstractUnicodeRepr): lowleveltype = Ptr(UNICODE) basetype = basestring base = UniChar CACHE = CONST_UNICODE_CACHE def __init__(self, *args): - BaseStringRepr.__init__(self, *args) + AbstractUnicodeRepr.__init__(self, *args) + self.ll = LLHelpers self.malloc = mallocunicode class CharRepr(AbstractCharRepr, StringRepr): @@ -802,6 +800,7 @@ StringRepr.repr = string_repr UnicodeRepr.repr = unicode_repr +UniCharRepr.repr = unicode_repr class BaseStringIteratorRepr(AbstractStringIteratorRepr): Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 16:04:57 2007 @@ -16,7 +16,10 @@ class AbstractCharRepr(AbstractStringRepr): pass -class AbstractUniCharRepr(Repr): +class AbstractUniCharRepr(AbstractStringRepr): + pass + +class AbstractUnicodeRepr(AbstractStringRepr): pass class __extend__(annmodel.SomeString): @@ -496,10 +499,12 @@ # # _________________________ Conversions _________________________ -class __extend__(pairtype(AbstractCharRepr, AbstractStringRepr)): +class __extend__(pairtype(AbstractCharRepr, AbstractStringRepr), + pairtype(AbstractUniCharRepr, AbstractUnicodeRepr)): def convert_from_to((r_from, r_to), v, llops): rstr = llops.rtyper.type_system.rstr - if r_from == rstr.char_repr and r_to == rstr.string_repr: + if (r_from == rstr.char_repr and r_to == rstr.string_repr) or\ + (r_from == rstr.unichar_repr and r_to == rstr.unicode_repr): return llops.gendirectcall(r_from.ll.ll_chr2str, v) return NotImplemented Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 16:04:57 2007 @@ -90,13 +90,14 @@ assert res is True def test_char_constant(self): + const = self.const def fn(s): - return s + '.' - res = self.interpret(fn, ['x']) + return s + const('.') + res = self.interpret(fn, [const('x')]) res = self.ll_to_string(res) assert len(res) == 2 - assert res[0] == 'x' - assert res[1] == '.' + assert res[0] == const('x') + assert res[1] == const('.') def test_char_isxxx(self): def fn(s): From fijal at codespeak.net Wed Nov 7 16:14:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 16:14:39 +0100 (CET) Subject: [pypy-svn] r48363 - pypy/branch/pypy-rpython-unicode/annotation Message-ID: <20071107151439.E02448189@code0.codespeak.net> Author: fijal Date: Wed Nov 7 16:14:37 2007 New Revision: 48363 Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Log: Remove unnecessary method (it's on SomeObject anyway) Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Wed Nov 7 16:14:37 2007 @@ -493,12 +493,6 @@ def method_isupper(chr): return s_Bool -class __extend__(SomeUnicodeCodePoint): - - def ord(uchr): - return SomeInteger(nonneg=True) - - class __extend__(SomeIterator): def iter(itr): From fijal at codespeak.net Wed Nov 7 16:15:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 16:15:09 +0100 (CET) Subject: [pypy-svn] r48364 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071107151509.3133F8189@code0.codespeak.net> Author: fijal Date: Wed Nov 7 16:15:08 2007 New Revision: 48364 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Eventual future support for unichar.isxxx, right now disabled (+ disable test) Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 16:15:08 2007 @@ -801,6 +801,8 @@ StringRepr.repr = string_repr UnicodeRepr.repr = unicode_repr UniCharRepr.repr = unicode_repr +UniCharRepr.char_repr = unichar_repr +CharRepr.char_repr = char_repr class BaseStringIteratorRepr(AbstractStringIteratorRepr): Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 16:15:08 2007 @@ -369,7 +369,8 @@ return r_str.ll.do_stringformat(hop, sourcevars) -class __extend__(AbstractCharRepr): +class __extend__(AbstractCharRepr, + AbstractUniCharRepr): def convert_const(self, value): if not isinstance(value, str) or len(value) != 1: @@ -395,13 +396,13 @@ return hop.inputconst(Bool, True) def rtype_ord(_, hop): - rstr = hop.rtyper.type_system.rstr - vlist = hop.inputargs(rstr.char_repr) + repr = hop.args_r[0].char_repr + vlist = hop.inputargs(repr) return hop.genop('cast_char_to_int', vlist, resulttype=Signed) def _rtype_method_isxxx(_, llfn, hop): - rstr = hop.rtyper.type_system.rstr - vlist = hop.inputargs(rstr.char_repr) + repr = hop.args_r[0].char_repr + vlist = hop.inputargs(repr) hop.exception_cannot_occur() return hop.gendirectcall(llfn, vlist[0]) @@ -581,7 +582,7 @@ c = ord(ch) return 65 <= c <= 90 - def ll_char_islower(ch): + def ll_char_islower(ch): c = ord(ch) return 97 <= c <= 122 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 16:15:08 2007 @@ -100,6 +100,7 @@ assert res[1] == const('.') def test_char_isxxx(self): + constchar = self.constchar def fn(s): return (s.isspace() | s.isdigit() << 1 | @@ -108,7 +109,7 @@ s.isupper() << 4 | s.islower() << 5) for i in range(128): - ch = chr(i) + ch = constchar(i) res = self.interpret(fn, [ch]) assert res == fn(ch) @@ -790,6 +791,7 @@ class BaseTestRstr(AbstractTestRstr): const = str + constchar = chr class TestLLtype(BaseTestRstr, LLRtypeMixin): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Wed Nov 7 16:15:08 2007 @@ -2,11 +2,16 @@ from pypy.rpython.test.tool import LLRtypeMixin from pypy.rpython.test.test_rstr import AbstractTestRstr +import py # ====> test_rstr.py class BaseTestRUnicode(AbstractTestRstr): const = unicode + constchar = unichr + + def test_char_isxxx(self): + py.test.skip("not supported") class TestLLtype(BaseTestRUnicode, LLRtypeMixin): pass From fijal at codespeak.net Wed Nov 7 16:23:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 16:23:08 +0100 (CET) Subject: [pypy-svn] r48365 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071107152308.6D30C8192@code0.codespeak.net> Author: fijal Date: Wed Nov 7 16:23:07 2007 New Revision: 48365 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: One test passing, one skipped Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 16:23:07 2007 @@ -802,7 +802,9 @@ UnicodeRepr.repr = unicode_repr UniCharRepr.repr = unicode_repr UniCharRepr.char_repr = unichar_repr +UnicodeRepr.char_repr = unichar_repr CharRepr.char_repr = char_repr +StringRepr.char_repr = char_repr class BaseStringIteratorRepr(AbstractStringIteratorRepr): Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 16:23:07 2007 @@ -312,48 +312,43 @@ rtype_inplace_add = rtype_add def rtype_eq((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) return hop.gendirectcall(r_str1.ll.ll_streq, v_str1, v_str2) def rtype_ne((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_streq, v_str1, v_str2) return hop.genop('bool_not', [vres], resulttype=Bool) def rtype_lt((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_lt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_le((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_le', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_ge((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_ge', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_gt((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_gt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr)): def rtype_contains((r_str, r_chr), hop): - rstr = hop.rtyper.type_system.rstr - v_str, v_chr = hop.inputargs(rstr.string_repr, rstr.char_repr) + string_repr = r_str.repr + char_repr = r_chr.char_repr + v_str, v_chr = hop.inputargs(string_repr, char_repr) return hop.gendirectcall(r_str.ll.ll_contains, v_str, v_chr) class __extend__(pairtype(AbstractStringRepr, AbstractTupleRepr)): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 16:23:07 2007 @@ -114,26 +114,31 @@ assert res == fn(ch) def test_char_compare(self): - res = self.interpret(lambda c1, c2: c1 == c2, ['a', 'b']) + const = self.const + res = self.interpret(lambda c1, c2: c1 == c2, [const('a'), + const('b')]) assert res is False - res = self.interpret(lambda c1, c2: c1 == c2, ['a', 'a']) + res = self.interpret(lambda c1, c2: c1 == c2, [const('a'), + const('a')]) assert res is True - res = self.interpret(lambda c1, c2: c1 <= c2, ['z', 'a']) + res = self.interpret(lambda c1, c2: c1 <= c2, [const('z'), + const('a')]) assert res is False def test_char_mul(self): + const = self.const def fn(c, mul): s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(s[i]) - ord('0') + res = res*10 + ord(const(s[i])) - ord(const('0')) c2 = c c2 *= mul res = 10 * res + (c2 == s) return res - res = self.interpret(fn, ['3', 5]) + res = self.interpret(fn, [const('3'), 5]) assert res == 333331 - res = self.interpret(fn, ['5', 3]) + res = self.interpret(fn, [const('5'), 3]) assert res == 5551 def test_unichar_const(self): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Wed Nov 7 16:23:07 2007 @@ -13,5 +13,8 @@ def test_char_isxxx(self): py.test.skip("not supported") + def test_char_mul(self): + py.test.skip("not supported by now") + class TestLLtype(BaseTestRUnicode, LLRtypeMixin): pass From fijal at codespeak.net Wed Nov 7 18:00:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 18:00:30 +0100 (CET) Subject: [pypy-svn] r48366 - pypy/branch/pypy-rpython-unicode/annotation Message-ID: <20071107170030.C0C448185@code0.codespeak.net> Author: fijal Date: Wed Nov 7 18:00:28 2007 New Revision: 48366 Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Log: Allow unichr * integer Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/binaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Wed Nov 7 18:00:28 2007 @@ -656,10 +656,9 @@ getitem_idx_key = getitem_idx - # uncomment if we really want to support that - #def mul((str1, int2)): # xxx do we want to support this - # getbookkeeper().count("str_mul", str1, int2) - # return SomeString() + def mul((str1, int2)): # xxx do we want to support this + getbookkeeper().count("str_mul", str1, int2) + return SomeUnicodeString() class __extend__(pairtype(SomeInteger, SomeString)): From fijal at codespeak.net Wed Nov 7 18:00:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 18:00:45 +0100 (CET) Subject: [pypy-svn] r48367 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem module test Message-ID: <20071107170045.714E58189@code0.codespeak.net> Author: fijal Date: Wed Nov 7 18:00:44 2007 New Revision: 48367 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/module/support.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py pypy/branch/pypy-rpython-unicode/rpython/test/tool.py Log: Whack, whack.. another tests are passing Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 18:00:44 2007 @@ -197,6 +197,13 @@ s.chars[0] = ch return s + def ll_str2unicode(str): + lgt = len(str.chars) + s = mallocunicode(lgt) + for i in range(lgt): + s.chars[i] = cast_primitive(UniChar, str.chars[i]) + return s + def ll_strhash(s): # unlike CPython, there is no reason to avoid to return -1 # but our malloc initializes the memory to zero, so we use zero as the Modified: pypy/branch/pypy-rpython-unicode/rpython/module/support.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/module/support.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/module/support.py Wed Nov 7 18:00:44 2007 @@ -29,6 +29,16 @@ return p to_rstr = staticmethod(to_rstr) + def to_runicode(s): + from pypy.rpython.lltypesystem.rstr import UNICODE, mallocunicode + if s is None: + return lltype.nullptr(UNICODE) + p = mallocunicode(len(s)) + for i in range(len(s)): + p.chars[i] = s[i] + return p + to_runicode = staticmethod(to_runicode) + def from_rstr(rs): if not rs: # null pointer return None Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 18:00:44 2007 @@ -82,7 +82,11 @@ v_str, = hop.inputargs(string_repr) c_zero = inputconst(Signed, 0) v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero) - return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) + if string_repr is hop.rtyper.type_system.rstr.string_repr: + return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed) + else: + assert string_repr is hop.rtyper.type_system.rstr.unicode_repr + return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed) def rtype_method_startswith(self, hop): str1_repr, str2_repr = self._str_reprs(hop) @@ -238,6 +242,12 @@ hop.exception_is_here() return hop.gendirectcall(self.ll.ll_int, v_str, v_base) + def rtype_unicode(self, hop): + repr = hop.args_r[0].repr + v_str = hop.inputarg(repr, 0) + hop.exception_cannot_occur() + return hop.gendirectcall(self.ll.ll_str2unicode, v_str) + def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it string_repr = hop.rtyper.type_system.rstr.string_repr @@ -414,18 +424,21 @@ def rtype_method_islower(self, hop): return self._rtype_method_isxxx(self.ll.ll_char_islower, hop) -class __extend__(pairtype(AbstractCharRepr, IntegerRepr)): +class __extend__(pairtype(AbstractCharRepr, IntegerRepr), + pairtype(AbstractUniCharRepr, IntegerRepr)): def rtype_mul((r_chr, r_int), hop): + char_repr = r_chr.char_repr rstr = hop.rtyper.type_system.rstr - v_char, v_int = hop.inputargs(rstr.char_repr, Signed) + v_char, v_int = hop.inputargs(char_repr, Signed) return hop.gendirectcall(r_chr.ll.ll_char_mul, v_char, v_int) rtype_inplace_mul = rtype_mul -class __extend__(pairtype(IntegerRepr, AbstractCharRepr)): +class __extend__(pairtype(IntegerRepr, AbstractCharRepr), + pairtype(IntegerRepr, AbstractUniCharRepr)): def rtype_mul((r_int, r_chr), hop): - rstr = hop.rtyper.type_system.rstr - v_int, v_char = hop.inputargs(Signed, rstr.char_repr) + char_repr = r_chr.char_repr + v_int, v_char = hop.inputargs(Signed, char_repr) return hop.gendirectcall(r_chr.ll.ll_char_mul, v_char, v_int) rtype_inplace_mul = rtype_mul Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Wed Nov 7 18:00:44 2007 @@ -10,11 +10,38 @@ const = unicode constchar = unichr + def test_unicode_explicit_conv(self): + def f(x): + return unicode(x) + + for v in ['x', u'x']: + res = self.interpret(f, [v]) + assert self.ll_to_unicode(res) == v + + def f(x): + if x > 1: + y = const('yxx') + else: + y = const('xx') + return unicode(y) + + const = str + assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + + def f(x): + if x > 1: + y = const('yxx') + else: + y = const('xx') + return unicode(y) + + # a copy, because llinterp caches functions + + const = unicode + assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + def test_char_isxxx(self): py.test.skip("not supported") - def test_char_mul(self): - py.test.skip("not supported by now") - class TestLLtype(BaseTestRUnicode, LLRtypeMixin): pass Modified: pypy/branch/pypy-rpython-unicode/rpython/test/tool.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/tool.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/tool.py Wed Nov 7 18:00:44 2007 @@ -34,10 +34,17 @@ def ll_to_string(self, s): return ''.join(s.chars) + def ll_to_unicode(self, s): + return u''.join(s.chars) + def string_to_ll(self, s): from pypy.rpython.module.support import LLSupport return LLSupport.to_rstr(s) + def unicode_to_ll(self, s): + from pypy.rpython.module.support import LLSupport + return LLSupport.to_runicode(s) + def ll_to_list(self, l): r = [] items = l.ll_items() From cfbolz at codespeak.net Wed Nov 7 19:27:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 19:27:22 +0100 (CET) Subject: [pypy-svn] r48368 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071107182722.C1B2B81BC@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 19:27:19 2007 New Revision: 48368 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py Log: sprinkle some debug prints, disabled by default. I want to see what is taking so long during the collections. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Nov 7 19:27:19 2007 @@ -12,6 +12,7 @@ # pointer to a young object. GCFLAG_NO_YOUNG_PTRS = 2 << GCFLAGSHIFT +DEBUG_PRINT = False class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional @@ -136,7 +137,12 @@ self.reset_young_gcflags() # we are doing a full collection anyway self.weakrefs_grow_older() self.reset_nursery() + if DEBUG_PRINT: + llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) + if DEBUG_PRINT and not size_changing: + llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size) + def trace_and_copy(self, obj): # during a full collect, all objects copied might come from the nursery and @@ -171,12 +177,13 @@ debug_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: - #llop.debug_print(lltype.Void, "minor collect") + if DEBUG_PRINT: + llop.debug_print(lltype.Void, "minor collect") # a nursery-only collection - scan = self.free + scan = beginning = self.free self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() - self.scan_objects_just_copied_out_of_nursery(scan) + scan = self.scan_objects_just_copied_out_of_nursery(scan) # at this point, all static and old objects have got their # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery if self.young_objects_with_weakrefs.non_empty(): @@ -184,6 +191,8 @@ self.notify_objects_just_moved() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, True) + if DEBUG_PRINT: + llop.debug_print(lltype.Void, "percent survived:", float(scan - beginning) / self.nursery_size) else: # no nursery - this occurs after a full collect, triggered either # just above or by some previous non-nursery-based allocation. @@ -201,23 +210,31 @@ # Follow the old_objects_pointing_to_young list and move the # young objects they point to out of the nursery. The 'forw' # fields are reset to their correct value along the way. + count = 0 obj = self.old_objects_pointing_to_young while obj: + count += 1 nextobj = self.header(obj).forw self.init_forwarding(obj) self.trace_and_drag_out_of_nursery(obj) obj = nextobj + if DEBUG_PRINT: + llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): roots = self.get_roots(with_static=False) + count = 0 while 1: root = roots.pop() if root == NULL: break + count += 1 obj = root.address[0] if self.is_in_nursery(obj): root.address[0] = self.copy(obj) + if DEBUG_PRINT: + llop.debug_print(lltype.Void, "collect_roots_in_nursery", count) free_non_gc_object(roots) def scan_objects_just_copied_out_of_nursery(self, scan): From rxe at codespeak.net Wed Nov 7 20:26:53 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 7 Nov 2007 20:26:53 +0100 (CET) Subject: [pypy-svn] r48369 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071107192653.8BC0C8165@code0.codespeak.net> Author: rxe Date: Wed Nov 7 20:26:51 2007 New Revision: 48369 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/funcnode.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/support.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: some mindless whacking with llvm 2.0 binaries in place of 1.9 ones; it's a start and some tests are passing now Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Wed Nov 7 20:26:51 2007 @@ -3,7 +3,7 @@ class ArrayNode(ConstantNode): __slots__ = "db value arraytype".split() - prefix = '%arrayinstance' + prefix = '@arrayinstance' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) @@ -47,7 +47,7 @@ else: ref = self.db.get_childref(p, c) - ref = "cast(%s* %s to %s*)" % (self.get_typerepr(), + ref = "bitcast(%s* %s to %s*)" % (self.get_typerepr(), ref, typeval) return ref @@ -57,11 +57,11 @@ assert p is None, "child PBC arrays are NOT needed by rtyper" fromptr = "%s*" % self.get_typerepr() - ref = "cast(%s %s to %s)" % (fromptr, self.name, toptr) + ref = "bitcast(%s %s to %s)" % (fromptr, self.name, toptr) return ref def get_childref(self, index): - return "getelementptr(%s* %s, int 0, uint 1, int %s)" % ( + return "getelementptr(%s* %s, i32 0, i32 1, i32 %s)" % ( self.get_typerepr(), self.name, index) @@ -87,7 +87,7 @@ return "[%s x %s]" % (arraylen, typeval) def get_childref(self, index): - return "getelementptr(%s* %s, int 0, int %s)" %( + return "getelementptr(%s* %s, i32 0, i32 %s)" %( self.get_typerepr(), self.name, index) @@ -124,7 +124,7 @@ class VoidArrayNode(ConstantNode): __slots__ = "db value".split() - prefix = '%voidarrayinstance' + prefix = '@voidarrayinstance' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Wed Nov 7 20:26:51 2007 @@ -91,12 +91,7 @@ def cmds_bytecode(self, base): # run llvm assembler and optimizer opts = self.optimizations() - - if llvm_version() < 2.0: - self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (base, opts, base)) - else: - # we generate 1.x .ll files, so upgrade these first - self.cmds.append("llvm-upgrade < %s.ll | llvm-as | opt %s -f -o %s.bc" % (base, opts, base)) + self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (base, opts, base)) def cmds_objects(self, base): # XXX why this hack??? Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 7 20:26:51 2007 @@ -88,14 +88,14 @@ def startimpl(self): self.newline() - self._append("implementation") + #self._append("implementation") self.newline() def br_uncond(self, blockname): self._indent("br label %%%s" %(blockname,)) def br(self, cond, blockname_false, blockname_true): - self._indent("br bool %s, label %%%s, label %%%s" + self._indent("br i1 %s, label %%%s, label %%%s" % (cond, blockname_true, blockname_false)) def switch(self, intty, cond, defaultdest, value_labels): @@ -111,7 +111,7 @@ if linkage is None: linkage = self.linkage self.newline() - self._append("%s%s %s {" % (linkage, cconv, decl,)) + self._append("define %s%s %s {" % (linkage, cconv, decl,)) def closefunc(self): self._append("}") @@ -138,10 +138,10 @@ self._indent("%s = %s %s %s, ubyte %s" % (targetvar, name, type_, ref1, ref2)) - def cast(self, targetvar, fromtype, fromvar, targettype): + def cast(self, targetvar, fromtype, fromvar, targettype, casttype='bitcast'): if fromtype == 'void' and targettype == 'void': return - self._indent("%(targetvar)s = cast %(fromtype)s " + self._indent("%(targetvar)s = %(casttype)s %(fromtype)s " "%(fromvar)s to %(targettype)s" % locals()) def getelementptr(self, targetvar, type, typevar, indices, getptr=True): Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Nov 7 20:26:51 2007 @@ -347,29 +347,29 @@ def __init__(self, database): self.database = database self.types = { - lltype.Char: "sbyte", - lltype.Bool: "bool", + lltype.Char: "i8", + lltype.Bool: "i1", lltype.SingleFloat: "float", lltype.Float: "double", - lltype.UniChar: "uint", + lltype.UniChar: "i8", lltype.Void: "void", - lltype.UnsignedLongLong: "ulong", - lltype.SignedLongLong: "long", - llmemory.Address: "sbyte*", + lltype.UnsignedLongLong: "i64", + lltype.SignedLongLong: "i64", + llmemory.Address: "i8*", #llmemory.WeakGcAddress: "sbyte*", } # 32 bit platform if sys.maxint == 2**31-1: self.types.update({ - lltype.Signed: "int", - lltype.Unsigned: "uint" }) + lltype.Signed: "i32", + lltype.Unsigned: "i32" }) # 64 bit platform elif sys.maxint == 2**63-1: self.types.update({ - lltype.Signed: "long", - lltype.Unsigned: "ulong" }) + lltype.Signed: "i64", + lltype.Unsigned: "i64" }) else: raise Exception("Unsupported platform - unknown word size") @@ -553,9 +553,9 @@ r = self.database.repr_type indices_as_str = ", ".join("%s %s" % (w, i) for w, i in indices) - return "cast(%s* getelementptr(%s* null, %s) to int)" % (r(to), - r(from_), - indices_as_str) + return "ptrtoint(%s* getelementptr(%s* null, %s) to i32)" % (r(to), + r(from_), + indices_as_str) def get_offset(self, value, initialindices=None): " return (from_type, (indices, ...), to_type) " Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Nov 7 20:26:51 2007 @@ -11,17 +11,23 @@ from pypy.tool.udir import udir support_functions = [ - "%raisePyExc_IOError", - "%raisePyExc_ValueError", - "%raisePyExc_OverflowError", - "%raisePyExc_ZeroDivisionError", - "%raisePyExc_RuntimeError", - "%raisePyExc_thread_error", - "%RPyString_FromString", - "%RPyString_AsString", - "%RPyString_Size", - "%RPyExceptionOccurred", - "%LLVM_RPython_StartupCode", + "@raisePyExc_IOError", + "@raisePyExc_ValueError", + "@raisePyExc_OverflowError", + "@raisePyExc_ZeroDivisionError", + "@raisePyExc_RuntimeError", + "@raisePyExc_thread_error", + "@RPyString_FromString", + "@RPyString_AsString", + "@RPyString_Size", + "@RPyExceptionOccurred", + "@LLVM_RPython_StartupCode", + ] + + +skip_lines = [ + "%RPyString = type opaque", + "__main", ] def get_module_file(name): @@ -59,14 +65,15 @@ # strip declares that are in funcnames for line in llcode.split('\n'): - # For some reason gcc introduces this and then we cant resolve it - # XXX Get rid of this - when got more time on our hands - if line.find("__main") >= 1: - continue - # get rid of any of the structs that llvm-gcc introduces to struct types line = line.replace("%struct.", "%") + # XXX slowwwwwww + for x in skip_lines: + if line.find(x) >= 0: + line = '' + break + # strip comments comment = line.find(';') if comment >= 0: @@ -74,18 +81,18 @@ line = line.rstrip() # find function names, declare them with the default calling convertion - if '(' in line and line[-1:] == '{': - returntype, s = line.split(' ', 1) - funcname , s = s.split('(', 1) - funcnames[funcname] = True - if line.find("internal") == -1: - if funcname not in ["%main", "%ctypes_RPython_StartupCode"]: - internal = 'internal ' - line = '%s%s %s' % (internal, default_cconv, line,) + #if '(' in line and line[-1:] == '{': + # returntype, s = line.split(' ', 1) + # funcname , s = s.split('(', 1) + # funcnames[funcname] = True + # if line.find("internal") == -1: + # if funcname not in ["@main", "@ctypes_RPython_StartupCode"]: + # internal = 'internal ' + # line = '%s%s %s' % (internal, default_cconv, line,) ll_lines.append(line) - # patch calls to function that we just declared with differnet cconv - ll_lines2, calltag, declaretag = [], 'call ', 'declare ' + # patch calls to function that we just declared with different cconv + ll_lines2, calltag, declaretag, definetag = [], 'call ', 'declare ', 'define ' for line in ll_lines: i = line.find(calltag) if i >= 0: @@ -102,15 +109,24 @@ cconv = default_cconv break line = "declare %s %s" % (cconv, line[len(declaretag):]) + if line[:len(definetag)] == definetag: + line = line.replace("internal ", "") + cconv = 'ccc' + for funcname in funcnames.keys(): + if line.find(funcname) >= 0: + cconv = default_cconv + break + line = "define %s %s" % (cconv, line[len(definetag):]) ll_lines2.append(line) - ll_lines2.append("declare ccc void %abort()") + ll_lines2.append("declare ccc void @abort()") llcode = '\n'.join(ll_lines2) - try: - decl, impl = llcode.split('implementation') - except: - raise "Can't compile external function code (llcode.c): ERROR:", llcode + decl, impl = '', llcode + #try: + # decl, impl = llcode.split('implementation') + #except: + # raise Exception("Can't compile external function code (llcode.c)") return decl, impl @@ -172,7 +188,7 @@ def predeclarefn(c_name, llname): function_names.append(llname) - assert llname[0] == "%" + assert llname[0] == "@" llname = llname[1:] assert '\n' not in llname ccode.append('#define\t%s\t%s\n' % (c_name, llname)) Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Wed Nov 7 20:26:51 2007 @@ -14,11 +14,11 @@ # signature of external functions differ from C's implementation ext_func_sigs = { - "%LL_stack_too_big" : ExtFuncSig("int", None), + "@LL_stack_too_big" : ExtFuncSig("int", None), } if maxint != 2**31-1: - ext_func_sigs["%LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) + ext_func_sigs["@LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) class SimplerExternalFuncNode(FuncNode): @@ -26,7 +26,7 @@ def __init__(self, db, value): self.db = db self.value = value - self.name = "%" + value._name + self.name = "@" + value._name def external_c_source(self): # return a list of unique includes and sources in C Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Wed Nov 7 20:26:51 2007 @@ -11,7 +11,7 @@ pass class FuncImplNode(FuncNode): - prefix = '%pypy_' + prefix = '@pypy_' __slots__ = "db value graph block_to_name bad_switch_block".split() def __init__(self, db, value): Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Wed Nov 7 20:26:51 2007 @@ -81,7 +81,7 @@ # malloc_size is unsigned right now codewriter.malloc(targetvar, "sbyte", size) # XXX uses own cconv - codewriter.call(None, 'void', '%llvm.memset' + postfix(), + codewriter.call(None, 'void', '@llvm.memset' + postfix(), ['sbyte*', 'ubyte', uword, uword], [targetvar, 0, size, boundary_size], cconv='ccc') @@ -110,7 +110,7 @@ word = self.db.get_machine_word() uword = self.db.get_machine_uword() - fnname = '%pypy_malloc' + (atomic and '_atomic' or '') + fnname = '@pypy_malloc' + (atomic and '_atomic' or '') ## XXX (arigo) disabled the ring buffer for comparison purposes ## XXX until we know if it's a valid optimization or not @@ -121,20 +121,20 @@ ## atomic = False # malloc_size is unsigned right now - sizeu = '%malloc_sizeu' + self.get_count() - codewriter.cast(sizeu, word, size, uword) - codewriter.call(targetvar, 'sbyte*', fnname, [word], [size]) + #sizeu = '%malloc_sizeu' + self.get_count() + #codewriter.cast(sizeu, word, size, uword) + codewriter.call(targetvar, 'i8*', fnname, [word], [size]) if atomic: # XXX uses own cconv - codewriter.call(None, 'void', '%llvm.memset' + postfix(), - ['sbyte*', 'ubyte', uword, uword], - [targetvar, 0, sizeu, boundary_size], + codewriter.call(None, 'void', '@llvm.memset' + postfix(), + ['i8*', 'i8', word, word], + [targetvar, 0, size, boundary_size], cconv='ccc') def op__collect(self, codewriter, opr): - codewriter.call(opr.retref, opr.rettype, "%pypy_gc__collect", + codewriter.call(opr.retref, opr.rettype, "@pypy_gc__collect", opr.argtypes, opr.argrefs) class RefcountingGcPolicy(RawGcPolicy): Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 7 20:26:51 2007 @@ -225,12 +225,12 @@ write_raise_exc(c_name, exc_repr, codewriter) def write_setup_impl(self, codewriter): - open_decl = "sbyte* %LLVM_RPython_StartupCode()" + open_decl = "i8* @LLVM_RPython_StartupCode()" codewriter.openfunc(open_decl) for node in self.db.getnodes(): node.writesetupcode(codewriter) - codewriter.ret("sbyte*", "null") + codewriter.ret("i8*", "null") codewriter.closefunc() def compile_module(self): Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Wed Nov 7 20:26:51 2007 @@ -1,35 +1,45 @@ extdeclarations = """ -declare ccc uint %strlen(sbyte*) -declare ccc void %llvm.memsetPOSTFIX(sbyte*, ubyte, UWORD, UWORD) -declare ccc void %llvm.memcpyPOSTFIX(sbyte*, sbyte*, UWORD, UWORD) +declare ccc i32 @strlen(i8*) """ extfunctions = """ -internal CC sbyte* %RPyString_AsString(%RPyString* %structstring) { - %source1ptr = getelementptr %RPyString* %structstring, int 0, uint 1, uint 1 - %source1 = cast [0 x sbyte]* %source1ptr to sbyte* - ret sbyte* %source1 +define internal CC %RPyString* @RPyString_FromString(i8* %s) { + %len = call ccc i32 @strlen(i8* %s) + %rpy = call CC %RPyString* @pypy_RPyString_New__Signed(i32 %len) + %rpystrptr = getelementptr %RPyString* %rpy, i32 0, i32 1, i32 1 + %rpystr = bitcast [0 x i8]* %rpystrptr to i8* + + call ccc void @llvm.memcpyPOSTFIX(i8* %rpystr, i8* %s, WORD %len, WORD 0) + + ret %RPyString* %rpy } -internal CC WORD %RPyString_Size(%RPyString* %structstring) { - %sizeptr = getelementptr %RPyString* %structstring, int 0, uint 1, uint 0 +define internal CC i8* @RPyString_AsString(%RPyString* %structstring) { + %source1ptr = getelementptr %RPyString* %structstring, i32 0, i32 1, i32 1 + %source1 = bitcast [0 x i8]* %source1ptr to i8* + ret i8* %source1 +} + +define internal CC WORD @RPyString_Size(%RPyString* %structstring) { + %sizeptr = getelementptr %RPyString* %structstring, i32 0, i32 1, i32 0 %size = load WORD* %sizeptr ret WORD %size } -internal CC %RPyString* %RPyString_FromString(sbyte* %s) { - %lenu = call ccc uint %strlen(sbyte* %s) - %lenuword = cast uint %lenu to UWORD - %lenword = cast uint %lenu to WORD - %rpy = call CC %RPyString* %pypy_RPyString_New__Signed(WORD %lenword) - %rpystrptr = getelementptr %RPyString* %rpy, int 0, uint 1, uint 1 - %rpystr = cast [0 x sbyte]* %rpystrptr to sbyte* - - call ccc void %llvm.memcpyPOSTFIX(sbyte* %rpystr, sbyte* %s, UWORD %lenuword, UWORD 0) - - ret %RPyString* %rpy +define internal CC double @pypyop_float_abs(double %x) { +block0: + %cond1 = fcmp ugt double %x, 0.0 + br i1 %cond1, label %return_block, label %block1 +block1: + %x2 = sub double 0.0, %x + br label %return_block +return_block: + %result = phi double [%x, %block0], [%x2, %block1] + ret double %result } +""" +""" internal CC WORD %pypyop_int_abs(WORD %x) { block0: %cond1 = setge WORD %x, 0 @@ -54,18 +64,6 @@ ret long %result } -internal CC double %pypyop_float_abs(double %x) { -block0: - %cond1 = setge double %x, 0.0 - br bool %cond1, label %return_block, label %block1 -block1: - %x2 = sub double 0.0, %x - br label %return_block -return_block: - %result = phi double [%x, %block0], [%x2, %block1] - ret double %result -} - """ extfunctions_standalone = """ @@ -81,10 +79,9 @@ """ - def write_raise_exc(c_name, exc_repr, codewriter): l = """ -internal CC void %%raise%s(sbyte* %%msg) { +define internal CC void @raise%s(i8* %%msg) { ;%%exception_value = cast %s to %%RPYTHON_EXCEPTION* ret void } Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Wed Nov 7 20:26:51 2007 @@ -52,12 +52,6 @@ def from_unichar(arg): return ord(arg) -def to_bool(res): - return bool(res) - -def to_unichar(res): - return unichr(res) - def from_str(arg): # XXX wont work over isolate : arg should be converted into a string first n = len(arg.chars) @@ -190,10 +184,10 @@ def build_lltype_to_ctypes_to_res(self, T): if T is lltype.Bool: - action = 'to_bool' + action = 'bool' elif T is lltype.UniChar: - action = 'to_unichar' + action = 'unichr' elif T is lltype.Unsigned: action = 'r_uint' Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Wed Nov 7 20:26:51 2007 @@ -39,51 +39,64 @@ self.functionref = '%pypyop_' + op.opname class OpWriter(object): - + + shift_operations = { + # ZZZ all these + 'int_lshift': 'shl', + 'int_rshift': 'shr', + + 'uint_lshift': 'shl', + 'uint_rshift': 'shr', + + 'llong_lshift': 'shl', + 'llong_rshift': 'shr', + } + binary_operations = { 'float_mul' : 'mul', 'float_add' : 'add', 'float_sub' : 'sub', - 'float_truediv' : 'div', - - 'ptr_eq' : 'seteq', - 'ptr_ne' : 'setne' } + #ZZZ'float_truediv' : 'div', + 'ptr_eq' : 'icmp eq', + 'ptr_ne' : 'icmp ne' } # generic numeric ops for tt in 'int llong ullong uint'.split(): for oo in 'mul add sub and or xor'.split(): binary_operations['%s_%s' % (tt, oo)] = oo - binary_operations['%s_floordiv' % tt] = 'div' - binary_operations['%s_mod' % tt] = 'rem' + + #ZZZbinary_operations['%s_floordiv' % tt] = 'div' + #ZZZbinary_operations['%s_mod' % tt] = 'rem' + # comparison ops - for tt in 'int llong ullong uint unichar float'.split(): - for oo in 'lt le eq ne ge gt'.split(): - binary_operations['%s_%s' % (tt, oo)] = 'set%s' % oo + for tt in 'int llong unichar'.split(): + for oo in 'eq ne'.split(): + binary_operations['%s_%s' % (tt, oo)] = 'icmp %s' % oo + for oo in 'lt le ge gt'.split(): + binary_operations['%s_%s' % (tt, oo)] = 'icmp s%s' % oo + + for tt in 'ullong uint'.split(): + for oo in 'eq ne'.split(): + binary_operations['%s_%s' % (tt, oo)] = 'icmp %s' % oo + for oo in 'lt le ge gt'.split(): + binary_operations['%s_s%s' % (tt, oo)] = 'icmp u%s' % oo + for tt in 'float'.split(): + for oo in 'lt le eq ne ge gt'.split(): + binary_operations['%s_%s' % (tt, oo)] = 'fcmp u%s' % oo - shift_operations = {'int_lshift': 'shl', - 'int_rshift': 'shr', - - 'uint_lshift': 'shl', - 'uint_rshift': 'shr', - - 'llong_lshift': 'shl', - 'llong_rshift': 'shr', - } - - char_operations = {'char_lt': 'setlt', - 'char_le': 'setle', - 'char_eq': 'seteq', - 'char_ne': 'setne', - 'char_ge': 'setge', - 'char_gt': 'setgt'} + char_operations = {'char_lt': 'icmp lt', + 'char_le': 'icmp le', + 'char_eq': 'icmp eq', + 'char_ne': 'icmp ne', + 'char_ge': 'icmp ge', + 'char_gt': 'icmp gt'} def __init__(self, db, codewriter): self.db = db self.codewriter = codewriter self.word = db.get_machine_word() - self.uword = db.get_machine_uword() def _tmp(self, count=1): if count == 1: @@ -97,13 +110,15 @@ if isinstance(ARRAYTYPE, lltype.Array): if not ARRAYTYPE._hints.get("nolength", False): # skip the length field - indices.append((self.uword, 1)) + indices.append((self.word, 1)) else: assert isinstance(ARRAYTYPE, lltype.FixedSizeArray) return indices def write_operation(self, op): - #log(op) + + if self.db.genllvm.config.translation.llvm.debug: + self.codewriter.comment(str(op)) if op.opname in ("direct_call", 'indirect_call'): opr = OpReprCall(op, self.db) @@ -113,12 +128,16 @@ if op.opname.startswith('gc'): meth = getattr(self.db.gcpolicy, 'op' + op.opname[2:]) meth(self.codewriter, opr) + elif op.opname in self.binary_operations: self.binaryop(opr) + elif op.opname in self.shift_operations: self.shiftop(opr) + elif op.opname in self.char_operations: self.char_binaryop(opr) + elif op.opname.startswith('cast_') or op.opname.startswith('truncate_'): if op.opname == 'cast_char_to_int': self.cast_char_to_int(opr) @@ -128,10 +147,6 @@ meth = getattr(self, op.opname, None) if not meth: raise Exception, "operation %s not found" % op.opname - - # XXX bit unclean - if self.db.genllvm.config.translation.llvm.debug: - self.codewriter.comment(str(op)) meth(opr) def _generic_pow(self, opr, onestr): @@ -169,7 +184,7 @@ def int_abs(self, opr): assert len(opr.argrefs) == 1 - functionref = '%pypyop_' + opr.op.opname + functionref = '@pypyop_' + opr.op.opname self.codewriter.call(opr.retref, opr.rettype, functionref, opr.argtypes, opr.argrefs) @@ -234,6 +249,7 @@ def cast_char_to_int(self, opr): " works for all casts " + XXX assert len(opr.argrefs) == 1 intermediate = self._tmp() self.codewriter.cast(intermediate, opr.argtypes[0], @@ -242,13 +258,38 @@ def cast_primitive(self, opr): " works for all casts " - #assert len(opr.argrefs) == 1 + fromtype = opr.argtypes[0] + totype = opr.rettype + casttype = "bitcast" + if '*' not in fromtype: + if fromtype[0] == 'i' and totype[0] == 'i': + fromsize = int(fromtype[1:]) + tosize = int(totype[1:]) + if tosize > fromsize: + # ZZZ signed + casttype = "zext" + elif tosize < fromsize: + casttype = "trunc" + else: + pass + else: + if (fromtype[0] == 'i' and totype == 'double'): + # ZZZ signed + casttype = "sitofp" + elif (fromtype == 'double' and totype[0] == 'i'): + # ZZZ signed + casttype = "fptosi" + else: + assert False, "this shouldtnt be possible" + else: + assert '*' in totype + self.codewriter.cast(opr.retref, opr.argtypes[0], - opr.argrefs[0], opr.rettype) + opr.argrefs[0], opr.rettype, casttype) same_as = cast_primitive def int_is_true(self, opr): - self.codewriter.binaryop("setne", opr.retref, opr.argtypes[0], + self.codewriter.binaryop("icmp ne", opr.retref, opr.argtypes[0], opr.argrefs[0], "0") uint_is_true = int_is_true llong_is_true = int_is_true @@ -258,11 +299,11 @@ opr.argrefs[0], "0.0") def ptr_nonzero(self, opr): - self.codewriter.binaryop("setne", opr.retref, opr.argtypes[0], + self.codewriter.binaryop("icmp ne", opr.retref, opr.argtypes[0], opr.argrefs[0], "null") def ptr_iszero(self, opr): - self.codewriter.binaryop("seteq", opr.retref, opr.argtypes[0], + self.codewriter.binaryop("icmp eq", opr.retref, opr.argtypes[0], opr.argrefs[0], "null") def direct_call(self, opr): @@ -284,7 +325,7 @@ def call_boehm_gc_alloc(self, opr): word = self.db.get_machine_word() - self.codewriter.call(opr.retref, 'sbyte*', '%pypy_malloc', + self.codewriter.call(opr.retref, 'i8*', '%pypy_malloc', [word], [opr.argrefs[0]]) def to_getelementptr(self, TYPE, args): @@ -302,19 +343,19 @@ indexref = self.db.repr_arg(arg) if isinstance(TYPE, lltype.FixedSizeArray): - indices.append(("int", indexref)) + indices.append(("i32", indexref)) TYPE = TYPE.OF elif isinstance(TYPE, lltype.Array): if not TYPE._hints.get("nolength", False): - indices.append(("uint", 1)) - indices.append(("int", indexref)) + indices.append(("i32", 1)) + indices.append(("i32", indexref)) TYPE = TYPE.OF elif isinstance(TYPE, lltype.Struct): assert name is not None TYPE = getattr(TYPE, name) - indices.append(("uint", indexref)) + indices.append(("i32", indexref)) else: raise Exception("unsupported type: %s" % TYPE) @@ -369,7 +410,7 @@ TYPE, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) if isinstance(TYPE, lltype.Array): # gets the length - indices.append(("uint", 0)) + indices.append(("i32", 0)) lengthref = self._tmp() self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices) else: @@ -393,7 +434,7 @@ assert index != -1 tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, opr.argtypes[0], - opr.argrefs[0], [(self.uword, index)]) + opr.argrefs[0], [(self.word, index)]) # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it @@ -435,9 +476,10 @@ def _op_adr_generic(self, opr, llvm_op): addr, res = self._tmp(2) - self.codewriter.cast(addr, opr.argtypes[0], opr.argrefs[0], self.word) + + self.codewriter.cast(addr, opr.argtypes[0], opr.argrefs[0], self.word, 'ptrtoint') self.codewriter.binaryop(llvm_op, res, self.word, addr, opr.argrefs[1]) - self.codewriter.cast(opr.retref, self.word, res, opr.rettype) + self.codewriter.cast(opr.retref, self.word, res, opr.rettype, 'inttoptr') def adr_add(self, opr): self._op_adr_generic(opr, "add") @@ -447,34 +489,34 @@ def _op_adr_cmp(self, opr, llvm_op): addr1, addr2 = self._tmp(2) - self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word) - self.codewriter.cast(addr2, opr.argtypes[1], opr.argrefs[1], self.word) - assert opr.rettype == "bool" + self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word, 'ptrtoint') + self.codewriter.cast(addr2, opr.argtypes[1], opr.argrefs[1], self.word, 'ptrtoint') + assert opr.rettype == "i1" self.codewriter.binaryop(llvm_op, opr.retref, self.word, addr1, addr2) def adr_eq(self, opr): - self._op_adr_cmp(opr, "seteq") + self._op_adr_cmp(opr, "icmp eq") def adr_ne(self, opr): - self._op_adr_cmp(opr, "setne") + self._op_adr_cmp(opr, "icmp ne") def adr_le(self, opr): - self._op_adr_cmp(opr, "setle") + self._op_adr_cmp(opr, "icmp le") def adr_gt(self, opr): - self._op_adr_cmp(opr, "setgt") + self._op_adr_cmp(opr, "icmp gt") def adr_lt(self, opr): - self._op_adr_cmp(opr, "setlt") + self._op_adr_cmp(opr, "icmp lt") def adr_ge(self, opr): - self._op_adr_cmp(opr, "setge") + self._op_adr_cmp(opr, "icmp ge") # XXX Not sure any of this makes sense - maybe seperate policy for # different flavours of mallocs? Well it depend on what happens the GC # developments def raw_malloc(self, opr): - self.codewriter.call(opr.retref, opr.rettype, "%raw_malloc", + self.codewriter.call(opr.retref, opr.rettype, "@raw_malloc", opr.argtypes, opr.argrefs) def raw_malloc_usage(self, opr): @@ -482,15 +524,15 @@ opr.rettype) def raw_free(self, opr): - self.codewriter.call(opr.retref, opr.rettype, "%raw_free", + self.codewriter.call(opr.retref, opr.rettype, "@raw_free", opr.argtypes, opr.argrefs) def raw_memcopy(self, opr): - self.codewriter.call(opr.retref, opr.rettype, "%raw_memcopy", + self.codewriter.call(opr.retref, opr.rettype, "@raw_memcopy", opr.argtypes, opr.argrefs) def raw_memclear(self, opr): - self.codewriter.call(opr.retref, opr.rettype, "%raw_memclear", + self.codewriter.call(opr.retref, opr.rettype, "@raw_memclear", opr.argtypes, opr.argrefs) def raw_store(self, opr): @@ -543,7 +585,7 @@ def debug_fatalerror(self, opr): # XXX message? - self.codewriter.call(None, "void", "%abort", [], []) + self.codewriter.call(None, "void", "@abort", [], []) def hint(self, opr): self.same_as(opr) @@ -554,7 +596,7 @@ 'false', opr.rettype) def debug_llinterpcall(self, opr): - self.codewriter.call(None, "void", "%abort", [], []) + self.codewriter.call(None, "void", "@abort", [], []) # cheat llvm self.codewriter.cast(opr.retref, opr.rettype, 'null', opr.rettype) Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Wed Nov 7 20:26:51 2007 @@ -19,7 +19,7 @@ """ __slots__ = "db value structtype _get_ref_cache _get_types".split() - prefix = '%structinstance_' + prefix = '@structinstance_' def __init__(self, db, value): self.db = db @@ -63,7 +63,7 @@ break pos += 1 - return "getelementptr(%s* %s, int 0, uint %s)" %( + return "getelementptr(%s* %s, i32 0, i32 %s)" %( self.get_typerepr(), self.get_ref(), pos) @@ -115,13 +115,13 @@ ref = self.db.get_childref(p, c) if isinstance(self.value, lltype._subarray): # ptr -> array of len 1 - ref = "cast(%s* %s to %s*)" % (self.db.repr_type(self.arraytype), + ref = "bitcast(%s* %s to %s*)" % (self.db.repr_type(self.arraytype), ref, self.db.repr_type(lltype.typeOf(self.value))) return ref def get_childref(self, index): - return "getelementptr(%s* %s, int 0, int %s)" % ( + return "getelementptr(%s* %s, i32 0, i32 %s)" % ( self.get_typerepr(), self.get_ref(), index) @@ -194,7 +194,7 @@ pos += 1 assert found - ref = "getelementptr(%s* %s, int 0, uint %s)" %( + ref = "getelementptr(%s* %s, i32 0, i32 %s)" %( self.get_typerepr(), super(StructVarsizeNode, self).get_ref(), pos) @@ -204,9 +204,9 @@ def get_ref(self): ref = super(StructVarsizeNode, self).get_ref() typeval = self.db.repr_type(lltype.typeOf(self.value)) - ref = "cast(%s* %s to %s*)" % (self.get_typerepr(), - ref, - typeval) + ref = "bitcast(%s* %s to %s*)" % (self.get_typerepr(), + ref, + typeval) return ref def get_pbcref(self, toptr): @@ -215,6 +215,7 @@ p, c = lltype.parentlink(self.value) assert p is None, "child varsize struct are NOT needed by rtyper" fromptr = "%s*" % self.get_typerepr() - refptr = "getelementptr(%s %s, int 0)" % (fromptr, ref) - ref = "cast(%s %s to %s)" % (fromptr, refptr, toptr) + refptr = "getelementptr(%s %s, i32 0)" % (fromptr, ref) + ref = "bitcast(%s %s to %s)" % (fromptr, refptr, toptr) return ref + Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Nov 7 20:26:51 2007 @@ -9,7 +9,7 @@ optimize_tests = False native_llvm_backend = True -MINIMUM_LLVM_VERSION = 1.9 +MINIMUM_LLVM_VERSION = 2.0 FLOAT_PRECISION = 8 ext_modules = [] From rxe at codespeak.net Wed Nov 7 21:32:05 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 7 Nov 2007 21:32:05 +0100 (CET) Subject: [pypy-svn] r48370 - pypy/dist/pypy/translator/llvm Message-ID: <20071107203205.5EDD38081@code0.codespeak.net> Author: rxe Date: Wed Nov 7 21:32:02 2007 New Revision: 48370 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/opwriter.py Log: wip for fixing up operations on ints Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 7 21:32:02 2007 @@ -135,8 +135,7 @@ ref1, ref2)) def shiftop(self, name, targetvar, type_, ref1, ref2): - self._indent("%s = %s %s %s, ubyte %s" % (targetvar, name, type_, - ref1, ref2)) + self._indent("%s = %s %s %s, %s" % (targetvar, name, type_, ref1, ref2)) def cast(self, targetvar, fromtype, fromvar, targettype, casttype='bitcast'): if fromtype == 'void' and targettype == 'void': @@ -197,6 +196,7 @@ self._indent("%s = alloca %s" % (targetvar, vartype)) def malloc(self, targetvar, vartype, numelements=1): + XXX if numelements == 1: self._indent("%s = malloc %s" % (targetvar, vartype)) else: Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Wed Nov 7 21:32:02 2007 @@ -41,22 +41,22 @@ class OpWriter(object): shift_operations = { - # ZZZ all these + # ZZZ i guessed rshifts - ashr or lshr ?????? 'int_lshift': 'shl', - 'int_rshift': 'shr', + 'int_rshift': 'lshr', 'uint_lshift': 'shl', - 'uint_rshift': 'shr', + 'uint_rshift': 'lshr', 'llong_lshift': 'shl', - 'llong_rshift': 'shr', + 'llong_rshift': 'lshr', } binary_operations = { 'float_mul' : 'mul', 'float_add' : 'add', 'float_sub' : 'sub', - #ZZZ'float_truediv' : 'div', + 'float_truediv' : 'fdiv', 'ptr_eq' : 'icmp eq', 'ptr_ne' : 'icmp ne' } @@ -65,9 +65,13 @@ for oo in 'mul add sub and or xor'.split(): binary_operations['%s_%s' % (tt, oo)] = oo - #ZZZbinary_operations['%s_floordiv' % tt] = 'div' - #ZZZbinary_operations['%s_mod' % tt] = 'rem' + for tt in 'llong int'.split(): + binary_operations['%s_floordiv' % tt] = 'sdiv' + binary_operations['%s_mod' % tt] = 'srem' + for tt in 'ullong uint'.split(): + binary_operations['%s_floordiv' % tt] = 'udiv' + binary_operations['%s_mod' % tt] = 'urem' # comparison ops for tt in 'int llong unichar'.split(): @@ -80,7 +84,7 @@ for oo in 'eq ne'.split(): binary_operations['%s_%s' % (tt, oo)] = 'icmp %s' % oo for oo in 'lt le ge gt'.split(): - binary_operations['%s_s%s' % (tt, oo)] = 'icmp u%s' % oo + binary_operations['%s_%s' % (tt, oo)] = 'icmp u%s' % oo for tt in 'float'.split(): for oo in 'lt le eq ne ge gt'.split(): @@ -238,14 +242,14 @@ op = opr.op name = self.shift_operations[op.opname] - if isinstance(op.args[1], Constant): - var = opr.argrefs[1] - else: - var = self._tmp() - self.codewriter.cast(var, opr.argtypes[1], opr.argrefs[1], 'ubyte') + var = opr.argrefs[1] + #if isinstance(op.args[1], Constant): + # var = opr.argrefs[1] + #else: + # var = self._tmp() + # self.codewriter.cast(var, opr.argtypes[1], opr.argrefs[1], 'i8') - self.codewriter.shiftop(name, opr.retref, opr.argtypes[0], - opr.argrefs[0], var) + self.codewriter.shiftop(name, opr.retref, opr.argtypes[0], opr.argrefs[0], var) def cast_char_to_int(self, opr): " works for all casts " From cfbolz at codespeak.net Wed Nov 7 22:05:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 22:05:25 +0100 (CET) Subject: [pypy-svn] r48371 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071107210525.CF71D8103@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 22:05:25 2007 New Revision: 48371 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Log: some help for amit to install the import hook Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Wed Nov 7 22:05:25 2007 @@ -6,10 +6,18 @@ class Module(MixedModule): """CLR module""" - appleveldefs = {} + appleveldefs = { + #'dotnetimporter': 'app_importer.importer' + } interpleveldefs = { '_CliObject_internal': 'interp_clr.W_CliObject', 'call_staticmethod': 'interp_clr.call_staticmethod', 'load_cli_class': 'interp_clr.load_cli_class', } + + def setup_after_space_initialization(self): + self.space.appexec([self], """(clr_module): + import sys + #sys.meta_path.append(clr_module.dotnetimporter()) + """) From fijal at codespeak.net Wed Nov 7 22:05:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 22:05:57 +0100 (CET) Subject: [pypy-svn] r48372 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem ootypesystem test Message-ID: <20071107210557.868CD8144@code0.codespeak.net> Author: fijal Date: Wed Nov 7 22:05:57 2007 New Revision: 48372 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Support for most methods which I think should be supported for unicode (right now). Still roughly 12 tests missing. Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 22:05:57 2007 @@ -94,7 +94,7 @@ class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char -class UniCharRepr(AbstractUniCharRepr): +class UniCharRepr(AbstractUniCharRepr, UnicodeRepr): lowleveltype = UniChar class __extend__(pairtype(PyObjRepr, AbstractStringRepr)): @@ -241,7 +241,10 @@ def ll_strip(s, ch, left, right): s_len = len(s.chars) if s_len == 0: - return emptystr + if typeOf(s).TO == STR: + return emptystr + else: + return emptyunicode lpos = 0 rpos = s_len - 1 if left: @@ -251,7 +254,11 @@ while lpos < rpos and s.chars[rpos] == ch: rpos -= 1 r_len = rpos - lpos + 1 - result = mallocstr(r_len) + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(r_len) i = 0 j = lpos while i < r_len: @@ -264,9 +271,16 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - return emptystr + if typeOf(s).TO == STR: + return emptystr + else: + return emptyunicode i = 0 - result = mallocstr(s_len) + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(s_len) while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': @@ -279,9 +293,16 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - return emptystr + if typeOf(s).TO == STR: + return emptystr + else: + return emptyunicode i = 0 - result = mallocstr(s_len) + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(s_len) while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': @@ -295,13 +316,20 @@ s_len = len(s_chars) num_items = length if num_items == 0: - return emptystr + if typeOf(s).TO == STR: + return emptystr + else: + return emptyunicode itemslen = 0 i = 0 while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen + s_len * (num_items - 1)) + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(itemslen + s_len * (num_items - 1)) res_chars = result.chars res_index = 0 i = 0 @@ -551,7 +579,11 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen) + if typeOf(items).TO.OF.TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(itemslen) res_chars = result.chars res_index = 0 i = 0 @@ -568,7 +600,11 @@ def ll_join_chars(length, chars): num_chars = length - result = mallocstr(num_chars) + if typeOf(chars).TO.OF == Char: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(num_chars) res_chars = result.chars i = 0 while i < num_chars: @@ -578,7 +614,11 @@ def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) - newstr = mallocstr(len1 - start) + if typeOf(s1).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + newstr = malloc(len1 - start) j = 0 while start < len1: newstr.chars[j] = s1.chars[start] @@ -593,7 +633,11 @@ if start == 0: return s1 stop = len(s1.chars) - newstr = mallocstr(stop - start) + if typeOf(s1).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + newstr = malloc(stop - start) j = 0 while start < stop: newstr.chars[j] = s1.chars[start] @@ -603,7 +647,11 @@ def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 - newstr = mallocstr(newlen) + if typeOf(s1).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + newstr = malloc(newlen) j = 0 while j < newlen: newstr.chars[j] = s1.chars[j] @@ -624,9 +672,13 @@ i = 0 j = 0 resindex = 0 + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode while j < strlen: if chars[j] == c: - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = malloc(j - i) newchars = item.chars k = i while k < j: @@ -635,7 +687,7 @@ resindex += 1 i = j + 1 j += 1 - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = malloc(j - i) newchars = item.chars k = i while k < j: @@ -647,7 +699,11 @@ def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) - newstr = mallocstr(length) + if typeOf(s).TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + newstr = malloc(length) src = s.chars dst = newstr.chars j = 0 @@ -804,6 +860,7 @@ unichar_repr.ll = LLHelpers unicode_repr = UnicodeRepr() emptystr = string_repr.convert_const("") +emptyunicode = unicode_repr.convert_const(u'') StringRepr.repr = string_repr UnicodeRepr.repr = unicode_repr Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Wed Nov 7 22:05:57 2007 @@ -301,6 +301,7 @@ unichar_repr.ll = LLHelpers emptystr = string_repr.convert_const("") StringRepr.repr = string_repr +StringRepr.char_repr = char_repr class StringIteratorRepr(AbstractStringIteratorRepr): lowleveltype = ootype.Record({'string': string_repr.lowleveltype, Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Wed Nov 7 22:05:57 2007 @@ -20,7 +20,12 @@ pass class AbstractUnicodeRepr(AbstractStringRepr): - pass + def rtype_method_upper(self, hop): + raise TypeError("Cannot do toupper on unicode string") + + def rtype_method_lower(self, hop): + raise TypeError("Cannot do toupper on unicode string") + class __extend__(annmodel.SomeString): def rtyper_makerepr(self, rtyper): @@ -102,13 +107,14 @@ def rtype_method_find(self, hop, reverse=False): # XXX binaryop - rstr = hop.rtyper.type_system.rstr - v_str = hop.inputarg(rstr.string_repr, arg=0) - if hop.args_r[1] == rstr.char_repr: - v_value = hop.inputarg(rstr.char_repr, arg=1) + string_repr = hop.args_r[0].repr + char_repr = hop.args_r[0].char_repr + v_str = hop.inputarg(string_repr, arg=0) + if hop.args_r[1] == char_repr: + v_value = hop.inputarg(char_repr, arg=1) llfn = reverse and self.ll.ll_rfind_char or self.ll.ll_find_char else: - v_value = hop.inputarg(rstr.string_repr, arg=1) + v_value = hop.inputarg(string_repr, arg=1) llfn = reverse and self.ll.ll_rfind or self.ll.ll_find if hop.nb_args > 2: v_start = hop.inputarg(Signed, arg=2) @@ -129,13 +135,13 @@ return self.rtype_method_find(hop, reverse=True) def rtype_method_count(self, hop): - rstr = hop.rtyper.type_system.rstr - v_str = hop.inputarg(rstr.string_repr, arg=0) + rstr = hop.args_r[0].repr + v_str = hop.inputarg(rstr.repr, arg=0) if hop.args_r[1] == rstr.char_repr: v_value = hop.inputarg(rstr.char_repr, arg=1) llfn = self.ll.ll_count_char else: - v_value = hop.inputarg(rstr.string_repr, arg=1) + v_value = hop.inputarg(rstr.repr, arg=1) llfn = self.ll.ll_count if hop.nb_args > 2: v_start = hop.inputarg(Signed, arg=2) @@ -153,8 +159,8 @@ return hop.gendirectcall(llfn, v_str, v_value, v_start, v_end) def rtype_method_strip(self, hop, left=True, right=True): - rstr = hop.rtyper.type_system.rstr - v_str = hop.inputarg(rstr.string_repr, arg=0) + rstr = hop.args_r[0].repr + v_str = hop.inputarg(rstr.repr, arg=0) v_char = hop.inputarg(rstr.char_repr, arg=1) v_left = hop.inputconst(Bool, left) v_right = hop.inputconst(Bool, right) @@ -167,13 +173,13 @@ return self.rtype_method_strip(hop, left=False, right=True) def rtype_method_upper(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_upper, v_str) def rtype_method_lower(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_lower, v_str) @@ -185,17 +191,17 @@ def rtype_method_join(self, hop): hop.exception_cannot_occur() - rstr = hop.rtyper.type_system.rstr + rstr = hop.args_r[0] if hop.s_result.is_constant(): - return inputconst(rstr.string_repr, hop.s_result.const) + return inputconst(rstr.repr, hop.s_result.const) r_lst = hop.args_r[1] if not isinstance(r_lst, hop.rtyper.type_system.rlist.BaseListRepr): raise TyperError("string.join of non-list: %r" % r_lst) - v_str, v_lst = hop.inputargs(rstr.string_repr, r_lst) + v_str, v_lst = hop.inputargs(rstr.repr, r_lst) v_length, v_items = self._list_length_items(hop, v_lst, r_lst.lowleveltype) if hop.args_s[0].is_constant() and hop.args_s[0].const == '': - if r_lst.item_repr == rstr.string_repr: + if r_lst.item_repr == rstr.repr: llfn = self.ll.ll_join_strs elif r_lst.item_repr == rstr.char_repr: llfn = self.ll.ll_join_chars @@ -203,15 +209,15 @@ raise TyperError("''.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_length, v_items) else: - if r_lst.item_repr == rstr.string_repr: + if r_lst.item_repr == rstr.repr: llfn = self.ll.ll_join else: raise TyperError("sep.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_str, v_length, v_items) def rtype_method_split(self, hop): - rstr = hop.rtyper.type_system.rstr - v_str, v_chr = hop.inputargs(rstr.string_repr, rstr.char_repr) + rstr = hop.args_r[0].repr + v_str, v_chr = hop.inputargs(rstr.repr, rstr.char_repr) try: list_type = hop.r_result.lowleveltype.TO except AttributeError: @@ -221,16 +227,16 @@ return hop.gendirectcall(self.ll.ll_split_chr, cLIST, v_str, v_chr) def rtype_method_replace(self, hop): - rstr = hop.rtyper.type_system.rstr + rstr = hop.args_r[0].repr if not (hop.args_r[1] == rstr.char_repr and hop.args_r[2] == rstr.char_repr): raise TyperError, 'replace only works for char args' - v_str, v_c1, v_c2 = hop.inputargs(rstr.string_repr, rstr.char_repr, rstr.char_repr) + v_str, v_c1, v_c2 = hop.inputargs(rstr.repr, rstr.char_repr, rstr.char_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_replace_chr_chr, v_str, v_c1, v_c2) def rtype_int(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr if hop.nb_args == 1: v_str, = hop.inputargs(string_repr) c_base = inputconst(Signed, 10) @@ -250,7 +256,7 @@ def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_is_here() return hop.gendirectcall(self.ll.ll_float, v_str) @@ -297,17 +303,17 @@ class __extend__(pairtype(AbstractStringRepr, AbstractSliceRepr)): def rtype_getitem((r_str, r_slic), hop): - rstr = hop.rtyper.type_system.rstr + string_repr = r_str.repr rslice = hop.rtyper.type_system.rslice if r_slic == rslice.startonly_slice_repr: - v_str, v_start = hop.inputargs(rstr.string_repr, rslice.startonly_slice_repr) + v_str, v_start = hop.inputargs(string_repr, rslice.startonly_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice_startonly, v_str, v_start) if r_slic == rslice.startstop_slice_repr: - v_str, v_slice = hop.inputargs(rstr.string_repr, rslice.startstop_slice_repr) + v_str, v_slice = hop.inputargs(string_repr, rslice.startstop_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice, v_str, v_slice) if r_slic == rslice.minusone_slice_repr: - v_str, v_ignored = hop.inputargs(rstr.string_repr, rslice.minusone_slice_repr) + v_str, v_ignored = hop.inputargs(string_repr, rslice.minusone_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice_minusone, v_str) raise TyperError(r_slic) @@ -354,7 +360,8 @@ return hop.genop('int_gt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) -class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr)): +class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr), + pairtype(AbstractUnicodeRepr, AbstractUniCharRepr)): def rtype_contains((r_str, r_chr), hop): string_repr = r_str.repr char_repr = r_chr.char_repr @@ -429,7 +436,6 @@ def rtype_mul((r_chr, r_int), hop): char_repr = r_chr.char_repr - rstr = hop.rtyper.type_system.rstr v_char, v_int = hop.inputargs(char_repr, Signed) return hop.gendirectcall(r_chr.ll.ll_char_mul, v_char, v_int) rtype_inplace_mul = rtype_mul Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 22:05:57 2007 @@ -191,16 +191,19 @@ assert self.interpret(fn, [u'X']) == True def test_is_none(self): + const = self.const def fn(i): - s1 = ['foo', None][i] + s1 = [const('foo'), None][i] return s1 is None assert self.interpret(fn, [0]) == False assert self.interpret(fn, [1]) == True def test_str_compare(self): + const = self.const def fn(i, j): - s1 = ['one', 'two', None] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar', None] + s1 = [const('one'), const('two'), None] + s2 = [const('one'), const('two'), const('o'), + const('on'), const('twos'), const('foobar'), None] return s1[i] == s2[j] for i in range(3): for j in range(7): @@ -208,8 +211,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] != s2[j] for i in range(2): for j in range(6): @@ -217,8 +220,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] < s2[j] for i in range(2): for j in range(6): @@ -226,8 +229,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] <= s2[j] for i in range(2): for j in range(6): @@ -235,8 +238,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] >= s2[j] for i in range(2): for j in range(6): @@ -244,8 +247,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] > s2[j] for i in range(2): for j in range(6): @@ -253,9 +256,10 @@ assert res is fn(i, j) def test_startswith(self): + const = self.const def fn(i, j): - s1 = ['', 'one', 'two'] - s2 = ['', 'one', 'two', 'o', 'on', 'ne', 'e', 'twos', 'foobar', 'fortytwo'] + s1 = [const(''), const('one'), const('two')] + s2 = [const(''), const('one'), const('two'), const('o'), const('on'), const('ne'), const('e'), const('twos'), const('foobar'), const('fortytwo')] return s1[i].startswith(s2[j]) for i in range(3): for j in range(10): @@ -263,9 +267,10 @@ assert res is fn(i, j) def test_endswith(self): + const = self.const def fn(i, j): - s1 = ['', 'one', 'two'] - s2 = ['', 'one', 'two', 'o', 'on', 'ne', 'e', 'twos', 'foobar', 'fortytwo'] + s1 = [const(''), const('one'), const('two')] + s2 = [const(''), const('one'), const('two'), const('o'), const('on'), const('ne'), const('e'), const('twos'), const('foobar'), const('fortytwo')] return s1[i].endswith(s2[j]) for i in range(3): for j in range(10): @@ -273,9 +278,10 @@ assert res is fn(i, j) def test_find(self): + const = self.const def fn(i, j): - s1 = ['one two three', 'abc abcdab abcdabcdabde'] - s2 = ['one', 'two', 'abcdab', 'one tou', 'abcdefgh', 'fortytwo', ''] + s1 = [const('one two three'), const('abc abcdab abcdabcdabde')] + s2 = [const('one'), const('two'), const('abcdab'), const('one tou'), const('abcdefgh'), const('fortytwo'), const('')] return s1[i].find(s2[j]) for i in range(2): for j in range(7): @@ -283,80 +289,89 @@ assert res == fn(i, j) def test_find_with_start(self): + const = self.const def fn(i): assert i >= 0 - return 'ababcabc'.find('abc', i) + return const('ababcabc').find(const('abc'), i) for i in range(9): res = self.interpret(fn, [i]) assert res == fn(i) def test_find_with_start_end(self): + const = self.const def fn(i, j): assert i >= 0 assert j >= 0 - return 'ababcabc'.find('abc', i, j) + return const('ababcabc').find(const('abc'), i, j) for (i, j) in [(1,7), (2,6), (3,7), (3,8)]: res = self.interpret(fn, [i, j]) assert res == fn(i, j) def test_find_empty_string(self): + const = self.const def f(i): assert i >= 0 - s = "abc" - x = s.find('') - x+= s.find('', i)*10 - x+= s.find('', i, i)*100 - x+= s.find('', i, i+1)*1000 + s = const("abc") + x = s.find(const('')) + x+= s.find(const(''), i)*10 + x+= s.find(const(''), i, i)*100 + x+= s.find(const(''), i, i+1)*1000 return x for i in range(5): res = self.interpret(f, [i]) assert res == f(i) def test_rfind(self): + const = self.const def fn(): - return 'aaa'.rfind('a') + 'aaa'.rfind('a', 1) + 'aaa'.rfind('a', 1, 2) + return const('aaa').rfind(const('a')) + const('aaa').rfind(const('a'), 1) + const('aaa').rfind(const('a'), 1, 2) res = self.interpret(fn, []) assert res == 2 + 2 + 1 def test_rfind_empty_string(self): + const = self.const def f(i): assert i >= 0 - s = "abc" - x = s.find('') - x+= s.find('', i)*10 - x+= s.find('', i, i)*100 - x+= s.find('', i, i+1)*1000 + s = const("abc") + x = s.find(const('')) + x+= s.find(const(''), i)*10 + x+= s.find(const(''), i, i)*100 + x+= s.find(const(''), i, i+1)*1000 return x for i in range(5): res = self.interpret(f, [i]) assert res == f(i) def test_find_char(self): + const = self.const def fn(ch): - pos1 = 'aiuwraz 483'.find(ch) - pos2 = 'aiuwraz 483'.rfind(ch) + pos1 = const('aiuwraz 483').find(ch) + pos2 = const('aiuwraz 483').rfind(ch) return pos1 + (pos2*100) - for ch in 'a ?3': + for ch in const('a ?3'): res = self.interpret(fn, [ch]) assert res == fn(ch) def test_strip(self): + const = self.const def both(): - return '!ab!'.strip('!') + return const('!ab!').strip(const('!')) def left(): - return '!ab!'.lstrip('!') + return const('!ab!').lstrip(const('!')) def right(): - return '!ab!'.rstrip('!') + return const('!ab!').rstrip(const('!')) res = self.interpret(both, []) - assert self.ll_to_string(res) == 'ab' + assert self.ll_to_string(res) == const('ab') res = self.interpret(left, []) - assert self.ll_to_string(res) == 'ab!' + assert self.ll_to_string(res) == const('ab!') res = self.interpret(right, []) - assert self.ll_to_string(res) == '!ab' + assert self.ll_to_string(res) == const('!ab') def test_upper(self): - strings = ['', ' ', 'upper', 'UpPeR', ',uppEr,'] - for i in range(256): strings.append(chr(i)) + const = self.const + constchar = self.constchar + strings = [const(''), const(' '), const('upper'), const('UpPeR'), const(',uppEr,')] + for i in range(256): strings.append(constchar(i)) def fn(i): return strings[i].upper() for i in range(len(strings)): @@ -364,7 +379,8 @@ assert self.ll_to_string(res) == fn(i) def test_lower(self): - strings = ['', ' ', 'lower', 'LoWeR', ',lowEr,'] + const = self.const + strings = [const(''), const(' '), const('lower'), const('LoWeR'), const(',lowEr,')] for i in range(256): strings.append(chr(i)) def fn(i): return strings[i].lower() @@ -373,21 +389,22 @@ assert self.ll_to_string(res) == fn(i) def test_join(self): - res = self.interpret(lambda: ''.join([]), []) + const = self.const + res = self.interpret(lambda: const('').join([]), []) assert self.ll_to_string(res) == "" - res = self.interpret(lambda: ''.join(['a', 'b', 'c']), []) + res = self.interpret(lambda: const('').join([const('a'), const('b'), const('c')]), []) assert self.ll_to_string(res) == "abc" - res = self.interpret(lambda: ''.join(['abc', 'de', 'fghi']), []) + res = self.interpret(lambda: const('').join([const('abc'), const('de'), const('fghi')]), []) assert self.ll_to_string(res) == "abcdefghi" - res = self.interpret(lambda: '.'.join(['abc', 'def']), []) - assert self.ll_to_string(res) == 'abc.def' + res = self.interpret(lambda: const('.').join([const('abc'), const('def')]), []) + assert self.ll_to_string(res) == const('abc.def') def fn(i, j): - s1 = [ '', ',', ' and '] - s2 = [ [], ['foo'], ['bar', 'baz', 'bazz']] + s1 = [ const(''), const(','), const(' and ')] + s2 = [ [], [const('foo')], [const('bar'), const('baz'), const('bazz')]] return s1[i].join(s2[j]) for i in range(3): for j in range(3): @@ -395,9 +412,9 @@ assert self.ll_to_string(res) == fn(i, j) def fn(i, j): - s1 = [ '', ',', ' and '] - s2 = [ [], ['foo'], ['bar', 'baz', 'bazz']] - s2[1].extend(['x']) + s1 = [ const(''), const(','), const(' and ')] + s2 = [ [], [const('foo')], [const('bar'), const('baz'), const('bazz')]] + s2[1].extend([const('x')]) return s1[i].join(s2[j]) for i in range(3): for j in range(3): @@ -405,73 +422,77 @@ assert self.ll_to_string(res) == fn(i, j) def test_str_slice(self): + const = self.const def fn(): - s = 'hello' + s = const('hello') s1 = s[:3] s2 = s[3:] s3 = s[3:10] - return s1+s2 == s and s2+s1 == 'lohel' and s1+s3 == s + return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s res = self.interpret(fn, ()) assert res def test_str_slice_minusone(self): + const = self.const def fn(): - s = 'hello' - z = 'h' + s = const('hello') + z = const('h') return s[:-1]+z[:-1] res = self.interpret(fn, ()) - assert self.ll_to_string(res) == 'hell' + assert self.ll_to_string(res) == const('hell') def test_strformat(self): + const = self.const def percentS(s): - return "before %s after" % (s,) + return const("before %s after") % (s,) - res = self.interpret(percentS, ['1']) - assert self.ll_to_string(res) == 'before 1 after' + res = self.interpret(percentS, [const('1')]) + assert self.ll_to_string(res) == const('before 1 after') def percentD(i): return "bing %d bang" % (i,) res = self.interpret(percentD, [23]) - assert self.ll_to_string(res) == 'bing 23 bang' + assert self.ll_to_string(res) == const('bing 23 bang') def percentX(i): - return "bing %x bang" % (i,) + return const("bing %x bang") % (i,) res = self.interpret(percentX, [23]) - assert self.ll_to_string(res) == 'bing 17 bang' + assert self.ll_to_string(res) == const('bing 17 bang') res = self.interpret(percentX, [-123]) - assert self.ll_to_string(res) == 'bing -7b bang' + assert self.ll_to_string(res) == const('bing -7b bang') def percentO(i): - return "bing %o bang" % (i,) + return const("bing %o bang") % (i,) res = self.interpret(percentO, [23]) - assert self.ll_to_string(res) == 'bing 27 bang' + assert self.ll_to_string(res) == const('bing 27 bang') res = self.interpret(percentO, [-123]) - assert self.ll_to_string(res) == 'bing -173 bang' + assert self.ll_to_string(res) == const('bing -173 bang') def moreThanOne(s, d, x, o): - return "string: %s decimal: %d hex: %x oct: %o" % (s, d, x, o) + return const("string: %s decimal: %d hex: %x oct: %o") % (s, d, x, o) - args = 'a', 2, 3, 4 + args = const('a'), 2, 3, 4 res = self.interpret(moreThanOne, list(args)) assert self.ll_to_string(res) == moreThanOne(*args) def test_strformat_nontuple(self): + const = self.const def percentD(i): - return "before %d after" % i + return const("before %d after") % i res = self.interpret(percentD, [1]) - assert self.ll_to_string(res) == 'before 1 after' + assert self.ll_to_string(res) == const('before 1 after') def percentS(i): - return "before %s after" % i + return const("before %s after") % i - res = self.interpret(percentS, ['D']) - assert self.ll_to_string(res) == 'before D after' + res = self.interpret(percentS, [const('D')]) + assert self.ll_to_string(res) == const('before D after') def test_strformat_instance(self): class C: @@ -547,52 +568,58 @@ assert res == expected def test_split(self): + const = self.const def fn(i): - s = ['', '0.1.2.4.8', '.1.2', '1.2.', '.1.2.4.'][i] - l = s.split('.') + s = [const(''), const('0.1.2.4.8'), const('.1.2'), const('1.2.'), const('.1.2.4.')][i] + l = s.split(const('.')) sum = 0 for num in l: if len(num): - sum += ord(num) - ord('0') + sum += ord(num) - ord(const('0')) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) assert res == fn(i) def test_contains(self): + const = self.const + constchar = self.constchar def fn(i): - s = 'Hello world' - return chr(i) in s + s = const('Hello world') + return constchar(i) in s for i in range(256): res = self.interpret(fn, [i])#, view=i==42) assert res == fn(i) def test_replace(self): + const = self.const def fn(c1, c2): - s = 'abbccc' + s = const('abbccc') s = s.replace(c1, c2) res = 0 for c in s: if c == c2: res += 1 return res - res = self.interpret(fn, ['a', 'c']) + res = self.interpret(fn, [const('a'), const('c')]) assert res == 4 - res = self.interpret(fn, ['c', 'b']) + res = self.interpret(fn, [const('c'), const('b')]) assert res == 5 def test_replace_TyperError(self): + const = self.const def fn(): - s = 'abbccc' - s = s.replace('a', 'baz') + s = const('abbccc') + s = s.replace(const('a'), const('baz')) raises(TyperError, self.interpret, fn, ()) def fn(): - s = 'abbccc' - s = s.replace('abb', 'c') + s = const('abbccc') + s = s.replace(const('abb'), const('c')) raises(TyperError, self.interpret, fn, ()) def test_int(self): - s1 = [ '42', '01001', 'abc', 'ABC', '4aBc', ' 12ef ', '+42', 'foo', '42foo', '42.1', '', '+ 42'] + const = self.const + s1 = [ const('42'), const('01001'), const('abc'), const('ABC'), const('4aBc'), const(' 12ef '), const('+42'), const('foo'), const('42foo'), const('42.1'), const(''), const('+ 42')] def fn(i, base): s = s1[i] res = int(s, base) @@ -608,7 +635,8 @@ assert res == expected def test_int_valueerror(self): - s1 = ['42g', '?', '+', '+ '] + const = self.const + s1 = [const('42g'), const('?'), const('+'), const('+ ')] def fn(i): try: return int(s1[i]) @@ -622,7 +650,8 @@ assert res == -654 def test_float(self): - f = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] + const = self.const + f = [const(''), const(' '), const('0'), const('1'), const('-1.5'), CONST('1.5E2'), const('2.5e-1'), const(' 0 '), const('?')] def fn(i): s = f[i] return float(s) Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Wed Nov 7 22:05:57 2007 @@ -40,8 +40,23 @@ const = unicode assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) - def test_char_isxxx(self): + def unsupported(self): py.test.skip("not supported") + test_char_isxxx = unsupported + test_upper = unsupported + test_lower = unsupported + test_strformat = unsupported + test_strformat_instance = unsupported + test_strformat_nontuple = unsupported + test_percentformat_instance = unsupported + test_percentformat_tuple = unsupported + test_percentformat_list = unsupported + test_int = unsupported + test_int_valueerror = unsupported + test_float = unsupported + class TestLLtype(BaseTestRUnicode, LLRtypeMixin): - pass + EMPTY_STRING_HASH = -1 + ll_to_string = LLRtypeMixin.ll_to_unicode + From regmee at codespeak.net Wed Nov 7 22:16:23 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Wed, 7 Nov 2007 22:16:23 +0100 (CET) Subject: [pypy-svn] r48373 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071107211623.ECB35817E@code0.codespeak.net> Author: regmee Date: Wed Nov 7 22:16:22 2007 New Revision: 48373 Added: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Log: importer hook implementation Added: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py ============================================================================== --- (empty file) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Wed Nov 7 22:16:22 2007 @@ -0,0 +1,61 @@ + +""" Importer class + # Meta hooks are called at the start of Import Processing + # Meta hooks can override the sys.path, frozen modules , built-in modules + # to register a Meta Hook simply add importer object to sys.meta_path + # a path hook is registered by adding an Importer factory to sys.path_hooks + # sys.path_hooks is a list of Class of the HOOK. + # whose __init__ is called when the calleable in the list is obtained. + # __init__ cant return anything so some __new__ method should return + This is used to enable the "import module" mechanism for .NET classes""" + +import imp +import sys + +class loader(object): + def __init__(self): + self.Names = [] + + def load_module(self, fullname): + try: + return sys.modules[fullname] + except KeyError: + pass + # Now create a new module and append it at the end of the sys.modules list + mod = imp.new_module(fullname) + mod.__file__ = "<%s>" % self.__class__.__name__ + mod.__loader__ = self + mod.__name__ = fullname + '''#if ispkg: + if : + mod.__path__ = [] + exec code in mod.__dict__''' + + # add it to the modules list + sys.modules[fullname] = mod + + return mod + +class importer(object): + + def __init__(self): + self.loader = loader() + + def find_module(self, fullname, path): + # path will be None for top-level Module and __path__ for sub-modules + print fullname + if path != None: + __path__ = path + try: + return sys.modules[fullname] + except KeyError: + pass + + # Now since the module was not found .. Call the Loader and load it. + try: + return self.loader + except ImportError: + print "Import Error exception raised hence you better quit" + return None + +#def load_cli_class(space, namespace, classname): From fijal at codespeak.net Wed Nov 7 23:19:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Nov 2007 23:19:34 +0100 (CET) Subject: [pypy-svn] r48374 - in pypy/branch/pypy-rpython-unicode/rpython: lltypesystem test Message-ID: <20071107221934.652F68187@code0.codespeak.net> Author: fijal Date: Wed Nov 7 23:19:32 2007 New Revision: 48374 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Log: * fix test * a bit of code simplification Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Wed Nov 7 23:19:32 2007 @@ -12,7 +12,8 @@ from pypy.rpython.lltypesystem import ll_str from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, UniChar, Ptr, malloc, \ - Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive, typeOf + Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive, typeOf,\ + staticAdtMethod, GcForwardReference from pypy.rpython.rmodel import Repr # ____________________________________________________________ @@ -24,11 +25,30 @@ # chars: array of Char # } -STR = GcStruct('rpy_string', ('hash', Signed), - ('chars', Array(Char, hints={'immutable': True, - 'isrpystring': True}))) -UNICODE = GcStruct('rpy_unicode', ('hash', Signed), - ('chars', Array(UniChar, hints={'immutable': True}))) +STR = GcForwardReference() +UNICODE = GcForwardReference() + +def new_malloc(TP): + def mallocstr(length): + debug_assert(length >= 0, "negative string length") + r = malloc(TP, length) + if not we_are_translated() or not malloc_zero_filled: + r.hash = 0 + return r + mallocstr._annspecialcase_ = 'specialize:semierased' + return mallocstr + +mallocstr = new_malloc(STR) +mallocunicode = new_malloc(UNICODE) + +STR.become(GcStruct('rpy_string', ('hash', Signed), + ('chars', Array(Char, hints={'immutable': True, + 'isrpystring': True})), + adtmeths={'malloc':staticAdtMethod(mallocstr)})) +UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed), + ('chars', Array(UniChar, hints={'immutable': True})), + adtmeths={'malloc':staticAdtMethod(mallocunicode)} + )) SIGNED_ARRAY = GcArray(Signed) CONST_STR_CACHE = WeakValueDictionary() CONST_UNICODE_CACHE = WeakValueDictionary() @@ -121,19 +141,6 @@ resulttype=pyobj_repr, _callable= lambda v: pyobjectptr(''.join(v.chars))) -def new_malloc(TP): - def mallocstr(length): - debug_assert(length >= 0, "negative string length") - r = malloc(TP, length) - if not we_are_translated() or not malloc_zero_filled: - r.hash = 0 - return r - mallocstr._annspecialcase_ = 'specialize:semierased' - return mallocstr - -mallocstr = new_malloc(STR) -mallocunicode = new_malloc(UNICODE) - # ____________________________________________________________ # # Low-level methods. These can be run for testing, but are meant to @@ -221,12 +228,7 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - if typeOf(s1).TO == STR and typeOf(s2).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - - newstr = malloc(len1 + len2) + newstr = s1.malloc(len1 + len2) j = 0 while j < len1: newstr.chars[j] = s1.chars[j] Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Wed Nov 7 23:19:32 2007 @@ -651,7 +651,7 @@ def test_float(self): const = self.const - f = [const(''), const(' '), const('0'), const('1'), const('-1.5'), CONST('1.5E2'), const('2.5e-1'), const(' 0 '), const('?')] + f = [const(''), const(' '), const('0'), const('1'), const('-1.5'), const('1.5E2'), const('2.5e-1'), const(' 0 '), const('?')] def fn(i): s = f[i] return float(s) From cfbolz at codespeak.net Wed Nov 7 23:46:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Nov 2007 23:46:44 +0100 (CET) Subject: [pypy-svn] r48375 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107224644.610DF8115@code0.codespeak.net> Author: cfbolz Date: Wed Nov 7 23:46:42 2007 New Revision: 48375 Added: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (contents, props changed) Log: again a bit of code to help amit Added: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- (empty file) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Wed Nov 7 23:46:42 2007 @@ -0,0 +1,11 @@ +from pypy.conftest import gettestobjspace + +class AppTestDotnet: + def setup_class(cls): + space = gettestobjspace(usemodules=('clr',)) + cls.space = space + + def test_import_hook_simple(self): + import clr + # import System.Math ... + From cfbolz at codespeak.net Thu Nov 8 00:03:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 00:03:44 +0100 (CET) Subject: [pypy-svn] r48377 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107230344.8B8298154@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 00:03:43 2007 New Revision: 48377 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: we need zipimport too, for stupid reasons Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 00:03:43 2007 @@ -2,7 +2,9 @@ class AppTestDotnet: def setup_class(cls): - space = gettestobjspace(usemodules=('clr',)) + # XXX the zipimport below is nonsense, of course, but leaving + # it out crashes + space = gettestobjspace(usemodules=('clr', 'zipimport')) cls.space = space def test_import_hook_simple(self): From regmee at codespeak.net Thu Nov 8 00:10:47 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 8 Nov 2007 00:10:47 +0100 (CET) Subject: [pypy-svn] r48378 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107231047.7B9498154@code0.codespeak.net> Author: regmee Date: Thu Nov 8 00:10:47 2007 New Revision: 48378 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: test for import System.Math Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 00:10:47 2007 @@ -9,5 +9,6 @@ def test_import_hook_simple(self): import clr + import System.Math # import System.Math ... From regmee at codespeak.net Thu Nov 8 00:13:08 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 8 Nov 2007 00:13:08 +0100 (CET) Subject: [pypy-svn] r48379 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071107231308.246788156@code0.codespeak.net> Author: regmee Date: Thu Nov 8 00:13:07 2007 New Revision: 48379 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: successful import for System.Math Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Thu Nov 8 00:13:07 2007 @@ -7,7 +7,7 @@ """CLR module""" appleveldefs = { - #'dotnetimporter': 'app_importer.importer' + 'dotnetimporter': 'app_importer.importer' } interpleveldefs = { @@ -19,5 +19,5 @@ def setup_after_space_initialization(self): self.space.appexec([self], """(clr_module): import sys - #sys.meta_path.append(clr_module.dotnetimporter()) + sys.meta_path.append(clr_module.dotnetimporter()) """) Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Thu Nov 8 00:13:07 2007 @@ -17,19 +17,25 @@ self.Names = [] def load_module(self, fullname): - try: - return sys.modules[fullname] - except KeyError: - pass + + # Now since the module was not found .. Call the Loader and load it. + if fullname == "System.Math": + import clr + return clr.load_cli_class('System','Math') + + if fullname == "System.Collections.ArrayList": + import clr + return clr.load_cli_class('System.Collections','ArrayList') + # Now create a new module and append it at the end of the sys.modules list mod = imp.new_module(fullname) mod.__file__ = "<%s>" % self.__class__.__name__ mod.__loader__ = self mod.__name__ = fullname - '''#if ispkg: - if : - mod.__path__ = [] - exec code in mod.__dict__''' + #if ispkg: + #if : + # mod.__path__ = [] + #exec code in mod.__dict__''' # add it to the modules list sys.modules[fullname] = mod @@ -50,8 +56,7 @@ return sys.modules[fullname] except KeyError: pass - - # Now since the module was not found .. Call the Loader and load it. + try: return self.loader except ImportError: Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Thu Nov 8 00:13:07 2007 @@ -188,6 +188,9 @@ - classname: the name of the class in the specified namespace (e.g. ``ArrayList``). """ + #import sys + #for module in sys.modules: + # print "mod ----> %s"%module fullname = '%s.%s' % (namespace, classname) w_cls = CliClassCache.get(fullname) if w_cls is None: From regmee at codespeak.net Thu Nov 8 00:26:06 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 8 Nov 2007 00:26:06 +0100 (CET) Subject: [pypy-svn] r48380 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107232606.07CC58166@code0.codespeak.net> Author: regmee Date: Thu Nov 8 00:26:05 2007 New Revision: 48380 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: few more lines to test System.Math Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 00:26:05 2007 @@ -10,5 +10,9 @@ def test_import_hook_simple(self): import clr import System.Math + + print " abs(-5) = %s"%System.Math.Abs(-5) + print " 2^5 = %s"%System.Math.Pow(2,5) + # import System.Math ... From cfbolz at codespeak.net Thu Nov 8 00:35:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 00:35:17 +0100 (CET) Subject: [pypy-svn] r48381 - pypy/branch/clr-module-improvements/pypy/module/__builtin__ Message-ID: <20071107233517.B2C818167@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 00:35:15 2007 New Revision: 48381 Modified: pypy/branch/clr-module-improvements/pypy/module/__builtin__/__init__.py Log: only install the zipimport hook when the zipimport module is requested Modified: pypy/branch/clr-module-improvements/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/__builtin__/__init__.py Thu Nov 8 00:35:15 2007 @@ -168,15 +168,11 @@ del self.loaders[name] def startup(self, space): - # install zipimport hook - try: + # install zipimport hook if --withmod-zipimport is used + if space.config.objspace.usemodules.zipimport: w_import = space.builtin.get('__import__') w_zipimport = space.call(w_import, space.newlist( [space.wrap('zipimport')])) - except OperationError, e: - if not e.match(space, space.w_ImportError): - raise - else: w_sys = space.getbuiltinmodule('sys') w_path_hooks = space.getattr(w_sys, space.wrap('path_hooks')) w_append = space.getattr(w_path_hooks, space.wrap('append')) From cfbolz at codespeak.net Thu Nov 8 00:36:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 00:36:38 +0100 (CET) Subject: [pypy-svn] r48382 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107233638.ABCE08167@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 00:36:38 2007 New Revision: 48382 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: this is no longer needed Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 00:36:38 2007 @@ -4,7 +4,7 @@ def setup_class(cls): # XXX the zipimport below is nonsense, of course, but leaving # it out crashes - space = gettestobjspace(usemodules=('clr', 'zipimport')) + space = gettestobjspace(usemodules=('clr', )) cls.space = space def test_import_hook_simple(self): From cfbolz at codespeak.net Thu Nov 8 00:38:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 00:38:43 +0100 (CET) Subject: [pypy-svn] r48383 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071107233843.C83A58167@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 00:38:43 2007 New Revision: 48383 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: of course the comment is nonsense now too. /me needs to go to bed Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 00:38:43 2007 @@ -2,8 +2,6 @@ class AppTestDotnet: def setup_class(cls): - # XXX the zipimport below is nonsense, of course, but leaving - # it out crashes space = gettestobjspace(usemodules=('clr', )) cls.space = space From rxe at codespeak.net Thu Nov 8 00:41:08 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 00:41:08 +0100 (CET) Subject: [pypy-svn] r48384 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071107234108.489878169@code0.codespeak.net> Author: rxe Date: Thu Nov 8 00:41:07 2007 New Revision: 48384 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/funcnode.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/module/genexterns.c pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/opaquenode.py pypy/dist/pypy/translator/llvm/opwriter.py Log: progress towards llvm2.0 syntax. Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Thu Nov 8 00:41:07 2007 @@ -53,22 +53,7 @@ self.cmds = [] def optimizations(self): - if llvm_version() < 2.0: - cmd = "gccas /dev/null -o /dev/null -debug-pass=Arguments 2>&1" - gccas_output = os.popen(cmd) - opts = gccas_output.read()[17:-1] + " " - - # these were added by Chris Lattner for some old version of llvm - # opts += "-globalopt -constmerge -ipsccp -deadargelim -inline " \ - # "-instcombine -scalarrepl -globalsmodref-aa -licm -load-vn " \ - # "-gcse -instcombine -simplifycfg -globaldce " - - # added try to reduce the amount of excessive inlining by us, llvm and gcc - # opts += "-inline-threshold=175 " #default: 200 - - else: - opts = '-std-compile-opts' - return opts + return '-std-compile-opts' def execute_cmds(self): c = stdoutcapture.Capture(mixed_out_err=True) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Nov 8 00:41:07 2007 @@ -403,11 +403,11 @@ for k, v in [ (rcarith.CByte, self.types[lltype.Char]), - (rcarith.CUByte, 'ubyte'), - (rcarith.CShort, 'short'), - (rcarith.CUShort, 'ushort'), - (rcarith.CInt, 'int'), - (rcarith.CUInt, 'uint'), + (rcarith.CUByte, 'i8'), + (rcarith.CShort, 'i16'), + (rcarith.CUShort, 'i16'), + (rcarith.CInt, 'i32'), + (rcarith.CUInt, 'i32'), (rcarith.CLong, self.types[lltype.Signed]), (rcarith.CULong, self.types[lltype.Unsigned]), (rcarith.CLonglong, self.types[lltype.SignedLongLong]), @@ -436,8 +436,10 @@ def repr_char(self, type_, value): x = ord(value) + print x if x >= 128: - r = "cast (ubyte %s to sbyte)" % x + # XXX check this really works + r = "trunc (i16 %s to i8)" % x else: r = str(x) return r @@ -486,11 +488,12 @@ return repr def repr_address(self, type_, value): + # XXX why-o-why isnt this an int ??? if not value: return 'null' ptr = value.ptr node, ref = self.database.repr_constant(ptr) - res = "cast(%s to sbyte*)" % (ref,) + res = "bitcast(%s to i8*)" % (ref,) return res def repr_weakgcaddress(self, type_, value): Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Thu Nov 8 00:41:07 2007 @@ -209,9 +209,9 @@ if c_name.startswith("RPyExc_"): c_name = c_name[1:] ccode.append("void raise%s(char *);\n" % c_name) - else: - # XXX we really shouldnt do this - predeclarefn(c_name, db.obj2node[obj._obj].ref) + #else: + # # XXX we really shouldnt do this + # predeclarefn(c_name, db.obj2node[obj._obj].ref) elif type(c_name) is str and type(obj) is int: ccode.append("#define\t%s\t%d\n" % (c_name, obj)) else: Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Thu Nov 8 00:41:07 2007 @@ -14,11 +14,12 @@ # signature of external functions differ from C's implementation ext_func_sigs = { - "@LL_stack_too_big" : ExtFuncSig("int", None), + "@LL_stack_too_big_" : ExtFuncSig("i32", None), } -if maxint != 2**31-1: - ext_func_sigs["@LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) +# XXX 64 bit is really broken since 2.x +#if maxint != 2**31-1: +# ext_func_sigs["@LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) class SimplerExternalFuncNode(FuncNode): @@ -58,12 +59,12 @@ codewriter.declare(self.getdecl()) class ExternalFuncNode(FuncNode): + prefix = '@' def __init__(self, db, value, extname=None): self.db = db self.value = value name = value._callable.__name__ - #assert name.startswith("ll") self.callable = value._callable if extname is not None: @@ -72,7 +73,10 @@ mapped_name = EXTERNALS[self.callable] self.make_name(mapped_name) - + if ext_func_sigs.get(self.ref + '_', None): + self.origref = self.ref + self.name += "_" + def setup(self): self.db.prepare_type(self.value._TYPE.RESULT) self.db.prepare_type_multi(self.value._TYPE._trueargs()) @@ -80,31 +84,31 @@ def _get_wrapper(self): wrapper = ext_func_sigs.get(self.ref, None) - if wrapper is None and maxint != 2**31-1: - #log("ref=%s" % self.ref) - rettype, args = self.getdecl_parts() - conversions = False - if rettype == "long": - rettype = "int" - conversions = True - elif rettype == "ulong": - rettype = "uint" - conversions = True - else: - rettype = None - for i, a in enumerate(args): - if a == "long": - args[i] = "int" - conversions = True - elif a == "ulong": - args[i] = "uint" - conversions = True - else: - args[i] = None - if conversions: - wrapper = ExtFuncSig(rettype, args) - #log(" rettype=%s" % str(rettype)) - #log(" args =%s" % str(args)) +# if wrapper is None and maxint != 2**31-1: +# #log("ref=%s" % self.ref) +# rettype, args = self.getdecl_parts() +# conversions = False +# if rettype == "long": +# rettype = "int" +# conversions = True +# elif rettype == "ulong": +# rettype = "uint" +# conversions = True +# else: +# rettype = None +# for i, a in enumerate(args): +# if a == "long": +# args[i] = "int" +# conversions = True +# elif a == "ulong": +# args[i] = "uint" +# conversions = True +# else: +# args[i] = None +# if conversions: +# wrapper = ExtFuncSig(rettype, args) +# #log(" rettype=%s" % str(rettype)) +# #log(" args =%s" % str(args)) return wrapper def getdecl_parts(self): @@ -166,15 +170,17 @@ expected_rettype = wrapper.rettype or rettype # call - codewriter.call(returnval, expected_rettype, self.ref, + codewriter.call(returnval, expected_rettype, self.origref, expected_argtypes, expected_argrefs) if wrapper.rettype: # cast to desired return type tmpval = returnval returnval = self.db.repr_tmpvar() + # XXX note we only have one case above - more will need + # to specialize the cast codewriter.cast(returnval, wrapper.rettype, - tmpval, rettype) + tmpval, rettype, 'trunc') codewriter.ret(rettype, returnval) codewriter.closefunc() Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Thu Nov 8 00:41:07 2007 @@ -66,7 +66,7 @@ self.write_block(codewriter, block) if self.bad_switch_block: codewriter.label('badswitch') - codewriter._indent('call void %abort()') + codewriter._indent('call void @abort()') codewriter._indent('unreachable') codewriter.closefunc() Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Thu Nov 8 00:41:07 2007 @@ -75,6 +75,7 @@ def _zeromalloc(self, codewriter, targetvar, size=1, atomic=False, exc_flag=False): """ assumes malloc of word size """ + XXX uword = self.db.get_machine_uword() boundary_size = 0 Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Thu Nov 8 00:41:07 2007 @@ -5,10 +5,17 @@ #endif #include "c/src/thread.h" -//#include "c/src/ll_os.h" -#include "c/src/ll_math.h" -#include "c/src/ll_strtod.h" -#include "c/src/stack.h" +#ifdef LL_NEED_MATH + #include "c/src/ll_math.h" +#endif + +#ifdef LL_NEED_STRTOD + #include "c/src/ll_strtod.h" +#endif + +#ifdef LL_NEED_STACK + #include "c/src/stack.h" +#endif // setup code for ThreadLock Opaque types /*char *RPyOpaque_LLVM_SETUP_ThreadLock(struct RPyOpaque_ThreadLock *lock, Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 8 00:41:07 2007 @@ -52,6 +52,9 @@ def from_unichar(arg): return ord(arg) +def from_float(arg): + return ctypes.c_double(arg) + def from_str(arg): # XXX wont work over isolate : arg should be converted into a string first n = len(arg.chars) @@ -108,7 +111,7 @@ # %(RT)r to_llargs = %(to_llargs)s -__entrypoint__ = _c.pypy_%(name)s +__entrypoint__.args = _c.pypy_%(name)s # %(ARGS)r ll_to_res = %(ll_to_res)s @@ -172,6 +175,9 @@ if A is lltype.UniChar: action = 'from_unichar' + elif A is lltype.Float: + action = 'from_float' + elif isinstance(A, lltype.Ptr) and A.TO is STR: action = 'from_str' else: Modified: pypy/dist/pypy/translator/llvm/opaquenode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opaquenode.py (original) +++ pypy/dist/pypy/translator/llvm/opaquenode.py Thu Nov 8 00:41:07 2007 @@ -47,6 +47,7 @@ T = self.value._TYPE # XXX similar non generic hacks to genc for now if T.tag == 'ThreadLock': + XXX argrefs = [self.get_ref()] argtypes = [self.db.repr_type(T) + "*"] lock = self.value.externalobj Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Nov 8 00:41:07 2007 @@ -90,12 +90,13 @@ for oo in 'lt le eq ne ge gt'.split(): binary_operations['%s_%s' % (tt, oo)] = 'fcmp u%s' % oo - char_operations = {'char_lt': 'icmp lt', - 'char_le': 'icmp le', - 'char_eq': 'icmp eq', - 'char_ne': 'icmp ne', - 'char_ge': 'icmp ge', - 'char_gt': 'icmp gt'} + # ZZZ check that operation should be checking unsigned + binary_operations.update({'char_lt': 'icmp ult', + 'char_le': 'icmp ule', + 'char_eq': 'icmp eq', + 'char_ne': 'icmp ne', + 'char_ge': 'icmp uge', + 'char_gt': 'icmp ugt'}) def __init__(self, db, codewriter): self.db = db @@ -139,48 +140,14 @@ elif op.opname in self.shift_operations: self.shiftop(opr) - elif op.opname in self.char_operations: - self.char_binaryop(opr) - elif op.opname.startswith('cast_') or op.opname.startswith('truncate_'): - if op.opname == 'cast_char_to_int': - self.cast_char_to_int(opr) - else: - self.cast_primitive(opr) + self.cast_primitive(opr) else: meth = getattr(self, op.opname, None) if not meth: raise Exception, "operation %s not found" % op.opname meth(opr) - def _generic_pow(self, opr, onestr): - - # XXX This broken as... will only work for constants - try: - value = "NO VALUE" - value = opr.op.args[1].value - operand = int(value) - except Exception, exc: - msg = 'XXX: Error: _generic_pow: Variable '\ - '%s - failed to convert to int %s' % (value, str(exc)) - self.codewriter.comment(msg) - raise Exception(msg) - - mult_type = opr.argtypes[0] - mult_val = opr.argrefs[0] - last_val = mult_val - - if operand < 1: - res_val = onestr - else: - res_val = mult_val - for ii in range(operand - 1): - res_val = self._tmp() - self.codewriter.binaryop("mul", res_val, mult_type, - last_val, mult_val) - last_val = res_val - self.codewriter.cast(opr.retref, mult_type, res_val, mult_type) - def _skipped(self, opr): self.codewriter.comment('***Skipping operation %s()' % opr.op.opname) keepalive = _skipped @@ -230,19 +197,12 @@ self.codewriter.binaryop(name, opr.retref, opr.argtypes[0], opr.argrefs[0], opr.argrefs[1]) - def char_binaryop(self, opr): - assert len(opr.argrefs) == 2 - name = self.char_operations[opr.op.opname] - c1, c2 = self._tmp(2) - self.codewriter.cast(c1, "sbyte", opr.argrefs[0], "ubyte") - self.codewriter.cast(c2, "sbyte", opr.argrefs[1], "ubyte") - self.codewriter.binaryop(name, opr.retref, "ubyte", c1, c2) - def shiftop(self, opr): op = opr.op name = self.shift_operations[op.opname] var = opr.argrefs[1] + # ZZZ why did we do this??? #if isinstance(op.args[1], Constant): # var = opr.argrefs[1] #else: @@ -251,21 +211,18 @@ self.codewriter.shiftop(name, opr.retref, opr.argtypes[0], opr.argrefs[0], var) - def cast_char_to_int(self, opr): - " works for all casts " - XXX - assert len(opr.argrefs) == 1 - intermediate = self._tmp() - self.codewriter.cast(intermediate, opr.argtypes[0], - opr.argrefs[0], "ubyte") - self.codewriter.cast(opr.retref, "ubyte", intermediate, opr.rettype) - def cast_primitive(self, opr): " works for all casts " fromtype = opr.argtypes[0] totype = opr.rettype casttype = "bitcast" if '*' not in fromtype: + if fromtype[0] == 'i8': + assert totype[0] == 'i' + tosize = int(totype[1:]) + assert tosize > 8 + casttype = "zext" + if fromtype[0] == 'i' and totype[0] == 'i': fromsize = int(fromtype[1:]) tosize = int(totype[1:]) @@ -299,7 +256,7 @@ llong_is_true = int_is_true def float_is_true(self, opr): - self.codewriter.binaryop("setne", opr.retref, opr.argtypes[0], + self.codewriter.binaryop("fcmp une", opr.retref, opr.argtypes[0], opr.argrefs[0], "0.0") def ptr_nonzero(self, opr): @@ -474,8 +431,8 @@ def adr_delta(self, opr): addr1, addr2 = self._tmp(2) - self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word) - self.codewriter.cast(addr2, opr.argtypes[1], opr.argrefs[1], self.word) + self.codewriter.cast(addr1, opr.argtypes[0], opr.argrefs[0], self.word, 'ptrtoint') + self.codewriter.cast(addr2, opr.argtypes[1], opr.argrefs[1], self.word, 'ptrtoint') self.codewriter.binaryop("sub", opr.retref, opr.rettype, addr1, addr2) def _op_adr_generic(self, opr, llvm_op): @@ -505,16 +462,16 @@ self._op_adr_cmp(opr, "icmp ne") def adr_le(self, opr): - self._op_adr_cmp(opr, "icmp le") + self._op_adr_cmp(opr, "icmp sle") def adr_gt(self, opr): - self._op_adr_cmp(opr, "icmp gt") + self._op_adr_cmp(opr, "icmp sgt") def adr_lt(self, opr): - self._op_adr_cmp(opr, "icmp lt") + self._op_adr_cmp(opr, "icmp slt") def adr_ge(self, opr): - self._op_adr_cmp(opr, "icmp ge") + self._op_adr_cmp(opr, "icmp sge") # XXX Not sure any of this makes sense - maybe seperate policy for # different flavours of mallocs? Well it depend on what happens the GC From rxe at codespeak.net Thu Nov 8 01:23:23 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 01:23:23 +0100 (CET) Subject: [pypy-svn] r48385 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071108002323.203988167@code0.codespeak.net> Author: rxe Date: Thu Nov 8 01:23:21 2007 New Revision: 48385 Modified: pypy/dist/pypy/translator/llvm/module/support.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py pypy/dist/pypy/translator/llvm/test/test_exc_operation.py pypy/dist/pypy/translator/llvm/test/test_typed.py Log: thats us pretty much passing everything as before, except for test_rint.py Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Thu Nov 8 01:23:21 2007 @@ -38,30 +38,28 @@ ret double %result } -""" -""" -internal CC WORD %pypyop_int_abs(WORD %x) { +define internal CC i32 @pypyop_int_abs(i32 %x) { block0: - %cond1 = setge WORD %x, 0 - br bool %cond1, label %return_block, label %block1 + %cond1 = icmp sge i32 %x, 0 + br i1 %cond1, label %return_block, label %block1 block1: - %x2 = sub WORD 0, %x + %x2 = sub i32 0, %x br label %return_block return_block: - %result = phi WORD [%x, %block0], [%x2, %block1] - ret WORD %result + %result = phi i32 [%x, %block0], [%x2, %block1] + ret i32 %result } -internal CC long %pypyop_llong_abs(long %x) { +define internal CC i64 @pypyop_llong_abs(i64 %x) { block0: - %cond1 = setge long %x, 0 - br bool %cond1, label %return_block, label %block1 + %cond1 = icmp sge i64 %x, 0 + br i1 %cond1, label %return_block, label %block1 block1: - %x2 = sub long 0, %x + %x2 = sub i64 0, %x br label %return_block return_block: - %result = phi long [%x, %block0], [%x2, %block1] - ret long %result + %result = phi i64 [%x, %block0], [%x2, %block1] + ret i64 %result } """ Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Nov 8 01:23:21 2007 @@ -88,7 +88,7 @@ for tt in 'float'.split(): for oo in 'lt le eq ne ge gt'.split(): - binary_operations['%s_%s' % (tt, oo)] = 'fcmp u%s' % oo + binary_operations['%s_%s' % (tt, oo)] = 'fcmp o%s' % oo # ZZZ check that operation should be checking unsigned binary_operations.update({'char_lt': 'icmp ult', @@ -234,14 +234,18 @@ else: pass else: - if (fromtype[0] == 'i' and totype == 'double'): + if (fromtype[0] == 'i' and totype in ['double', 'float']): # ZZZ signed casttype = "sitofp" - elif (fromtype == 'double' and totype[0] == 'i'): + elif (fromtype in ['double', 'float'] and totype[0] == 'i'): # ZZZ signed casttype = "fptosi" else: - assert False, "this shouldtnt be possible" + if fromtype != totype: + if fromtype == "double": + casttype = 'fptrunc' + else: + casttype = 'fpext' else: assert '*' in totype @@ -553,8 +557,7 @@ def is_early_constant(self, opr): # If it gets this far it is always false - self.codewriter.cast(opr.retref, 'bool', - 'false', opr.rettype) + self.codewriter.cast(opr.retref, 'i1', 'false', opr.rettype) def debug_llinterpcall(self, opr): self.codewriter.call(None, "void", "@abort", [], []) Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Thu Nov 8 01:23:21 2007 @@ -94,7 +94,7 @@ class FixedSizeArrayNode(StructNode): - prefix = '%fixarrayinstance_' + prefix = '@fixarrayinstance_' def __init__(self, db, struct): super(FixedSizeArrayNode, self).__init__(db, struct) Modified: pypy/dist/pypy/translator/llvm/test/test_exc_operation.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_exc_operation.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_exc_operation.py Thu Nov 8 01:23:21 2007 @@ -72,8 +72,6 @@ for i in (-sys.maxint-1, -sys.maxint, 0, sys.maxint-1, sys.maxint): assert f(i) == abs_int_ovf(i) -############################ - def test_int_overflow(): def fn(i): try: @@ -98,9 +96,7 @@ assert fn(0) == 1234 def test_int_mod_ovf_zer(): - #without raisingop2direct_call the operation is not found - #with raisingop2direct_call the wrong result is returned - py.test.skip("operation int_mod_ovf_zer not found") + py.test.skip("XXX fix this : the wrong result is returned") def fn(i): try: return snippet.mod_func(i) @@ -139,7 +135,6 @@ assert f(1) == 1234 def test_uint_arith(): - py.test.skip("zer operator exception not implemented") def fn(i): try: return ~(i*(i+1))/(i-1) @@ -174,8 +169,8 @@ assert f(-sys.maxint) == 123 def test_int_mul_ovf(): - if sys.maxint != 2**31-1: - py.test.skip("WIP on 64 bit architectures") + #if sys.maxint != 2**31-1: + # py.test.skip("WIP on 64 bit architectures") def mul_func(i): try: return ovfcheck(i * 100) Modified: pypy/dist/pypy/translator/llvm/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_typed.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_typed.py Thu Nov 8 01:23:21 2007 @@ -3,7 +3,7 @@ import py from py.test import raises from pypy.translator.test import snippet -from pypy.rlib.rarithmetic import r_uint, ovfcheck, ovfcheck_lshift +from pypy.rlib.rarithmetic import r_uint, r_longlong, ovfcheck, ovfcheck_lshift from pypy.translator.llvm.test.runtest import * @@ -288,11 +288,18 @@ assert str(f(i)) == str(fn(i)) def test_int_abs(): - def int_abs_(n): + def int_abs(n): return abs(n) - f = compile_function(int_abs_, [int]) + f = compile_function(int_abs, [int]) for i in (-25, 0, 75): - assert f(i) == int_abs_(i) + assert f(i) == int_abs(i) + +def test_long_abs(): + def long_abs(n): + return abs(r_longlong(n)) + f = compile_function(long_abs, [int]) + for i in (-25, 0, 75): + assert f(i) == long_abs(i) def test_float_abs(): def float_abs_(n): From fijal at codespeak.net Thu Nov 8 01:43:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 01:43:15 +0100 (CET) Subject: [pypy-svn] r48386 - pypy/branch/pypy-rpython-unicode/rpython/lltypesystem Message-ID: <20071108004315.680268162@code0.codespeak.net> Author: fijal Date: Thu Nov 8 01:43:13 2007 New Revision: 48386 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Log: First round of simplifications and getting rid of ugly typeOf switch Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Thu Nov 8 01:43:13 2007 @@ -41,13 +41,21 @@ mallocstr = new_malloc(STR) mallocunicode = new_malloc(UNICODE) +def emptystrfun(): + return emptystr + +def emptyunicodefun(): + return emptyunicode + STR.become(GcStruct('rpy_string', ('hash', Signed), ('chars', Array(Char, hints={'immutable': True, 'isrpystring': True})), - adtmeths={'malloc':staticAdtMethod(mallocstr)})) + adtmeths={'malloc' : staticAdtMethod(mallocstr), + 'empty' : staticAdtMethod(emptystrfun)})) UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed), ('chars', Array(UniChar, hints={'immutable': True})), - adtmeths={'malloc':staticAdtMethod(mallocunicode)} + adtmeths={'malloc' : staticAdtMethod(mallocunicode), + 'empty' : staticAdtMethod(emptyunicodefun)} )) SIGNED_ARRAY = GcArray(Signed) CONST_STR_CACHE = WeakValueDictionary() @@ -243,10 +251,7 @@ def ll_strip(s, ch, left, right): s_len = len(s.chars) if s_len == 0: - if typeOf(s).TO == STR: - return emptystr - else: - return emptyunicode + return s.empty() lpos = 0 rpos = s_len - 1 if left: @@ -256,11 +261,7 @@ while lpos < rpos and s.chars[rpos] == ch: rpos -= 1 r_len = rpos - lpos + 1 - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - result = malloc(r_len) + result = s.malloc(r_len) i = 0 j = lpos while i < r_len: @@ -273,16 +274,9 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - if typeOf(s).TO == STR: - return emptystr - else: - return emptyunicode + return s.empty() i = 0 - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - result = malloc(s_len) + result = s.malloc(s_len) while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': @@ -295,16 +289,9 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - if typeOf(s).TO == STR: - return emptystr - else: - return emptyunicode + return s.empty() i = 0 - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - result = malloc(s_len) + result = s.malloc(s_len) while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': @@ -318,20 +305,13 @@ s_len = len(s_chars) num_items = length if num_items == 0: - if typeOf(s).TO == STR: - return emptystr - else: - return emptyunicode + return s.empty() itemslen = 0 i = 0 while i < num_items: itemslen += len(items[i].chars) i += 1 - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - result = malloc(itemslen + s_len * (num_items - 1)) + result = s.malloc(itemslen + s_len * (num_items - 1)) res_chars = result.chars res_index = 0 i = 0 @@ -616,11 +596,7 @@ def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) - if typeOf(s1).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - newstr = malloc(len1 - start) + newstr = s1.malloc(len1 - start) j = 0 while start < len1: newstr.chars[j] = s1.chars[start] @@ -635,11 +611,7 @@ if start == 0: return s1 stop = len(s1.chars) - if typeOf(s1).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - newstr = malloc(stop - start) + newstr = s1.malloc(stop - start) j = 0 while start < stop: newstr.chars[j] = s1.chars[start] @@ -649,11 +621,7 @@ def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 - if typeOf(s1).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - newstr = malloc(newlen) + newstr = s1.malloc(newlen) j = 0 while j < newlen: newstr.chars[j] = s1.chars[j] @@ -674,13 +642,9 @@ i = 0 j = 0 resindex = 0 - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode while j < strlen: if chars[j] == c: - item = items[resindex] = malloc(j - i) + item = items[resindex] = s.malloc(j - i) newchars = item.chars k = i while k < j: @@ -689,7 +653,7 @@ resindex += 1 i = j + 1 j += 1 - item = items[resindex] = malloc(j - i) + item = items[resindex] = s.malloc(j - i) newchars = item.chars k = i while k < j: @@ -701,11 +665,7 @@ def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) - if typeOf(s).TO == STR: - malloc = mallocstr - else: - malloc = mallocunicode - newstr = malloc(length) + newstr = s.malloc(length) src = s.chars dst = newstr.chars j = 0 From rxe at codespeak.net Thu Nov 8 01:55:29 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 01:55:29 +0100 (CET) Subject: [pypy-svn] r48387 - pypy/dist/pypy/translator/llvm Message-ID: <20071108005529.A296A8164@code0.codespeak.net> Author: rxe Date: Thu Nov 8 01:55:29 2007 New Revision: 48387 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: aarggh - bed Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 8 01:55:29 2007 @@ -111,7 +111,7 @@ # %(RT)r to_llargs = %(to_llargs)s -__entrypoint__.args = _c.pypy_%(name)s +__entrypoint__.args = %(args)s # %(ARGS)r ll_to_res = %(ll_to_res)s From fijal at codespeak.net Thu Nov 8 01:55:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 01:55:44 +0100 (CET) Subject: [pypy-svn] r48388 - in pypy/branch/pypy-rpython-unicode: annotation rpython/test Message-ID: <20071108005544.3873A8164@code0.codespeak.net> Author: fijal Date: Thu Nov 8 01:55:43 2007 New Revision: 48388 Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py pypy/branch/pypy-rpython-unicode/annotation/unaryop.py pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: All string tests, which are decided to be supported by now, passes with unicode (for lltypesystem). There are few todos: * fix the if typeOf(...) evil hacks * think about unicode mod sth * think about str(some_unicode) Modified: pypy/branch/pypy-rpython-unicode/annotation/binaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/binaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Thu Nov 8 01:55:43 2007 @@ -615,10 +615,11 @@ lst1.listdef.resize() delitem.can_only_throw = [] -class __extend__(pairtype(SomeString, SomeSlice)): +class __extend__(pairtype(SomeString, SomeSlice), + pairtype(SomeUnicodeString, SomeSlice)): def getitem((str1, slic)): - return SomeString() + return str1.basestringclass() getitem.can_only_throw = [] class __extend__(pairtype(SomeString, SomeInteger)): @@ -660,11 +661,12 @@ getbookkeeper().count("str_mul", str1, int2) return SomeUnicodeString() -class __extend__(pairtype(SomeInteger, SomeString)): +class __extend__(pairtype(SomeInteger, SomeString), + pairtype(SomeInteger, SomeUnicodeString)): def mul((int1, str2)): # xxx do we want to support this getbookkeeper().count("str_mul", str2, int1) - return SomeString() + return str2.basestringclass() class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), pairtype(SomeUnicodeString, SomeUnicodeCodePoint), Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Thu Nov 8 01:55:43 2007 @@ -429,18 +429,20 @@ return SomeInteger(nonneg=True) def method_strip(str, chr): - return SomeString() + return str.basestringclass() def method_lstrip(str, chr): - return SomeString() + return str.basestringclass() def method_rstrip(str, chr): - return SomeString() + return str.basestringclass() def method_join(str, s_list): getbookkeeper().count("str_join", str) s_item = s_list.listdef.read_item() if isinstance(s_item, SomeImpossibleValue): + if isinstance(str, SomeUnicodeString): + return immutablevalue(u"") return immutablevalue("") return str.basestringclass() @@ -464,11 +466,12 @@ def method_replace(str, s1, s2): return str.basestringclass() - def method_lower(str): - return str.basestringclass() - +class __extend__(SomeString): def method_upper(str): - return str.basestringclass() + return SomeString() + + def method_lower(str): + return SomeString() class __extend__(SomeChar): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Thu Nov 8 01:55:43 2007 @@ -666,34 +666,38 @@ assert res == expected def test_char_mul_n(self): + const = self.const def f(c, n): return c*n - res = self.interpret(f, ['a', 4]) + res = self.interpret(f, [const('a'), 4]) assert self.ll_to_string(res) == 'a'*4 - res = self.interpret(f, ['a', 0]) + res = self.interpret(f, [const('a'), 0]) assert self.ll_to_string(res) == "" def test_char_mul_negative(self): + const = self.const def f(c): return c * -3 - res = self.interpret(f, ['a']) + res = self.interpret(f, [const('a')]) assert self.ll_to_string(res) == '' def test_n_mul_char(self): + const = self.const def f(c, n): return n*c - res = self.interpret(f, ['a', 4]) + res = self.interpret(f, [const('a'), 4]) assert self.ll_to_string(res) == 'a'*4 - res = self.interpret(f, ['a', 0]) + res = self.interpret(f, [const('a'), 0]) assert self.ll_to_string(res) == "" def test_hash(self): + const = self.const def fn(i): if i == 0: - s = '' + s = const('') else: - s = "xxx" + s = const("xxx") return hash(s) res = self.interpret(fn, [0]) assert res == self.EMPTY_STRING_HASH @@ -701,45 +705,52 @@ assert typeOf(res) == Signed def test_call_str_on_string(self): + const = self.const def fn(i): - s = "x" * i - return str(s) + s = const("x") * i + return const(s) res = self.interpret(fn, [3]) assert self.ll_to_string(res) == 'xxx' def test_count_char(self): + const = self.const def fn(i): - s = "".join(["abcasd"] * i) - return s.count("a") + s.count("a", 2) + s.count("b", 1, 6) + s = const("").join([const("abcasd")] * i) + return s.count(const("a")) + s.count(const("a"), 2) + \ + s.count(const("b"), 1, 6) res = self.interpret(fn, [4]) assert res == 8 + 7 + 1 def test_count(self): + const = self.const def fn(i): - s = "".join(["abcabsd"] * i) + s = const("").join([const("abcabsd")] * i) one = i / i # confuse the annotator - return (s.count("abc") + "abcde".count("") + - "abcda".count("a" * one)) + return (s.count(const("abc")) + const("abcde").count(const("")) + + const("abcda").count(const("a") * one)) res = self.interpret(fn, [4]) assert res == 4 + 6 + 2 def test_count_overlapping_occurences(self): + const = self.const def fn(): - return 'ababa'.count('aba') + return const('ababa').count(const('aba')) res = self.interpret(fn, []) assert res == 1 def test_hlstr(self): + const = self.const from pypy.rpython.annlowlevel import hlstr def f(s): - return "*"+hlstr(s)+"*" == "*abba*" + return const("*")+const(hlstr(s))+const("*") == const("*abba*") - res = self.interpret(f, [self.string_to_ll("abba")]) + res = self.interpret(f, [self.string_to_ll(const("abba"))]) assert res def test_getitem_exc(self): + const = self.const def f(x): - s = "z" + s = const("z") return s[x] res = self.interpret(f, [0]) @@ -752,13 +763,13 @@ assert False def f(x): - s = "z" + s = const("z") try: return s[x] except IndexError: - return 'X' + return const('X') except Exception: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -766,11 +777,11 @@ assert res == 'X' def f(x): - s = "z" + s = const("z") try: return s[x] except Exception: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -778,11 +789,11 @@ assert res == ' ' def f(x): - s = "z" + s = const("z") try: return s[x] except ValueError: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -794,10 +805,11 @@ assert False def test_fold_concat(self): + const = self.const def g(tail): - return "head"+tail + return const("head")+tail def f(): - return g("tail") + return g(const("tail")) from pypy import conftest t, typer, fgraph = self.gengraph(f, [], backendopt=True) Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 01:55:43 2007 @@ -55,6 +55,7 @@ test_int = unsupported test_int_valueerror = unsupported test_float = unsupported + test_hlstr = unsupported class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 From fijal at codespeak.net Thu Nov 8 02:09:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 02:09:56 +0100 (CET) Subject: [pypy-svn] r48389 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071108010956.076B48164@code0.codespeak.net> Author: fijal Date: Thu Nov 8 02:09:56 2007 New Revision: 48389 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: This is how we test things. Works better :) Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 02:09:56 2007 @@ -9,8 +9,8 @@ import clr import System.Math - print " abs(-5) = %s"%System.Math.Abs(-5) - print " 2^5 = %s"%System.Math.Pow(2,5) + assert System.Math.Abs(-5) == -5 + assert System.Math.Pow(2,5) == 2**5 # import System.Math ... From fijal at codespeak.net Thu Nov 8 02:12:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 02:12:33 +0100 (CET) Subject: [pypy-svn] r48390 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071108011233.C0B848165@code0.codespeak.net> Author: fijal Date: Thu Nov 8 02:12:33 2007 New Revision: 48390 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: put the space after comma Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 02:12:33 2007 @@ -10,7 +10,7 @@ import System.Math assert System.Math.Abs(-5) == -5 - assert System.Math.Pow(2,5) == 2**5 + assert System.Math.Pow(2, 5) == 2**5 # import System.Math ... From rxe at codespeak.net Thu Nov 8 02:16:34 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 02:16:34 +0100 (CET) Subject: [pypy-svn] r48391 - pypy/dist/pypy/translator/llvm Message-ID: <20071108011634.7CFE48162@code0.codespeak.net> Author: rxe Date: Thu Nov 8 02:16:33 2007 New Revision: 48391 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: aaaaaarrrggh ** 2 - gnite Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 8 02:16:33 2007 @@ -111,7 +111,7 @@ # %(RT)r to_llargs = %(to_llargs)s -__entrypoint__.args = %(args)s +__entrypoint__.argtypes = %(args)s # %(ARGS)r ll_to_res = %(ll_to_res)s From tismer at codespeak.net Thu Nov 8 04:36:47 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 8 Nov 2007 04:36:47 +0100 (CET) Subject: [pypy-svn] r48392 - pypy/dist/pypy/objspace/flow/test Message-ID: <20071108033647.07D318163@code0.codespeak.net> Author: tismer Date: Thu Nov 8 04:36:46 2007 New Revision: 48392 Added: pypy/dist/pypy/objspace/flow/test/test___import_.py Log: adding a tiny check for sc_import annotator functionality Added: pypy/dist/pypy/objspace/flow/test/test___import_.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/flow/test/test___import_.py Thu Nov 8 04:36:46 2007 @@ -0,0 +1,12 @@ +from py.test import raises + +def test___import_(): + import os + mod = os.name + raises(TypeError, __import__, ()) + x = __import__(mod) + x = __import__(mod, None) + x = __import__(mod, None, None) + x = __import__(mod, None, None, None) + raises(TypeError, __import__, (mod, None, None, None, None)) + # XXX this will have to be adjusted for Python 2.5 pretty soon-ish :-) \ No newline at end of file From tismer at codespeak.net Thu Nov 8 04:42:53 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 8 Nov 2007 04:42:53 +0100 (CET) Subject: [pypy-svn] r48393 - in pypy/dist/pypy: module/posix module/posix/test rpython/module rpython/module/test Message-ID: <20071108034253.3AA2D8164@code0.codespeak.net> Author: tismer Date: Thu Nov 8 04:42:52 2007 New Revision: 48393 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: added _getfullpathname for windows. including tests, yeah! Especially tested the bad case of "d:stuff" not being propagated through this limited ntpath.py implementation! Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Thu Nov 8 04:42:52 2007 @@ -88,6 +88,9 @@ interpleveldefs['geteuid'] = 'interp_posix.geteuid' if hasattr(os, 'getgid'): interpleveldefs['getgid'] = 'interp_posix.getgid' + # not visible via os, inconsistency in nt: + if hasattr(posix, '_getfullpathname'): + interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' for name in RegisterOs.w_star: if hasattr(os, name): Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Thu Nov 8 04:42:52 2007 @@ -236,6 +236,17 @@ raise wrap_oserror(space, e) remove.unwrap_spec = [ObjSpace, str] +def _getfullpathname(space, path): + """helper for ntpath.abspath """ + posix = __import__(os.name) # nt specific + try: + fullpath = posix._getfullpathname(path) + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(fullpath) +_getfullpathname.unwrap_spec = [ObjSpace, str] + def getcwd(space): """Return the current working directory.""" try: Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Thu Nov 8 04:42:52 2007 @@ -223,6 +223,16 @@ stream = os.popen('echo 1') assert stream.read() == '1\n' + if hasattr(__import__(os.name), '_getfullpathname'): + def test__getfullpathname(self): + # nt specific + posix = self.posix + import os + sysdrv = os.getenv("SystemDrive", "C:") + # just see if it does anything + path = sysdrv + 'hubber' + assert os.sep in posix._getfullpathname(path) + def test_utime(self): os = self.posix import os.path Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Thu Nov 8 04:42:52 2007 @@ -519,6 +519,57 @@ export_name="ll_os.ll_os_access", oofakeimpl=os_access_oofakeimpl) + @registering_if(posix, '_getfullpathname') + def register_posix__getfullpathname(self): + # this nt function is not exposed via os, but needed + # to get a correct implementation of os.abspath + # XXX why do we ignore WINAPI conventions everywhere? + class CConfig: + _includes_ = ['Windows.h'] + MAX_PATH = platform.ConstantInteger('MAX_PATH') + DWORD = platform.SimpleType("DWORD", rffi.ULONG) + LPCTSTR = platform.SimpleType("LPCTSTR", rffi.CCHARP) + LPTSTR = platform.SimpleType("LPTSTR", rffi.CCHARP) + LPTSTRP = platform.SimpleType("LPTSTR*", rffi.CCHARPP) + + config = platform.configure(CConfig) + MAX_PATH = config['MAX_PATH'] + DWORD = config['DWORD'] + LPCTSTR = config['LPCTSTR'] + LPTSTR = config['LPTSTR'] + LPTSTRP = config['LPTSTRP'] + # XXX unicode? + GetFullPathName = self.llexternal('GetFullPathNameA', + [LPCTSTR, DWORD, LPTSTR, LPTSTRP], DWORD) + GetLastError = self.llexternal('GetLastError', [], DWORD) + ##DWORD WINAPI GetFullPathName( + ## __in LPCTSTR lpFileName, + ## __in DWORD nBufferLength, + ## __out LPTSTR lpBuffer, + ## __out LPTSTR* lpFilePart + ##); + + def _getfullpathname_llimpl(lpFileName): + nBufferLength = MAX_PATH + 1 + lpBuffer = lltype.malloc(LPTSTR.TO, nBufferLength, flavor='raw') + try: + res = GetFullPathName( + lpFileName, rffi.cast(DWORD, nBufferLength), + lpBuffer, lltype.nullptr(LPTSTRP.TO)) + if res == 0: + error = GetLastError() + raise OSError(error, "_getfullpathname failed") + # XXX ntpath expects WindowsError :-( + result = rffi.charp2str(lpBuffer) + return result + finally: + lltype.free(lpBuffer, flavor='raw') + + return extdef([str], # a single argument which is a str + str, # returns a string + "ll_os.posix__getfullpathname", + llimpl=_getfullpathname_llimpl) + @registering(os.getcwd) def register_os_getcwd(self): os_getcwd = self.llexternal(underscore_on_windows + 'getcwd', Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Thu Nov 8 04:42:52 2007 @@ -31,7 +31,17 @@ for value in times: assert isinstance(value, float) - +def test__getfullpathname(): + if os.name != 'nt': + py.test.skip('nt specific function') + posix = __import__(os.name) + sysdrv = os.getenv('SystemDrive', 'C:') + stuff = sysdrv + 'stuff' + data = getllimpl(posix._getfullpathname)(stuff) + assert data == posix._getfullpathname(stuff) + # the most intriguing failure of ntpath.py should not repeat, here: + assert not data.endswith(stuff) + def test_getcwd(): data = getllimpl(os.getcwd)() assert data == os.getcwd() From tismer at codespeak.net Thu Nov 8 04:44:55 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 8 Nov 2007 04:44:55 +0100 (CET) Subject: [pypy-svn] r48394 - in pypy/dist/pypy/translator/goal: . test2 Message-ID: <20071108034455.5D8B28167@code0.codespeak.net> Author: tismer Date: Thu Nov 8 04:44:54 2007 New Revision: 48394 Modified: pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/test2/test_app_main.py Log: changed app_main to implement a correct app-leve "we_are_translated" function. There are still lots of features which I would like to test for windows, but no idea how to do this. Help??? Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Thu Nov 8 04:44:54 2007 @@ -188,11 +188,12 @@ modules changeable, again. """ -try: - # for some reason, some tests break with this import - from pypy.rlib.objectmodel import we_are_translated -except ImportError: - we_are_translated = lambda: False +def we_are_translated(): + # this function does not exist on app-level. + # Don't confuse it with + # from pypy.rlib.objectmodel import we_are_translated + # which I did. + return hasattr(sys, 'pypy_translation_info') class fake_os: def __init__(self): Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/dist/pypy/translator/goal/test2/test_app_main.py Thu Nov 8 04:44:54 2007 @@ -338,7 +338,6 @@ assert 'mymodule running' in data assert 'Name: __main__' in data # ignoring case for windows. abspath behaves different from autopath - # concerning drive letters right now - # XXX fix nt module to implement _getfullpathname - assert ('File: ' + p).lower() in data.lower() - assert ('Argv: ' + repr([p, 'extra'])).lower() in data.lower() + # concerning drive letters right now. + assert ('File: ' + p) in data + assert ('Argv: ' + repr([p, 'extra'])) in data From tismer at codespeak.net Thu Nov 8 04:55:25 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 8 Nov 2007 04:55:25 +0100 (CET) Subject: [pypy-svn] r48395 - pypy/dist/pypy/objspace/flow/test Message-ID: <20071108035525.14D4D8178@code0.codespeak.net> Author: tismer Date: Thu Nov 8 04:55:24 2007 New Revision: 48395 Modified: pypy/dist/pypy/objspace/flow/test/test___import_.py Log: making a trivial test a bit less trivial Modified: pypy/dist/pypy/objspace/flow/test/test___import_.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test___import_.py (original) +++ pypy/dist/pypy/objspace/flow/test/test___import_.py Thu Nov 8 04:55:24 2007 @@ -4,9 +4,9 @@ import os mod = os.name raises(TypeError, __import__, ()) - x = __import__(mod) - x = __import__(mod, None) - x = __import__(mod, None, None) - x = __import__(mod, None, None, None) + assert __import__(mod).__name__ == os.name + assert __import__(mod, None).__name__ == os.name + assert __import__(mod, None, None).__name__ == os.name + assert __import__(mod, None, None, None).__name__ == os.name raises(TypeError, __import__, (mod, None, None, None, None)) # XXX this will have to be adjusted for Python 2.5 pretty soon-ish :-) \ No newline at end of file From tismer at codespeak.net Thu Nov 8 05:02:07 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 8 Nov 2007 05:02:07 +0100 (CET) Subject: [pypy-svn] r48396 - pypy/dist/pypy/objspace/flow/test Message-ID: <20071108040207.C4AB880CE@code0.codespeak.net> Author: tismer Date: Thu Nov 8 05:02:07 2007 New Revision: 48396 Added: pypy/dist/pypy/objspace/flow/test/test___import__.py - copied unchanged from r48395, pypy/dist/pypy/objspace/flow/test/test___import_.py Removed: pypy/dist/pypy/objspace/flow/test/test___import_.py Log: sorry, forgot an underscore for this file name. If somebody wants to merge it into anything else, no objection! From antocuni at codespeak.net Thu Nov 8 10:42:29 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Nov 2007 10:42:29 +0100 (CET) Subject: [pypy-svn] r48397 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071108094229.F3AA58175@code0.codespeak.net> Author: antocuni Date: Thu Nov 8 10:42:28 2007 New Revision: 48397 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: - Abs(-5) is 5, not -5 :-) - Add an assert to check that Math is loaded only one time - Add a failing test for non existent modules Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Nov 8 10:42:28 2007 @@ -9,8 +9,15 @@ import clr import System.Math - assert System.Math.Abs(-5) == -5 + assert System.Math.Abs(-5) == 5 assert System.Math.Pow(2, 5) == 2**5 - # import System.Math ... + Math = clr.load_cli_class('System', 'Math') + assert Math is System.Math + def test_ImportError(self): + skip('Fixme!') + def fn(): + import non_existent_module + raises(ImportError, fn()) + From rxe at codespeak.net Thu Nov 8 10:48:16 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 10:48:16 +0100 (CET) Subject: [pypy-svn] r48398 - pypy/dist/pypy/translator/llvm Message-ID: <20071108094816.4DD7A8175@code0.codespeak.net> Author: rxe Date: Thu Nov 8 10:48:15 2007 New Revision: 48398 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/opwriter.py Log: improve signed-ness operations a bit. fix call_boehm_gc_alloc. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Nov 8 10:48:15 2007 @@ -400,7 +400,8 @@ self.types[from_] = type if from_ not in self.reprs: self.reprs[from_] = self.repr_default - + + # XXX dont do this for k, v in [ (rcarith.CByte, self.types[lltype.Char]), (rcarith.CUByte, 'i8'), Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Nov 8 10:48:15 2007 @@ -213,9 +213,36 @@ def cast_primitive(self, opr): " works for all casts " - fromtype = opr.argtypes[0] totype = opr.rettype + fromtype = opr.argtypes[0] + to_lltype = opr.op.result.concretetype + from_lltype = opr.op.args[0].concretetype + + def issigned(ct): + # XXX MESS MESS MESS... but there doesnt seem to a be an + # obvious consistent place to get all numbers by type any more + # :-( + from pypy.rpython.lltypesystem.rffi import platform + m = platform.numbertype_to_rclass + assert ct in m or ct in [ + lltype.Bool, + lltype.Char, + lltype.UniChar, + lltype.Signed, + lltype.Unsigned, + lltype.SignedLongLong, + lltype.UnsignedLongLong] + + if ct in [lltype.Signed, lltype.SignedLongLong]: + return True + elif ct in m: + return m[ct].SIGNED + else: + return False + # XXX :-( + casttype = "bitcast" + if '*' not in fromtype: if fromtype[0] == 'i8': assert totype[0] == 'i' @@ -227,19 +254,26 @@ fromsize = int(fromtype[1:]) tosize = int(totype[1:]) if tosize > fromsize: - # ZZZ signed - casttype = "zext" + if issigned(from_lltype): + casttype = "sext" + else: + casttype = "zext" elif tosize < fromsize: casttype = "trunc" else: pass else: if (fromtype[0] == 'i' and totype in ['double', 'float']): - # ZZZ signed - casttype = "sitofp" + if issigned(from_lltype): + casttype = "sitofp" + else: + casttype = "uitofp" + elif (fromtype in ['double', 'float'] and totype[0] == 'i'): - # ZZZ signed - casttype = "fptosi" + if issigned(to_lltype): + casttype = "fptosi" + else: + casttype = "fptoui" else: if fromtype != totype: if fromtype == "double": @@ -290,7 +324,7 @@ def call_boehm_gc_alloc(self, opr): word = self.db.get_machine_word() - self.codewriter.call(opr.retref, 'i8*', '%pypy_malloc', + self.codewriter.call(opr.retref, 'i8*', '@pypy_malloc', [word], [opr.argrefs[0]]) def to_getelementptr(self, TYPE, args): From rxe at codespeak.net Thu Nov 8 10:49:11 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 10:49:11 +0100 (CET) Subject: [pypy-svn] r48399 - pypy/dist/pypy/translator/llvm/module Message-ID: <20071108094911.562058175@code0.codespeak.net> Author: rxe Date: Thu Nov 8 10:49:11 2007 New Revision: 48399 Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c Log: more cleaning up and reduce compile times some more for tests Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Thu Nov 8 10:49:11 2007 @@ -4,7 +4,6 @@ # define HAVE_RPY_LIST_OF_STRING #endif -#include "c/src/thread.h" #ifdef LL_NEED_MATH #include "c/src/ll_math.h" #endif @@ -14,26 +13,10 @@ #endif #ifdef LL_NEED_STACK + #include "c/src/thread.h" #include "c/src/stack.h" #endif -// setup code for ThreadLock Opaque types -/*char *RPyOpaque_LLVM_SETUP_ThreadLock(struct RPyOpaque_ThreadLock *lock, - int initially_locked) { - - struct RPyOpaque_ThreadLock tmp = RPyOpaque_INITEXPR_ThreadLock; - memcpy(lock, &tmp, sizeof(struct RPyOpaque_ThreadLock)); - - if (!RPyThreadLockInit(lock)) { - return "Thread lock init error"; - } - if ((initially_locked) && !RPyThreadAcquireLock(lock, 1)) { - return "Cannot acquire thread lock at init"; - } - return NULL; -} -*/ - // raw malloc code char *raw_malloc(long size) { return malloc(size); From rxe at codespeak.net Thu Nov 8 11:25:22 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 11:25:22 +0100 (CET) Subject: [pypy-svn] r48401 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071108102522.6E7AE814C@code0.codespeak.net> Author: rxe Date: Thu Nov 8 11:25:22 2007 New Revision: 48401 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: some more module wrapping crap to shoehorn llvm into test framework... still a lot of these tests fail - they should be fixed or skipped or special cased or something. Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Thu Nov 8 11:25:22 2007 @@ -1,3 +1,5 @@ +" XXX THIS IS ONLY FOR TESTING XXX " + import py import ctypes from pypy.rpython.lltypesystem import lltype @@ -16,7 +18,7 @@ rpyexc_occured = _c.pypy__rpyexc_occured rpyexc_occured.argtypes = [] rpyexc_occured.restype = ctypes.c_int - + rpyexc_fetch_type = _c.pypy_rpyexc_fetch_type rpyexc_fetch_type.argtypes = [] rpyexc_fetch_type.restype = ctypes.c_void_p @@ -31,19 +33,17 @@ _setup = False -class LLVMException(Exception): - pass - def entrypoint(*args): global _setup if not _setup: if not startup_code(): - raise LLVMException("Failed to startup") + raise Exception("Failed to startup") _setup = True args = [f(a) for a, f in zip(args, to_llargs)] result = __entrypoint__(*args) if rpyexc_occured(): - raise LLVMException("Exception raised") + exc = rpyexc_fetch_type() + return {'value':to_exception_type(exc), 'type':'exceptiontypename'} return ll_to_res(result) def identity(res): @@ -55,6 +55,9 @@ def from_float(arg): return ctypes.c_double(arg) +def to_unichar(res): + return unichr(res) + def from_str(arg): # XXX wont work over isolate : arg should be converted into a string first n = len(arg.chars) @@ -87,16 +90,28 @@ else: return None -def to_tuple(res, size, C_TYPE, action): +def struct_to_tuple(res, size, C_TYPE, action): if res: t = ctypes.cast(res, ctypes.POINTER(C_TYPE * size)).contents - return tuple([action(ii) for ii in t]) + return {'type':'tuple', 'value':tuple([action(ii) for ii in t])} else: return None -def to_list(res, C_TYPE, action): +def list_to_array(res, action): if res: - size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value + class List(ctypes.Structure): + _fields_ = [("length", ctypes.c_int), + ("items", ctypes.c_void_p)] + list = ctypes.cast(res, ctypes.POINTER(List)).contents + size = list.length + return action(list.items, size) + else: + return None + +def array_to_list(res, C_TYPE, action, size=-1): + if res: + if size == -1: + size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value class Array(ctypes.Structure): _fields_ = [("size", ctypes.c_int), ("data", C_TYPE * size)] @@ -104,6 +119,12 @@ return [action(array.data[ii]) for ii in range(size)] else: return None + +def to_exception_type(addr): + addr_str = ctypes.cast(addr+12, ctypes.POINTER(ctypes.c_int)).contents.value + size = ctypes.cast(addr_str, ctypes.POINTER(ctypes.c_int)).contents.value - 1 + name = ctypes.string_at(addr_str+4, size) + return name """ epilog = """ @@ -208,18 +229,25 @@ action = 'to_str' elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Struct): - fields = [getattr(T.TO, name) for name in T.TO._names_without_voids()] + S = T.TO + fields = [(getattr(S, name), name) for name in S._names_without_voids()] if fields: - F0 = fields[0] - _c_type, _action = self.build_lltype_to_ctypes_to_res(F0) - action = self.create_simple_closure('res', 'to_tuple(res, %s, %s, %s)' % (len(fields), - _c_type, - _action)) + F0, name = fields[0] + if name.startswith("item"): + _c_type, _action = self.build_lltype_to_ctypes_to_res(F0) + action = self.create_simple_closure('res', 'struct_to_tuple(res, %s, %s, %s)' % (len(fields), + _c_type, + _action)) + elif name == "length" and fields[1][1] == "items": + _c_type, _action = self.build_lltype_to_ctypes_to_res(fields[1][0]) + action = self.create_simple_closure('res', 'list_to_array(res, %s)' % (_action)) + else: + py.test.skip("unspported struct %r" % S) elif isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.Array): OF = T.TO.OF _c_type, _action = self.build_lltype_to_ctypes_to_res(OF) - action = self.create_simple_closure('res', 'to_list(res, %s, %s)' % (_c_type, _action)) + action = self.create_simple_closure('res, size=-1', 'array_to_list(res, %s, %s, size=size)' % (_c_type, _action)) else: assert not isinstance(T, lltype.Ptr) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Thu Nov 8 11:25:22 2007 @@ -74,7 +74,8 @@ 'translation.backendopt.none': not optimize, 'translation.gc': 'boehm', 'translation.llvm_via_c' : not native_llvm_backend -} + } + options.update(extra_opts) config.set(**options) driver = TranslationDriver(config=config) @@ -125,7 +126,6 @@ def __repr__(self): return 'ExceptionWrapper(%s)' % repr(self.class_name) - class LLVMTest(BaseRtypingTest, LLRtypeMixin): def __init__(self): self._func = None @@ -157,8 +157,13 @@ def interpret(self, fn, args, annotation=None): f = self._compile(fn, args, annotation) res = f(*args) - if isinstance(res, ExceptionWrapper): - raise res + + # a start to making this work over Isolate + if isinstance(res, dict): + if res["type"] == "exceptiontypename": + raise ExceptionWrapper(res["value"]) + elif res["type"] == "tuple": + res = StructTuple(res["value"]) return res def interpret_raises(self, exception, fn, args): @@ -194,3 +199,11 @@ def read_attr(self, obj, name): py.test.skip('read_attr not supported on llvm tests') + +class StructTuple(tuple): + def __getattr__(self, name): + if name.startswith('item'): + i = int(name[len('item'):]) + return self[i] + else: + raise AttributeError, name From rxe at codespeak.net Thu Nov 8 12:37:16 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 12:37:16 +0100 (CET) Subject: [pypy-svn] r48402 - pypy/dist/pypy/translator/llvm Message-ID: <20071108113716.940CE815C@code0.codespeak.net> Author: rxe Date: Thu Nov 8 12:37:15 2007 New Revision: 48402 Modified: pypy/dist/pypy/translator/llvm/genllvm.py Log: yet another hack for function name clashes Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Nov 8 12:37:15 2007 @@ -171,6 +171,7 @@ name = entry_node.name if name in self.function_count: self.function_count[name] += 1 + Node.nodename_count[name] = self.function_count[name] + 1 name += '_%d' % self.function_count[name] entry_node.name = name else: From rxe at codespeak.net Thu Nov 8 12:39:02 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 12:39:02 +0100 (CET) Subject: [pypy-svn] r48403 - pypy/dist/pypy/translator/llvm Message-ID: <20071108113902.860E3815F@code0.codespeak.net> Author: rxe Date: Thu Nov 8 12:39:02 2007 New Revision: 48403 Modified: pypy/dist/pypy/translator/llvm/database.py Log: oops - wrong size Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Nov 8 12:39:02 2007 @@ -351,7 +351,7 @@ lltype.Bool: "i1", lltype.SingleFloat: "float", lltype.Float: "double", - lltype.UniChar: "i8", + lltype.UniChar: "i16", lltype.Void: "void", lltype.UnsignedLongLong: "i64", lltype.SignedLongLong: "i64", From rxe at codespeak.net Thu Nov 8 12:51:29 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 12:51:29 +0100 (CET) Subject: [pypy-svn] r48404 - pypy/dist/pypy/translator/llvm Message-ID: <20071108115129.2150B815F@code0.codespeak.net> Author: rxe Date: Thu Nov 8 12:51:28 2007 New Revision: 48404 Modified: pypy/dist/pypy/translator/llvm/database.py Log: use rffi ll-number types, and not rctypes ones Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Nov 8 12:51:28 2007 @@ -393,7 +393,7 @@ except ImportError: pass else: - from pypy.rpython.rctypes import rcarithmetic as rcarith + from pypy.rpython.lltypesystem import rffi def update(from_, type): if from_ not in self.types: @@ -401,18 +401,17 @@ if from_ not in self.reprs: self.reprs[from_] = self.repr_default - # XXX dont do this for k, v in [ - (rcarith.CByte, self.types[lltype.Char]), - (rcarith.CUByte, 'i8'), - (rcarith.CShort, 'i16'), - (rcarith.CUShort, 'i16'), - (rcarith.CInt, 'i32'), - (rcarith.CUInt, 'i32'), - (rcarith.CLong, self.types[lltype.Signed]), - (rcarith.CULong, self.types[lltype.Unsigned]), - (rcarith.CLonglong, self.types[lltype.SignedLongLong]), - (rcarith.CULonglong, self.types[lltype.UnsignedLongLong])]: + (rffi.SIGNEDCHAR, 'i18'), + (rffi.UCHAR, 'i18'), + (rffi.SHORT, 'i16'), + (rffi.USHORT, 'i16'), + (rffi.INT, 'i32'), + (rffi.UINT, 'i32'), + (rffi.LONG, self.types[lltype.Signed]), + (rffi.ULONG, self.types[lltype.Signed]), + (rffi.LONGLONG, 'i64'), + (rffi.ULONGLONG, 'i64')]: update(k, v) def __getitem__(self, key): From rxe at codespeak.net Thu Nov 8 13:14:33 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 13:14:33 +0100 (CET) Subject: [pypy-svn] r48405 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071108121433.176DF815C@code0.codespeak.net> Author: rxe Date: Thu Nov 8 13:14:31 2007 New Revision: 48405 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: add lltype.UniChar to sizeof() Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Nov 8 13:14:31 2007 @@ -404,6 +404,8 @@ tp = ULONG # XXX! if tp is lltype.Char: return 1 + if tp is lltype.UniChar: + return 2 if tp is lltype.Float: return 8 assert isinstance(tp, lltype.Number) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Thu Nov 8 13:14:31 2007 @@ -247,6 +247,7 @@ cache = { lltype.Signed: ctypes.c_long, lltype.Unsigned: ctypes.c_ulong, + lltype.UniChar: ctypes.c_ushort, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, SIGNEDCHAR: ctypes.c_byte, @@ -266,6 +267,7 @@ assert sizeof(ll) == ctypes.sizeof(ctp) assert not size_and_sign(lltype.Signed)[1] assert not size_and_sign(lltype.Char)[1] + assert not size_and_sign(lltype.UniChar)[1] assert size_and_sign(UINT)[1] def test_rffi_offsetof(): From rxe at codespeak.net Thu Nov 8 13:16:14 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 13:16:14 +0100 (CET) Subject: [pypy-svn] r48406 - pypy/dist/pypy/translator/llvm Message-ID: <20071108121614.C46798162@code0.codespeak.net> Author: rxe Date: Thu Nov 8 13:16:14 2007 New Revision: 48406 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: rationalize issigned() madness. thanks fijal Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Nov 8 13:16:14 2007 @@ -219,27 +219,10 @@ from_lltype = opr.op.args[0].concretetype def issigned(ct): - # XXX MESS MESS MESS... but there doesnt seem to a be an - # obvious consistent place to get all numbers by type any more - # :-( - from pypy.rpython.lltypesystem.rffi import platform - m = platform.numbertype_to_rclass - assert ct in m or ct in [ - lltype.Bool, - lltype.Char, - lltype.UniChar, - lltype.Signed, - lltype.Unsigned, - lltype.SignedLongLong, - lltype.UnsignedLongLong] - - if ct in [lltype.Signed, lltype.SignedLongLong]: - return True - elif ct in m: - return m[ct].SIGNED - else: - return False - # XXX :-( + if ct is lltype.Bool: + return False + from pypy.rpython.lltypesystem.rffi import size_and_sign + return size_and_sign(ct)[1] casttype = "bitcast" From rxe at codespeak.net Thu Nov 8 13:30:06 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 13:30:06 +0100 (CET) Subject: [pypy-svn] r48407 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071108123006.8A6058179@code0.codespeak.net> Author: rxe Date: Thu Nov 8 13:30:05 2007 New Revision: 48407 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: unichars are normally 4 bytes Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Nov 8 13:30:05 2007 @@ -405,7 +405,7 @@ if tp is lltype.Char: return 1 if tp is lltype.UniChar: - return 2 + return 4 if tp is lltype.Float: return 8 assert isinstance(tp, lltype.Number) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Thu Nov 8 13:30:05 2007 @@ -247,7 +247,7 @@ cache = { lltype.Signed: ctypes.c_long, lltype.Unsigned: ctypes.c_ulong, - lltype.UniChar: ctypes.c_ushort, + lltype.UniChar: ctypes.c_uint, lltype.Char: ctypes.c_ubyte, DOUBLE: ctypes.c_double, SIGNEDCHAR: ctypes.c_byte, From cfbolz at codespeak.net Thu Nov 8 13:46:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 13:46:19 +0100 (CET) Subject: [pypy-svn] r48408 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108124619.432508172@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 13:46:19 2007 New Revision: 48408 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Log: introduce Unicode. check in to have Anto take over. Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 13:46:19 2007 @@ -322,16 +322,14 @@ return set() -# WARNING: the name 'String' is rebound at the end of file -class String(BuiltinADTType): - SELFTYPE_T = object() +class AbstractString(BuiltinADTType): def __init__(self): self._null = _null_string(self) generic_types = { self.SELFTYPE_T: self } self._GENERIC_METHODS = frozendict({ - "ll_stritem_nonneg": Meth([Signed], Char), + "ll_stritem_nonneg": Meth([Signed], self.CHAR), "ll_strlen": Meth([], Signed), "ll_strconcat": Meth([self.SELFTYPE_T], self.SELFTYPE_T), "ll_streq": Meth([self.SELFTYPE_T], Bool), @@ -341,25 +339,19 @@ "ll_find": Meth([self.SELFTYPE_T, Signed, Signed], Signed), "ll_rfind": Meth([self.SELFTYPE_T, Signed, Signed], Signed), "ll_count": Meth([self.SELFTYPE_T, Signed, Signed], Signed), - "ll_find_char": Meth([Char, Signed, Signed], Signed), - "ll_rfind_char": Meth([Char, Signed, Signed], Signed), - "ll_count_char": Meth([Char, Signed, Signed], Signed), - "ll_strip": Meth([Char, Bool, Bool], self.SELFTYPE_T), + "ll_find_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_rfind_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_count_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_strip": Meth([self.CHAR, Bool, Bool], self.SELFTYPE_T), "ll_upper": Meth([], self.SELFTYPE_T), "ll_lower": Meth([], self.SELFTYPE_T), "ll_substring": Meth([Signed, Signed], self.SELFTYPE_T), # ll_substring(start, count) - "ll_split_chr": Meth([Char], List(self.SELFTYPE_T)), - "ll_contains": Meth([Char], Bool), - "ll_replace_chr_chr": Meth([Char, Char], self.SELFTYPE_T), + "ll_split_chr": Meth([self.CHAR], List(self.SELFTYPE_T)), + "ll_contains": Meth([self.CHAR], Bool), + "ll_replace_chr_chr": Meth([self.CHAR, self.CHAR], self.SELFTYPE_T), }) self._setup_methods(generic_types) - def _enforce(self, value): - TYPE = typeOf(value) - if TYPE == Char: - return make_string(value) - else: - return BuiltinADTType._enforce(self, value) # TODO: should it return _null or ''? def _defl(self): @@ -373,6 +365,35 @@ def _specialize(self, generic_types): return self +# WARNING: the name 'String' is rebound at the end of file +class String(AbstractString): + SELFTYPE_T = object() + CHAR = Char + + def _enforce(self, value): + # XXX share this with Unicode? + TYPE = typeOf(value) + if TYPE == self.CHAR: + return make_string(value) + else: + return BuiltinADTType._enforce(self, value) + + +# WARNING: the name 'Unicode' is rebound at the end of file +class Unicode(AbstractString): + SELFTYPE_T = object() + Char = UniChar + + def _enforce(self, value): + TYPE = typeOf(value) + if TYPE == self.CHAR: + return make_unicode(value) + else: + return BuiltinADTType._enforce(self, value) + + + + # WARNING: the name 'StringBuilder' is rebound at the end of file class StringBuilder(BuiltinADTType): def __init__(self): @@ -837,6 +858,10 @@ assert isinstance(value, str) return _string(String, value) +def make_unicode(value): + assert isinstance(value, unicode) + return _string(Unicode, value) + def make_instance(INSTANCE): inst = _instance(INSTANCE) if STATICNESS: From rxe at codespeak.net Thu Nov 8 13:54:04 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 13:54:04 +0100 (CET) Subject: [pypy-svn] r48409 - pypy/dist/pypy/translator/llvm Message-ID: <20071108125404.8086E8179@code0.codespeak.net> Author: rxe Date: Thu Nov 8 13:54:04 2007 New Revision: 48409 Modified: pypy/dist/pypy/translator/llvm/database.py Log: use size_and_sign() for rffi lltype sizes. UniChar should be 4 bytes. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Nov 8 13:54:04 2007 @@ -351,7 +351,7 @@ lltype.Bool: "i1", lltype.SingleFloat: "float", lltype.Float: "double", - lltype.UniChar: "i16", + lltype.UniChar: "i32", lltype.Void: "void", lltype.UnsignedLongLong: "i64", lltype.SignedLongLong: "i64", @@ -401,18 +401,11 @@ if from_ not in self.reprs: self.reprs[from_] = self.repr_default - for k, v in [ - (rffi.SIGNEDCHAR, 'i18'), - (rffi.UCHAR, 'i18'), - (rffi.SHORT, 'i16'), - (rffi.USHORT, 'i16'), - (rffi.INT, 'i32'), - (rffi.UINT, 'i32'), - (rffi.LONG, self.types[lltype.Signed]), - (rffi.ULONG, self.types[lltype.Signed]), - (rffi.LONGLONG, 'i64'), - (rffi.ULONGLONG, 'i64')]: - update(k, v) + for tp in [rffi.SIGNEDCHAR, rffi.UCHAR, rffi.SHORT, + rffi.USHORT, rffi.INT, rffi.UINT, rffi.LONG, rffi.ULONG, + rffi.LONGLONG, rffi.ULONGLONG]: + bits = rffi.size_and_sign(tp)[0] * 8 + update(tp, 'i%s' % bits) def __getitem__(self, key): return self.types[key] From antocuni at codespeak.net Thu Nov 8 14:20:12 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Nov 2007 14:20:12 +0100 (CET) Subject: [pypy-svn] r48410 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108132012.73DA3817A@code0.codespeak.net> Author: antocuni Date: Thu Nov 8 14:20:11 2007 New Revision: 48410 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Log: remove outdated comment Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 14:20:11 2007 @@ -11,23 +11,6 @@ # conversion class StringRepr(AbstractStringRepr): - """ - Some comments about the state of ootype strings at the end of Tokyo sprint - - What was accomplished: - - The rstr module was split in an lltype and ootype version. - - There is the beginnings of a String type in ootype. - - The runtime representation of Strings is a subclass of the builtin str. - The idea is that this saves us from boilerplate code implementing the - builtin str methods. - - Nothing more was done because of lack of time and paralysis in the face - of too many problems. Among other things, to write any meaningful tests - we first need conversion from Chars to Strings (because - test_llinterp.interpret won't accept strings as arguments). We will need a - new low-level operation (convert_char_to_oostring or some such) for this. - """ - lowleveltype = ootype.String def __init__(self, *args): From fijal at codespeak.net Thu Nov 8 14:20:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 14:20:33 +0100 (CET) Subject: [pypy-svn] r48411 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071108132033.31538817A@code0.codespeak.net> Author: fijal Date: Thu Nov 8 14:20:32 2007 New Revision: 48411 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Shuffle tests around, so unichar tests are in test_runicode now, not in test_rstr. Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Thu Nov 8 14:20:32 2007 @@ -141,55 +141,6 @@ res = self.interpret(fn, [const('5'), 3]) assert res == 5551 - def test_unichar_const(self): - def fn(c): - return c - assert self.interpret(fn, [u'\u03b1']) == u'\u03b1' - - def test_unichar_eq(self): - def fn(c1, c2): - return c1 == c2 - assert self.interpret(fn, [u'\u03b1', u'\u03b1']) == True - assert self.interpret(fn, [u'\u03b1', u'\u03b2']) == False - - def test_unichar_ord(self): - def fn(c): - return ord(c) - assert self.interpret(fn, [u'\u03b1']) == ord(u'\u03b1') - - def test_unichar_hash(self): - def fn(c): - d = {c: 42} - return d[c] - assert self.interpret(fn, [u'\u03b1']) == 42 - - def test_convert_char_to_unichar(self): - def g(c): - return ord(c) - def fn(n): - if n < 0: - c = unichr(-n) - else: - c = chr(n) - return g(c) - assert self.interpret(fn, [65]) == 65 - assert self.interpret(fn, [-5555]) == 5555 - - def test_char_unichar_eq(self): - def fn(c1, c2): - return c1 == c2 - assert self.interpret(fn, [u'(', '(']) == True - assert self.interpret(fn, [u'\u1028', '(']) == False - assert self.interpret(fn, ['(', u'(']) == True - assert self.interpret(fn, ['(', u'\u1028']) == False - - def test_char_unichar_eq_2(self): - def fn(c1): - return c1 == 'X' - assert self.interpret(fn, [u'(']) == False - assert self.interpret(fn, [u'\u1058']) == False - assert self.interpret(fn, [u'X']) == True - def test_is_none(self): const = self.const def fn(i): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 14:20:32 2007 @@ -1,6 +1,6 @@ -from pypy.rpython.test.tool import LLRtypeMixin +from pypy.rpython.test.tool import LLRtypeMixin, OORtypeMixin from pypy.rpython.test.test_rstr import AbstractTestRstr import py @@ -40,6 +40,56 @@ const = unicode assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + + def test_unichar_const(self): + def fn(c): + return c + assert self.interpret(fn, [u'\u03b1']) == u'\u03b1' + + def test_unichar_eq(self): + def fn(c1, c2): + return c1 == c2 + assert self.interpret(fn, [u'\u03b1', u'\u03b1']) == True + assert self.interpret(fn, [u'\u03b1', u'\u03b2']) == False + + def test_unichar_ord(self): + def fn(c): + return ord(c) + assert self.interpret(fn, [u'\u03b1']) == ord(u'\u03b1') + + def test_unichar_hash(self): + def fn(c): + d = {c: 42} + return d[c] + assert self.interpret(fn, [u'\u03b1']) == 42 + + def test_convert_char_to_unichar(self): + def g(c): + return ord(c) + def fn(n): + if n < 0: + c = unichr(-n) + else: + c = chr(n) + return g(c) + assert self.interpret(fn, [65]) == 65 + assert self.interpret(fn, [-5555]) == 5555 + + def test_char_unichar_eq(self): + def fn(c1, c2): + return c1 == c2 + assert self.interpret(fn, [u'(', '(']) == True + assert self.interpret(fn, [u'\u1028', '(']) == False + assert self.interpret(fn, ['(', u'(']) == True + assert self.interpret(fn, ['(', u'\u1028']) == False + + def test_char_unichar_eq_2(self): + def fn(c1): + return c1 == 'X' + assert self.interpret(fn, [u'(']) == False + assert self.interpret(fn, [u'\u1058']) == False + assert self.interpret(fn, [u'X']) == True + def unsupported(self): py.test.skip("not supported") @@ -61,3 +111,5 @@ EMPTY_STRING_HASH = -1 ll_to_string = LLRtypeMixin.ll_to_unicode +#class TestOOtype(BaseTestRUnicode, OORtypeMixin): +# pass From rxe at codespeak.net Thu Nov 8 14:24:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 14:24:03 +0100 (CET) Subject: [pypy-svn] r48412 - pypy/dist/pypy/translator/llvm Message-ID: <20071108132403.4A8C0817D@code0.codespeak.net> Author: rxe Date: Thu Nov 8 14:24:02 2007 New Revision: 48412 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: add cast_ptr_to_int operation. size_and_sign() seems to be returning signed for char/unichar - special case for now. Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Nov 8 14:24:02 2007 @@ -141,7 +141,10 @@ self.shiftop(opr) elif op.opname.startswith('cast_') or op.opname.startswith('truncate_'): - self.cast_primitive(opr) + if op.opname == "cast_ptr_to_int": + self.cast_ptr_to_int(opr) + else: + self.cast_primitive(opr) else: meth = getattr(self, op.opname, None) if not meth: @@ -219,10 +222,11 @@ from_lltype = opr.op.args[0].concretetype def issigned(ct): - if ct is lltype.Bool: + # XXX why does size_and_sign() think lltype.Char, lltype.UniChar are signed?? + if ct in [lltype.Bool, lltype.Char, lltype.UniChar]: return False from pypy.rpython.lltypesystem.rffi import size_and_sign - return size_and_sign(ct)[1] + return not size_and_sign(ct)[1] casttype = "bitcast" @@ -270,6 +274,10 @@ opr.argrefs[0], opr.rettype, casttype) same_as = cast_primitive + def cast_ptr_to_int(self, opr): + self.codewriter.cast(opr.retref, opr.argtypes[0], + opr.argrefs[0], opr.rettype, 'ptrtoint') + def int_is_true(self, opr): self.codewriter.binaryop("icmp ne", opr.retref, opr.argtypes[0], opr.argrefs[0], "0") From rxe at codespeak.net Thu Nov 8 14:25:25 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 8 Nov 2007 14:25:25 +0100 (CET) Subject: [pypy-svn] r48413 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071108132525.427148186@code0.codespeak.net> Author: rxe Date: Thu Nov 8 14:25:24 2007 New Revision: 48413 Modified: pypy/dist/pypy/translator/llvm/test/test_exception.py pypy/dist/pypy/translator/llvm/test/test_rlist.py pypy/dist/pypy/translator/llvm/test/test_rstr.py Log: these tests need some tweaking to work with genllvm Modified: pypy/dist/pypy/translator/llvm/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_exception.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_exception.py Thu Nov 8 14:25:24 2007 @@ -7,7 +7,11 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMException(LLVMTest, BaseTestException): - pass + def test_raise_and_catch_other(self): + py.test.skip('XXX special case me') + + def test_raise_prebuilt_and_catch_other(self): + py.test.skip('XXX special case me') class TestException(Exception): pass Modified: pypy/dist/pypy/translator/llvm/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rlist.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rlist.py Thu Nov 8 14:25:24 2007 @@ -3,4 +3,5 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMList(LLVMTest, BaseTestRlist): - pass + def test_insert_bug(self): + py.test.skip('XXX special case me') Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rstr.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rstr.py Thu Nov 8 14:25:24 2007 @@ -3,4 +3,13 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMStr(LLVMTest, BaseTestRstr): - pass + def test_int(self): + py.test.skip('XXX special case me') + + def test_float(self): + py.test.skip('XXX special case me') + + def test_hash(self): + py.test.skip('XXX special case me') + + From fijal at codespeak.net Thu Nov 8 14:31:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 14:31:36 +0100 (CET) Subject: [pypy-svn] r48414 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071108133136.A9FF8818A@code0.codespeak.net> Author: fijal Date: Thu Nov 8 14:31:36 2007 New Revision: 48414 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: This is no longer needed I think Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 14:31:36 2007 @@ -38,8 +38,13 @@ # a copy, because llinterp caches functions const = unicode - assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + def test_str_unicode_const(self): + def f(): + return str(u'xxx') + + assert self.ll_to_string def test_unichar_const(self): def fn(c): @@ -109,7 +114,6 @@ class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 - ll_to_string = LLRtypeMixin.ll_to_unicode #class TestOOtype(BaseTestRUnicode, OORtypeMixin): # pass From fijal at codespeak.net Thu Nov 8 15:39:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 15:39:34 +0100 (CET) Subject: [pypy-svn] r48415 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071108143934.4C2BB8162@code0.codespeak.net> Author: fijal Date: Thu Nov 8 15:39:32 2007 New Revision: 48415 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Test str(unicodeconstant) is constant propagated and works out of the box. Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 15:39:32 2007 @@ -44,7 +44,7 @@ def f(): return str(u'xxx') - assert self.ll_to_string + assert self.ll_to_string(self.interpret(f, [])) == 'xxx' def test_unichar_const(self): def fn(c): From fijal at codespeak.net Thu Nov 8 15:52:47 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 15:52:47 +0100 (CET) Subject: [pypy-svn] r48416 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem test Message-ID: <20071108145247.CC725814A@code0.codespeak.net> Author: fijal Date: Thu Nov 8 15:52:44 2007 New Revision: 48416 Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: ll_str for unicode Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Thu Nov 8 15:52:44 2007 @@ -119,6 +119,18 @@ self.ll = LLHelpers self.malloc = mallocunicode + def ll_str(self, s): + # XXX crazy that this is here, but I don't want to break + # rmodel logic + lgt = len(s.chars) + result = mallocstr(lgt) + for i in range(lgt): + c = s.chars[i] + if ord(c) > 127: + raise ValueError("character not in ascii range") + result.chars[i] = cast_primitive(Char, c) + return result + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Thu Nov 8 15:52:44 2007 @@ -264,7 +264,6 @@ def ll_str(self, s): return s - class __extend__(pairtype(AbstractStringRepr, Repr)): def rtype_mod((r_str, _), hop): # for the case where the 2nd argument is a tuple, see the Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 15:52:44 2007 @@ -46,6 +46,13 @@ assert self.ll_to_string(self.interpret(f, [])) == 'xxx' + def test_str_unicode_nonconst(self): + def f(x): + y = u'xxx' + unichr(x) + return str(y) + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def test_unichar_const(self): def fn(c): return c From fijal at codespeak.net Thu Nov 8 16:02:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 16:02:00 +0100 (CET) Subject: [pypy-svn] r48417 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071108150200.A0C79818A@code0.codespeak.net> Author: fijal Date: Thu Nov 8 16:02:00 2007 New Revision: 48417 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: One more case... Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 16:02:00 2007 @@ -52,6 +52,7 @@ return str(y) assert self.ll_to_string(self.interpret(f, [38])) == f(38) + self.interpret_raises(ValueError, f, [1234]) def test_unichar_const(self): def fn(c): From fijal at codespeak.net Thu Nov 8 16:08:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 16:08:03 +0100 (CET) Subject: [pypy-svn] r48418 - in pypy/branch/pypy-rpython-unicode: annotation rpython rpython/test Message-ID: <20071108150803.BFC228179@code0.codespeak.net> Author: fijal Date: Thu Nov 8 16:08:03 2007 New Revision: 48418 Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: unicode.decode('ascii') Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Thu Nov 8 16:08:03 2007 @@ -466,6 +466,15 @@ def method_replace(str, s1, s2): return str.basestringclass() +class __extend__(SomeUnicodeString): + def method_decode(uni, s_enc): + if not s_enc.is_constant(): + raise TypeError("Non-constant decoding not supported") + enc = s_enc.const + if enc != 'ascii': + raise TypeError("Decoding %s not supported for unicode" % (enc,)) + return SomeString() + class __extend__(SomeString): def method_upper(str): return SomeString() Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Thu Nov 8 16:08:03 2007 @@ -264,6 +264,11 @@ def ll_str(self, s): return s +class __extend__(AbstractUnicodeRepr): + def rtype_method_decode(self, hop): + v_self = hop.inputarg(self, 0) + return hop.gendirectcall(self.ll_str, v_self) + class __extend__(pairtype(AbstractStringRepr, Repr)): def rtype_mod((r_str, _), hop): # for the case where the 2nd argument is a tuple, see the Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 16:08:03 2007 @@ -54,6 +54,13 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) self.interpret_raises(ValueError, f, [1234]) + def test_unicode_decode(self): + def f(x): + y = u'xxx' + return (y + unichr(x)).decode('ascii') + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def test_unichar_const(self): def fn(c): return c From lac at codespeak.net Thu Nov 8 17:39:34 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Thu, 8 Nov 2007 17:39:34 +0100 (CET) Subject: [pypy-svn] r48422 - pypy/extradoc/talk/roadshow-google Message-ID: <20071108163934.6A94D818A@code0.codespeak.net> Author: lac Date: Thu Nov 8 17:39:33 2007 New Revision: 48422 Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp Log: Fixed details in slides. Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp ============================================================================== Binary files. No diff available. From pedronis at codespeak.net Thu Nov 8 17:47:46 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 8 Nov 2007 17:47:46 +0100 (CET) Subject: [pypy-svn] r48423 - in pypy/extradoc/talk/roadshow-google: . ui Message-ID: <20071108164746.A44938191@code0.codespeak.net> Author: pedronis Date: Thu Nov 8 17:47:46 2007 New Revision: 48423 Added: pypy/extradoc/talk/roadshow-google/motivation.txt - copied, changed from r48343, pypy/extradoc/talk/roadshow-ibm/talk.txt pypy/extradoc/talk/roadshow-google/overview1.png - copied unchanged from r48343, pypy/extradoc/talk/roadshow-ibm/overview1.png pypy/extradoc/talk/roadshow-google/overview2.png - copied unchanged from r48343, pypy/extradoc/talk/roadshow-ibm/overview2.png pypy/extradoc/talk/roadshow-google/ui/ - copied from r48343, pypy/extradoc/talk/roadshow-ibm/ui/ Log: motivation slides From arigo at codespeak.net Thu Nov 8 18:01:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Nov 2007 18:01:50 +0100 (CET) Subject: [pypy-svn] r48424 - pypy/extradoc/talk/roadshow-google Message-ID: <20071108170150.376EB8194@code0.codespeak.net> Author: arigo Date: Thu Nov 8 18:01:49 2007 New Revision: 48424 Modified: pypy/extradoc/talk/roadshow-google/overview2.png (props changed) Log: Remove svn:executable. From antocuni at codespeak.net Thu Nov 8 18:39:24 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Nov 2007 18:39:24 +0100 (CET) Subject: [pypy-svn] r48425 - in pypy/branch/pypy-rpython-unicode/rpython: ootypesystem ootypesystem/test test Message-ID: <20071108173924.BB6D5813F@code0.codespeak.net> Author: antocuni Date: Thu Nov 8 18:39:23 2007 New Revision: 48425 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py pypy/branch/pypy-rpython-unicode/rpython/test/tool.py Log: first step to add Unicode support to ootype; it's not completed because of a blocker I could not solve yet; see the failing test in ootypesystem/test/test_ooann.py Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 18:39:23 2007 @@ -382,7 +382,7 @@ # WARNING: the name 'Unicode' is rebound at the end of file class Unicode(AbstractString): SELFTYPE_T = object() - Char = UniChar + CHAR = UniChar def _enforce(self, value): TYPE = typeOf(value) @@ -1074,6 +1074,14 @@ def __cmp__(self, other): return cmp(self._str, other._str) + def make_string(self, value): + if self._TYPE is String: + return make_string(value) + elif self._TYPE is Unicode: + return make_unicode(value) + else: + assert False, 'Unknown type %s' % self._TYPE + def ll_stritem_nonneg(self, i): # NOT_RPYTHON s = self._str @@ -1086,7 +1094,7 @@ def ll_strconcat(self, s): # NOT_RPYTHON - return make_string(self._str + s._str) + return self.make_string(self._str + s._str) def ll_streq(self, s): # NOT_RPYTON @@ -1135,24 +1143,24 @@ s = s.lstrip(ch) if right: s = s.rstrip(ch) - return make_string(s) + return self.make_string(s) def ll_upper(self): # NOT_RPYTHON - return make_string(self._str.upper()) + return self.make_string(self._str.upper()) def ll_lower(self): # NOT_RPYTHON - return make_string(self._str.lower()) + return self.make_string(self._str.lower()) def ll_substring(self, start, count): # NOT_RPYTHON - return make_string(self._str[start:start+count]) + return self.make_string(self._str[start:start+count]) def ll_split_chr(self, ch): # NOT_RPYTHON res = _list(List(String)) - res._list = [make_string(s) for s in self._str.split(ch)] + res._list = [self.make_string(s) for s in self._str.split(ch)] return res def ll_contains(self, ch): @@ -1161,7 +1169,7 @@ def ll_replace_chr_chr(self, ch1, ch2): # NOT_RPYTHON - return make_string(self._str.replace(ch1, ch2)) + return self.make_string(self._str.replace(ch1, ch2)) class _null_string(_null_mixin(_string), _string): def __init__(self, STRING): @@ -1573,6 +1581,7 @@ ROOT = Instance('Root', None, _is_root=True) String = String() +Unicode = Unicode() StringBuilder = StringBuilder() WeakReference = WeakReference() dead_wref = new(WeakReference) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 18:39:23 2007 @@ -10,8 +10,7 @@ # String and a Char directly without passing to Char-->String # conversion -class StringRepr(AbstractStringRepr): - lowleveltype = ootype.String +class AbstractOOStringRepr(AbstractStringRepr): def __init__(self, *args): AbstractStringRepr.__init__(self, *args) @@ -19,13 +18,16 @@ def convert_const(self, value): if value is None: - return ootype.String._null - if not isinstance(value, str): + return self.lowleveltype._null + if not isinstance(value, self.basetype): raise TyperError("not a str: %r" % (value,)) - return ootype.make_string(value) + return self.make_string(value) + + def make_string(self, value): + raise NotImplementedError def make_iterator_repr(self): - return string_iterator_repr + return self.string_iterator_repr def _list_length_items(self, hop, v_lst, LIST): # ootypesystem list has a different interface that @@ -36,6 +38,21 @@ return c_length, v_lst +class StringRepr(AbstractOOStringRepr): + lowleveltype = ootype.String + basetype = str + + def make_string(self, value): + return ootype.make_string(value) + +class UnicodeRepr(AbstractOOStringRepr): + lowleveltype = ootype.Unicode + basetype = basestring + + def make_string(self, value): + return ootype.make_unicode(value) + + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char @@ -277,14 +294,19 @@ del add_helpers do_stringformat = LLHelpers.do_stringformat -string_repr = StringRepr() char_repr = CharRepr() unichar_repr = UniCharRepr() char_repr.ll = LLHelpers unichar_repr.ll = LLHelpers -emptystr = string_repr.convert_const("") + +string_repr = StringRepr() StringRepr.repr = string_repr StringRepr.char_repr = char_repr +emptystr = string_repr.convert_const("") +unicode_repr = UnicodeRepr() +UnicodeRepr.repr = unicode_repr +UnicodeRepr.char_repr = unichar_repr + class StringIteratorRepr(AbstractStringIteratorRepr): lowleveltype = ootype.Record({'string': string_repr.lowleveltype, @@ -294,8 +316,22 @@ self.ll_striter = ll_striter self.ll_strnext = ll_strnext +class UnicodeIteratorRepr(AbstractStringIteratorRepr): + lowleveltype = ootype.Record({'string': unicode_repr.lowleveltype, + 'index': ootype.Signed}) + + def __init__(self): + self.ll_striter = ll_unicodeiter + self.ll_strnext = ll_strnext + def ll_striter(string): - iter = ootype.new(string_iterator_repr.lowleveltype) + iter = ootype.new(string_repr.string_iterator_repr.lowleveltype) + iter.string = string + iter.index = 0 + return iter + +def ll_unicodeiter(string): + iter = ootype.new(unicode_repr.string_iterator_repr.lowleveltype) iter.string = string iter.index = 0 return iter @@ -308,8 +344,9 @@ iter.index = index + 1 return string.ll_stritem_nonneg(index) -string_iterator_repr = StringIteratorRepr() +StringRepr.string_iterator_repr = StringIteratorRepr() +UnicodeRepr.string_iterator_repr = UnicodeIteratorRepr() # these should be in rclass, but circular imports prevent (also it's # not that insane that a string constant is built in this file). Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py Thu Nov 8 18:39:23 2007 @@ -324,3 +324,18 @@ return c.foo(c) a = RPythonAnnotator() py.test.raises(TypeError, a.build_types, f, []) + +def test_unicode_iterator(): + py.test.skip("That's a nonsense!") + from pypy.rpython.ootypesystem import rstr + ITER = rstr.UnicodeRepr.string_iterator_repr.lowleveltype + + def fn(): + it = new(ITER) + return it.string + a = RPythonAnnotator() + res = a.build_types(fn, []) + + assert ITER._field_type("string") is Unicode + assert isinstance(res, annmodel.SomeOOInstance) + assert res.ootype is Unicode Modified: pypy/branch/pypy-rpython-unicode/rpython/test/tool.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/tool.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/tool.py Thu Nov 8 18:39:23 2007 @@ -82,6 +82,8 @@ def ll_to_string(self, s): return s._str + ll_to_unicode = ll_to_string + def string_to_ll(self, s): from pypy.rpython.module.support import OOSupport return OOSupport.to_rstr(s) From fijal at codespeak.net Thu Nov 8 18:43:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 18:43:51 +0100 (CET) Subject: [pypy-svn] r48426 - pypy/branch/pypy-rpython-unicode/objspace/flow/test Message-ID: <20071108174351.C46DF815C@code0.codespeak.net> Author: fijal Date: Thu Nov 8 18:43:51 2007 New Revision: 48426 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: Fix test Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Thu Nov 8 18:43:51 2007 @@ -720,7 +720,7 @@ def test_unicode_constfold(self): def myfunc(): - return unicode(1234) + return unicode("1234") graph = self.codetest(myfunc) assert graph.startblock.exits[0].target is graph.returnblock From fijal at codespeak.net Thu Nov 8 19:04:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 19:04:23 +0100 (CET) Subject: [pypy-svn] r48427 - pypy/branch/pypy-rpython-unicode/objspace/std/test Message-ID: <20071108180423.9C6258136@code0.codespeak.net> Author: fijal Date: Thu Nov 8 19:04:22 2007 New Revision: 48427 Modified: pypy/branch/pypy-rpython-unicode/objspace/std/test/test_unicodeobject.py Log: Skipped test that should work Modified: pypy/branch/pypy-rpython-unicode/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/std/test/test_unicodeobject.py Thu Nov 8 19:04:22 2007 @@ -264,3 +264,17 @@ assert unicode(None) == u'None' assert unicode(123) == u'123' assert unicode([2, 3]) == u'[2, 3]' + + def test_call_unicode(self): + skip("does not work") + class X: + def __unicode__(self): + return u'x' + + try: + unicode(X(), 'ascii') + except TypeError, t: + assert 'need string or buffer' in str(t) + else: + raise Exception("DID NOT RAISE") + From antocuni at codespeak.net Thu Nov 8 20:16:08 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Nov 2007 20:16:08 +0100 (CET) Subject: [pypy-svn] r48428 - in pypy/branch/pypy-rpython-unicode/rpython/ootypesystem: . test Message-ID: <20071108191608.6D49E8142@code0.codespeak.net> Author: antocuni Date: Thu Nov 8 20:16:06 2007 New Revision: 48428 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py Log: fix this test; the problem was the _defl method of AbstractString, which shouldn't have been there. Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 20:16:06 2007 @@ -352,10 +352,6 @@ }) self._setup_methods(generic_types) - - # TODO: should it return _null or ''? - def _defl(self): - return make_string("") def _example(self): return self._defl() @@ -370,6 +366,10 @@ SELFTYPE_T = object() CHAR = Char + # TODO: should it return _null or ''? + def _defl(self): + return make_string('') + def _enforce(self, value): # XXX share this with Unicode? TYPE = typeOf(value) @@ -384,6 +384,10 @@ SELFTYPE_T = object() CHAR = UniChar + # TODO: should it return _null or ''? + def _defl(self): + return make_unicode(u'') + def _enforce(self, value): TYPE = typeOf(value) if TYPE == self.CHAR: Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py Thu Nov 8 20:16:06 2007 @@ -326,7 +326,6 @@ py.test.raises(TypeError, a.build_types, f, []) def test_unicode_iterator(): - py.test.skip("That's a nonsense!") from pypy.rpython.ootypesystem import rstr ITER = rstr.UnicodeRepr.string_iterator_repr.lowleveltype From antocuni at codespeak.net Thu Nov 8 20:43:50 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Nov 2007 20:43:50 +0100 (CET) Subject: [pypy-svn] r48433 - in pypy/branch/pypy-rpython-unicode/rpython: . ootypesystem Message-ID: <20071108194350.131868141@code0.codespeak.net> Author: antocuni Date: Thu Nov 8 20:43:50 2007 New Revision: 48433 Modified: pypy/branch/pypy-rpython-unicode/rpython/llinterp.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Log: more work towards ootype unicode Modified: pypy/branch/pypy-rpython-unicode/rpython/llinterp.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/llinterp.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/llinterp.py Thu Nov 8 20:43:50 2007 @@ -1079,6 +1079,9 @@ def op_oostring(self, obj, base): return ootype.oostring(obj, base) + def op_oounicode(self, obj, base): + return ootype.oounicode(obj, base) + def op_ooparse_int(self, s, base): try: return ootype.ooparse_int(s, base) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py Thu Nov 8 20:43:50 2007 @@ -24,6 +24,19 @@ vlist = hop.inputargs(hop.args_r[0], ootype.Signed) return hop.genop('oostring', vlist, resulttype = ootype.String) +class Entry_oounicode(ExtRegistryEntry): + _about_ = ootype.oounicode + + def compute_result_annotation(self, obj_s, base_s): + assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) + assert isinstance(base_s, annmodel.SomeInteger) + return annmodel.SomeOOInstance(ootype.Unicode) + + def specialize_call(self, hop): + assert isinstance(hop.args_s[0],annmodel.SomeUnicodeCodePoint) + vlist = hop.inputargs(hop.args_r[0], ootype.Signed) + return hop.genop('oounicode', vlist, resulttype = ootype.Unicode) + class Entry_ootype_string(ExtRegistryEntry): _type_ = ootype._string Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 20:43:50 2007 @@ -1557,6 +1557,17 @@ obj = '<%s object>' % obj._inst._TYPE._name return make_string(str(obj)) +def oounicode(obj, base): + """ + Convert an unichar into an unicode string. + + base must be -1, for consistency with oostring. + """ + assert base == -1 + assert isinstance(obj, unicode) + assert len(obj) == 1 + return make_unicode(obj) + def ooparse_int(s, base): return int(s._str, base) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 20:43:50 2007 @@ -1,7 +1,8 @@ +from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rpython.rstr import AbstractStringRepr,AbstractCharRepr,\ AbstractUniCharRepr, AbstractStringIteratorRepr,\ - AbstractLLHelpers + AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.rmodel import IntegerRepr from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar from pypy.rpython.ootypesystem import ootype @@ -45,25 +46,35 @@ def make_string(self, value): return ootype.make_string(value) -class UnicodeRepr(AbstractOOStringRepr): +class UnicodeRepr(AbstractOOStringRepr, AbstractUnicodeRepr): lowleveltype = ootype.Unicode basetype = basestring def make_string(self, value): return ootype.make_unicode(value) - class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char -class UniCharRepr(AbstractUniCharRepr): +class UniCharRepr(AbstractUniCharRepr, UnicodeRepr): lowleveltype = UniChar + +class __extend__(pairtype(UniCharRepr, UnicodeRepr)): + def convert_from_to((r_from, r_to), v, llops): + rstr = llops.rtyper.type_system.rstr + if r_from == unichar_repr and r_to == unicode_repr: + return llops.gendirectcall(r_from.ll.ll_unichr2unicode, v) + return NotImplemented + class LLHelpers(AbstractLLHelpers): def ll_chr2str(ch): return ootype.oostring(ch, -1) + def ll_unichr2unicode(ch): + return ootype.oounicode(ch, -1) + def ll_strhash(s): return ootype.oohash(s) From cfbolz at codespeak.net Thu Nov 8 20:50:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 20:50:14 +0100 (CET) Subject: [pypy-svn] r48435 - pypy/branch/pypy-rpython-unicode/translator/test Message-ID: <20071108195014.05D488142@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 20:50:14 2007 New Revision: 48435 Modified: pypy/branch/pypy-rpython-unicode/translator/test/test_geninterp.py Log: test for geninterp unicode behaviour Modified: pypy/branch/pypy-rpython-unicode/translator/test/test_geninterp.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/translator/test/test_geninterp.py (original) +++ pypy/branch/pypy-rpython-unicode/translator/test/test_geninterp.py Thu Nov 8 20:50:14 2007 @@ -44,6 +44,9 @@ def import_sys_func(): import sys return sys.__name__ + + def unicode_test(x): + return unicode(x, 'ascii') """ def setup_class(cls): @@ -275,3 +278,9 @@ fn = self.build_interpfunc(snippet.t_neg_long) result = fn() assert result == -132L and type(result) is long + + def test_unicode_with_encoding(self): + fn = self.build_interpfunc(snippet.unicode_test) + result = fn("abc") + assert result == u"abc" and type(result) is unicode + From fijal at codespeak.net Thu Nov 8 20:55:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 20:55:04 +0100 (CET) Subject: [pypy-svn] r48439 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108195504.369328146@code0.codespeak.net> Author: fijal Date: Thu Nov 8 20:55:03 2007 New Revision: 48439 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Log: that was easy :) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 20:55:03 2007 @@ -72,6 +72,9 @@ def ll_chr2str(ch): return ootype.oostring(ch, -1) + def ll_str2unicode(s): + return s + def ll_unichr2unicode(ch): return ootype.oounicode(ch, -1) From fijal at codespeak.net Thu Nov 8 21:19:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 21:19:53 +0100 (CET) Subject: [pypy-svn] r48444 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108201953.BB3BE8141@code0.codespeak.net> Author: fijal Date: Thu Nov 8 21:19:53 2007 New Revision: 48444 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Log: Squash few failures, now 7 out of 13 tests are failing Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 21:19:53 2007 @@ -400,13 +400,13 @@ # WARNING: the name 'StringBuilder' is rebound at the end of file class StringBuilder(BuiltinADTType): - def __init__(self): + def __init__(self, STRINGTP, CHARTP): self._null = _null_string_builder(self) self._GENERIC_METHODS = frozendict({ "ll_allocate": Meth([Signed], Void), - "ll_append_char": Meth([Char], Void), - "ll_append": Meth([String], Void), - "ll_build": Meth([], String), + "ll_append_char": Meth([CHARTP], Void), + "ll_append": Meth([STRINGTP], Void), + "ll_build": Meth([], STRINGTP), }) self._setup_methods({}) @@ -1191,7 +1191,7 @@ # do nothing def ll_append_char(self, ch): - assert isinstance(ch, str) and len(ch) == 1 + assert isinstance(ch, basestring) and len(ch) == 1 self._buf.append(ch) def ll_append(self, s): @@ -1199,7 +1199,10 @@ self._buf.append(s._str) def ll_build(self): - return make_string(''.join(self._buf)) + if self._TYPE is StringBuilder: + return make_string(''.join(self._buf)) + else: + return make_unicode(u''.join(self._buf)) class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): @@ -1597,6 +1600,9 @@ ROOT = Instance('Root', None, _is_root=True) String = String() Unicode = Unicode() -StringBuilder = StringBuilder() +UnicodeBuilder = StringBuilder(Unicode, UniChar) +StringBuilder = StringBuilder(String, Char) +String.builder = StringBuilder +Unicode.builder = UnicodeBuilder WeakReference = WeakReference() dead_wref = new(WeakReference) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 21:19:53 2007 @@ -4,14 +4,15 @@ AbstractUniCharRepr, AbstractStringIteratorRepr,\ AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.rmodel import IntegerRepr -from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar +from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar, typeOf from pypy.rpython.ootypesystem import ootype +from pypy.rpython.rmodel import Repr # TODO: investigate if it's possible and it's worth to concatenate a # String and a Char directly without passing to Char-->String # conversion -class AbstractOOStringRepr(AbstractStringRepr): +class BaseOOStringRepr(Repr): def __init__(self, *args): AbstractStringRepr.__init__(self, *args) @@ -39,14 +40,14 @@ return c_length, v_lst -class StringRepr(AbstractOOStringRepr): +class StringRepr(BaseOOStringRepr, AbstractStringRepr): lowleveltype = ootype.String basetype = str def make_string(self, value): return ootype.make_string(value) -class UnicodeRepr(AbstractOOStringRepr, AbstractUnicodeRepr): +class UnicodeRepr(BaseOOStringRepr, AbstractUnicodeRepr): lowleveltype = ootype.Unicode basetype = basestring @@ -72,8 +73,8 @@ def ll_chr2str(ch): return ootype.oostring(ch, -1) - def ll_str2unicode(s): - return s + #def ll_str2unicode(s): + # return ootype.oounicode(s, -1) def ll_unichr2unicode(ch): return ootype.oounicode(ch, -1) @@ -87,7 +88,10 @@ def ll_char_mul(ch, times): if times < 0: times = 0 - buf = ootype.new(ootype.StringBuilder) + if typeOf(ch) == Char: + buf = ootype.new(ootype.StringBuilder) + else: + buf = ootype.new(ootype.UnicodeBuilder) buf.ll_allocate(times) i = 0 while i Author: fijal Date: Thu Nov 8 22:07:25 2007 New Revision: 48447 Modified: pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Log: Remove unichr() and unicode() from descroperatoin Modified: pypy/branch/pypy-rpython-unicode/objspace/descroperation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/descroperation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Thu Nov 8 22:07:25 2007 @@ -291,12 +291,6 @@ if w_del is not None: space.get_and_call_function(w_del, w_obj) - def unichr(space, w_obj): - raise NotImplementedError - - def unicode(space, w_obj): - raise NotImplementedError - def cmp(space, w_v, w_w): if space.is_w(w_v, w_w): @@ -598,7 +592,7 @@ setattr(DescrOperation,_name,_impl_maker(_symbol,_specialnames)) elif _name not in ['is_', 'id','type','issubtype', # not really to be defined in DescrOperation - 'ord']: + 'ord', 'unichr', 'unicode']: raise Exception, "missing def for operation %s" % _name From fijal at codespeak.net Thu Nov 8 22:14:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 22:14:20 +0100 (CET) Subject: [pypy-svn] r48448 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108211420.4FE29816A@code0.codespeak.net> Author: fijal Date: Thu Nov 8 22:14:18 2007 New Revision: 48448 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Log: Create proper str2unicode Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 22:14:18 2007 @@ -73,8 +73,13 @@ def ll_chr2str(ch): return ootype.oostring(ch, -1) - #def ll_str2unicode(s): - # return ootype.oounicode(s, -1) + def ll_str2unicode(s): + res = ootype.new(ootype.UnicodeBuilder) + lgt = s.ll_strlen() + res.ll_allocate(lgt) + for i in range(lgt): + res.ll_append_char(s.ll_stritem_nonneg(i)) + return res.ll_build() def ll_unichr2unicode(ch): return ootype.oounicode(ch, -1) From fijal at codespeak.net Thu Nov 8 22:34:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 22:34:55 +0100 (CET) Subject: [pypy-svn] r48449 - in pypy/branch/pypy-rpython-unicode/objspace/flow: . test Message-ID: <20071108213455.5E4828120@code0.codespeak.net> Author: fijal Date: Thu Nov 8 22:34:54 2007 New Revision: 48449 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: Remove unichr as spaceop Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py Thu Nov 8 22:34:54 2007 @@ -33,6 +33,8 @@ args_w = args_w + [Constant(None)] elif opname == 'getattr' and len(args_w) == 3: return space.do_operation('simple_call', Constant(getattr), *args_w) + elif opname == 'unicode' and len(args_w) == 2: + return space.do_operation('simple_call', Constant(unicode), *args_w) else: raise Exception, "should call %r with exactly %d arguments" % ( fn, Arity[opname]) Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Thu Nov 8 22:34:54 2007 @@ -712,12 +712,6 @@ return x[s] graph = self.codetest(myfunc) - def test_unichr_constfold(self): - def myfunc(): - return unichr(1234) - graph = self.codetest(myfunc) - assert graph.startblock.exits[0].target is graph.returnblock - def test_unicode_constfold(self): def myfunc(): return unicode("1234") From fijal at codespeak.net Thu Nov 8 22:38:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 22:38:01 +0100 (CET) Subject: [pypy-svn] r48450 - in pypy/branch/pypy-rpython-unicode: annotation objspace/flow Message-ID: <20071108213801.35FFC8146@code0.codespeak.net> Author: fijal Date: Thu Nov 8 22:38:00 2007 New Revision: 48450 Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Log: Remove unichr as spaceop Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Thu Nov 8 22:38:00 2007 @@ -205,11 +205,6 @@ def invert(self): return SomeInteger(knowntype=self.knowntype) - - def unichr(obj): - getbookkeeper().count('unichr', obj) - return SomeUnicodeCodePoint() - invert.can_only_throw = [] def pos(self): Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Thu Nov 8 22:38:00 2007 @@ -149,7 +149,6 @@ ('issubtype', issubclass), ('repr', repr), ('str', str), - ('unichr', unichr), ('unicode', unicode), ('len', len), ('hash', hash), From cfbolz at codespeak.net Thu Nov 8 22:38:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 22:38:12 +0100 (CET) Subject: [pypy-svn] r48451 - pypy/branch/pypy-rpython-unicode/interpreter Message-ID: <20071108213812.104B2814B@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 22:38:11 2007 New Revision: 48451 Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py Log: remove unicode and unichr from the list of base space operations Modified: pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py (original) +++ pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py Thu Nov 8 22:38:11 2007 @@ -910,8 +910,6 @@ ('issubtype', 'issubtype', 2, []), # not for old-style classes ('repr', 'repr', 1, ['__repr__']), ('str', 'str', 1, ['__str__']), - ('unichr', 'unichr', 1, []), - ('unicode', 'unicode', 1, ['__unicode__']), ('len', 'len', 1, ['__len__']), ('hash', 'hash', 1, ['__hash__']), ('getattr', 'getattr', 2, ['__getattribute__']), From fijal at codespeak.net Thu Nov 8 22:39:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 22:39:48 +0100 (CET) Subject: [pypy-svn] r48452 - in pypy/branch/pypy-rpython-unicode/objspace/flow: . test Message-ID: <20071108213948.E2B298146@code0.codespeak.net> Author: fijal Date: Thu Nov 8 22:39:48 2007 New Revision: 48452 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: Revert 48449 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py Thu Nov 8 22:39:48 2007 @@ -33,8 +33,6 @@ args_w = args_w + [Constant(None)] elif opname == 'getattr' and len(args_w) == 3: return space.do_operation('simple_call', Constant(getattr), *args_w) - elif opname == 'unicode' and len(args_w) == 2: - return space.do_operation('simple_call', Constant(unicode), *args_w) else: raise Exception, "should call %r with exactly %d arguments" % ( fn, Arity[opname]) Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Thu Nov 8 22:39:48 2007 @@ -712,6 +712,13 @@ return x[s] graph = self.codetest(myfunc) + def test_unichr_constfold(self): + py.test.skip("not working") + def myfunc(): + return unichr(1234) + graph = self.codetest(myfunc) + assert graph.startblock.exits[0].target is graph.returnblock + def test_unicode_constfold(self): def myfunc(): return unicode("1234") From cfbolz at codespeak.net Thu Nov 8 22:46:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 22:46:02 +0100 (CET) Subject: [pypy-svn] r48453 - pypy/branch/pypy-rpython-unicode/objspace/flow Message-ID: <20071108214602.3BF108186@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 22:46:01 2007 New Revision: 48453 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Log: remove this too Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Thu Nov 8 22:46:01 2007 @@ -149,7 +149,6 @@ ('issubtype', issubclass), ('repr', repr), ('str', str), - ('unicode', unicode), ('len', len), ('hash', hash), ('getattr', getattr), From cfbolz at codespeak.net Thu Nov 8 22:47:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 22:47:04 +0100 (CET) Subject: [pypy-svn] r48454 - pypy/branch/pypy-rpython-unicode/objspace/flow/test Message-ID: <20071108214704.6F0A3818C@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 22:47:04 2007 New Revision: 48454 Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Log: skip this for now Modified: pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Thu Nov 8 22:47:04 2007 @@ -720,6 +720,7 @@ assert graph.startblock.exits[0].target is graph.returnblock def test_unicode_constfold(self): + py.test.skip("not working for now") def myfunc(): return unicode("1234") graph = self.codetest(myfunc) From cfbolz at codespeak.net Thu Nov 8 22:50:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 22:50:47 +0100 (CET) Subject: [pypy-svn] r48455 - pypy/branch/pypy-rpython-unicode/rpython Message-ID: <20071108215047.B71AB813B@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 22:50:47 2007 New Revision: 48455 Modified: pypy/branch/pypy-rpython-unicode/rpython/rbuiltin.py Log: at least this is needed Modified: pypy/branch/pypy-rpython-unicode/rpython/rbuiltin.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rbuiltin.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rbuiltin.py Thu Nov 8 22:50:47 2007 @@ -218,6 +218,9 @@ assert hop.nb_args == 1 return hop.args_r[0].rtype_unichr(hop) +def rtype_builtin_unicode(hop): + return hop.args_r[0].rtype_unicode(hop) + def rtype_builtin_list(hop): return hop.args_r[0].rtype_bltn_list(hop) From cfbolz at codespeak.net Thu Nov 8 22:55:54 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Nov 2007 22:55:54 +0100 (CET) Subject: [pypy-svn] r48456 - pypy/branch/pypy-rpython-unicode/rpython Message-ID: <20071108215554.C9948813B@code0.codespeak.net> Author: cfbolz Date: Thu Nov 8 22:55:54 2007 New Revision: 48456 Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py Log: more tests passing Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Thu Nov 8 22:55:54 2007 @@ -249,6 +249,8 @@ return hop.gendirectcall(self.ll.ll_int, v_str, v_base) def rtype_unicode(self, hop): + if hop.args_s[0].is_constant(): + return hop.inputconst(hop.r_result, hop.s_result.const) repr = hop.args_r[0].repr v_str = hop.inputarg(repr, 0) hop.exception_cannot_occur() From fijal at codespeak.net Thu Nov 8 23:24:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 23:24:19 +0100 (CET) Subject: [pypy-svn] r48457 - pypy/branch/pypy-rpython-unicode/rpython/ootypesystem Message-ID: <20071108222419.F05748147@code0.codespeak.net> Author: fijal Date: Thu Nov 8 23:24:18 2007 New Revision: 48457 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Log: Some more tests passing Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py Thu Nov 8 23:24:18 2007 @@ -85,11 +85,13 @@ def compute_result_annotation(self, str_s): assert isinstance(str_s, annmodel.SomeOOInstance)\ - and str_s.ootype is ootype.String + and (str_s.ootype is ootype.String or + str_s.ootype is ootype.Unicode) return annmodel.SomeInteger() def specialize_call(self, hop): assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)\ - and hop.args_s[0].ootype is ootype.String + and (hop.args_s[0].ootype is ootype.String or + hop.args_s[0].ootype is ootype.Unicode) vlist = hop.inputargs(hop.args_r[0]) return hop.genop('oohash', vlist, resulttype=ootype.Signed) Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Thu Nov 8 23:24:18 2007 @@ -1163,7 +1163,7 @@ def ll_split_chr(self, ch): # NOT_RPYTHON - res = _list(List(String)) + res = _list(List(self._TYPE)) res._list = [self.make_string(s) for s in self._str.split(ch)] return res @@ -1541,7 +1541,8 @@ return inst._identityhash() def oohash(inst): - assert typeOf(inst) is String # for now only strings are supported + assert typeOf(inst) is String or typeOf(inst) is Unicode + # for now only strings and unicode are supported return hash(inst._str) def oostring(obj, base): From fijal at codespeak.net Thu Nov 8 23:37:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Nov 2007 23:37:42 +0100 (CET) Subject: [pypy-svn] r48458 - in pypy/branch/pypy-rpython-unicode/rpython: ootypesystem test Message-ID: <20071108223742.78816816E@code0.codespeak.net> Author: fijal Date: Thu Nov 8 23:37:41 2007 New Revision: 48458 Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: Yay! Turn on ootypesystem tests, as they're passing Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Thu Nov 8 23:37:41 2007 @@ -4,7 +4,8 @@ AbstractUniCharRepr, AbstractStringIteratorRepr,\ AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.rmodel import IntegerRepr -from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar, typeOf +from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar, typeOf,\ + cast_primitive from pypy.rpython.ootypesystem import ootype from pypy.rpython.rmodel import Repr @@ -54,6 +55,17 @@ def make_string(self, value): return ootype.make_unicode(value) + def ll_str(self, value): + sb = ootype.new(ootype.StringBuilder) + lgt = value.ll_strlen() + sb.ll_allocate(lgt) + for i in range(lgt): + c = value.ll_stritem_nonneg(i) + if ord(c) > 127: + raise ValueError("%d > 127, not ascii" % ord(c)) + sb.ll_append_char(cast_primitive(Char, c)) + return sb.ll_build() + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char @@ -78,7 +90,7 @@ lgt = s.ll_strlen() res.ll_allocate(lgt) for i in range(lgt): - res.ll_append_char(s.ll_stritem_nonneg(i)) + res.ll_append_char(cast_primitive(UniChar, s.ll_stritem_nonneg(i))) return res.ll_build() def ll_unichr2unicode(ch): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Thu Nov 8 23:37:41 2007 @@ -130,5 +130,5 @@ class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 -#class TestOOtype(BaseTestRUnicode, OORtypeMixin): -# pass +class TestOOtype(BaseTestRUnicode, OORtypeMixin): + EMPTY_STRING_HASH = 0 From cfbolz at codespeak.net Fri Nov 9 10:27:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 10:27:45 +0100 (CET) Subject: [pypy-svn] r48461 - pypy/branch/pypy-rpython-unicode/rpython/test Message-ID: <20071109092745.0C8F18186@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 10:27:42 2007 New Revision: 48461 Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: a failing test about unicode conversion raising an exception Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Fri Nov 9 10:27:42 2007 @@ -46,6 +46,33 @@ assert self.ll_to_string(self.interpret(f, [])) == 'xxx' + def test_conversion_errors(self): + py.test.skip("do we want this test to pass?") + def f(x): + if x: + string = '\x80\x81' + uni = u'\x80\x81' + else: + string = '\x82\x83' + uni = u'\x83\x84\x84' + try: + str(uni) + except UnicodeEncodeError: + pass + else: + return -1 + try: + unicode(string) + except UnicodeDecodeError: + return len(string) + len(uni) + else: + return -2 + assert f(True) == 4 + assert f(False) == 5 + res = self.interpret(f, [True]) + assert res == 4 + + def test_str_unicode_nonconst(self): def f(x): y = u'xxx' + unichr(x) From cfbolz at codespeak.net Fri Nov 9 11:26:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 11:26:23 +0100 (CET) Subject: [pypy-svn] r48463 - in pypy/branch/pypy-rpython-unicode: annotation rpython rpython/test Message-ID: <20071109102623.26A0081C5@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 11:26:22 2007 New Revision: 48463 Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: unicode.decode doesn't make any sense! it should be unicode.encode Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Fri Nov 9 11:26:22 2007 @@ -462,7 +462,7 @@ return str.basestringclass() class __extend__(SomeUnicodeString): - def method_decode(uni, s_enc): + def method_encode(uni, s_enc): if not s_enc.is_constant(): raise TypeError("Non-constant decoding not supported") enc = s_enc.const Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Fri Nov 9 11:26:22 2007 @@ -267,7 +267,7 @@ return s class __extend__(AbstractUnicodeRepr): - def rtype_method_decode(self, hop): + def rtype_method_encode(self, hop): v_self = hop.inputarg(self, 0) return hop.gendirectcall(self.ll_str, v_self) Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Fri Nov 9 11:26:22 2007 @@ -81,10 +81,10 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) self.interpret_raises(ValueError, f, [1234]) - def test_unicode_decode(self): + def test_unicode_encode(self): def f(x): y = u'xxx' - return (y + unichr(x)).decode('ascii') + return (y + unichr(x)).encode('ascii') assert self.ll_to_string(self.interpret(f, [38])) == f(38) From cfbolz at codespeak.net Fri Nov 9 11:52:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 11:52:41 +0100 (CET) Subject: [pypy-svn] r48464 - in pypy/branch/pypy-rpython-unicode/rpython: . lltypesystem ootypesystem test Message-ID: <20071109105241.0009B8165@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 11:52:40 2007 New Revision: 48464 Modified: pypy/branch/pypy-rpython-unicode/rpython/extfunctable.py pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: better exception raising for encode Modified: pypy/branch/pypy-rpython-unicode/rpython/extfunctable.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/extfunctable.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/extfunctable.py Fri Nov 9 11:52:40 2007 @@ -196,5 +196,7 @@ IndexError : True, AssertionError : True, RuntimeError : True, + UnicodeDecodeError: True, + UnicodeEncodeError: True, } Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Fri Nov 9 11:52:40 2007 @@ -127,7 +127,7 @@ for i in range(lgt): c = s.chars[i] if ord(c) > 127: - raise ValueError("character not in ascii range") + raise UnicodeEncodeError("character not in ascii range") result.chars[i] = cast_primitive(Char, c) return result Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Fri Nov 9 11:52:40 2007 @@ -62,7 +62,7 @@ for i in range(lgt): c = value.ll_stritem_nonneg(i) if ord(c) > 127: - raise ValueError("%d > 127, not ascii" % ord(c)) + raise UnicodeEncodeError("%d > 127, not ascii" % ord(c)) sb.ll_append_char(cast_primitive(Char, c)) return sb.ll_build() Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Fri Nov 9 11:52:40 2007 @@ -269,6 +269,7 @@ class __extend__(AbstractUnicodeRepr): def rtype_method_encode(self, hop): v_self = hop.inputarg(self, 0) + hop.exception_is_here() return hop.gendirectcall(self.ll_str, v_self) class __extend__(pairtype(AbstractStringRepr, Repr)): Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Fri Nov 9 11:52:40 2007 @@ -79,7 +79,7 @@ return str(y) assert self.ll_to_string(self.interpret(f, [38])) == f(38) - self.interpret_raises(ValueError, f, [1234]) + self.interpret_raises(UnicodeEncodeError, f, [1234]) def test_unicode_encode(self): def f(x): @@ -88,6 +88,19 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def test_unicode_encode_error(self): + def f(x): + y = u'xxx' + try: + x = (y + unichr(x)).encode('ascii') + return len(x) + except UnicodeEncodeError: + return -1 + + assert self.interpret(f, [38]) == f(38) + assert self.interpret(f, [138]) == f(138) + + def test_unichar_const(self): def fn(c): return c From cfbolz at codespeak.net Fri Nov 9 12:29:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 12:29:16 +0100 (CET) Subject: [pypy-svn] r48465 - in pypy/branch/pypy-rpython-unicode: annotation rpython rpython/lltypesystem rpython/ootypesystem rpython/test Message-ID: <20071109112916.AC6D28176@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 12:29:14 2007 New Revision: 48465 Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py pypy/branch/pypy-rpython-unicode/rpython/rstr.py pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Log: add a decode method to strings Modified: pypy/branch/pypy-rpython-unicode/annotation/unaryop.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/annotation/unaryop.py (original) +++ pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Fri Nov 9 12:29:14 2007 @@ -464,11 +464,12 @@ class __extend__(SomeUnicodeString): def method_encode(uni, s_enc): if not s_enc.is_constant(): - raise TypeError("Non-constant decoding not supported") + raise TypeError("Non-constant encoding not supported") enc = s_enc.const if enc != 'ascii': - raise TypeError("Decoding %s not supported for unicode" % (enc,)) + raise TypeError("Encoding %s not supported for unicode" % (enc,)) return SomeString() + method_encode.can_only_throw = [UnicodeEncodeError] class __extend__(SomeString): def method_upper(str): @@ -477,6 +478,15 @@ def method_lower(str): return SomeString() + def method_decode(str, s_enc): + if not s_enc.is_constant(): + raise TypeError("Non-constant encoding not supported") + enc = s_enc.const + if enc != 'ascii': + raise TypeError("Encoding %s not supported for strings" % (enc,)) + return SomeUnicodeString() + method_decode.can_only_throw = [UnicodeDecodeError] + class __extend__(SomeChar): def len(chr): Modified: pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Fri Nov 9 12:29:14 2007 @@ -228,6 +228,8 @@ lgt = len(str.chars) s = mallocunicode(lgt) for i in range(lgt): + if ord(str.chars[i]) > 127: + raise UnicodeDecodeError s.chars[i] = cast_primitive(UniChar, str.chars[i]) return s Modified: pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Fri Nov 9 12:29:14 2007 @@ -90,7 +90,10 @@ lgt = s.ll_strlen() res.ll_allocate(lgt) for i in range(lgt): - res.ll_append_char(cast_primitive(UniChar, s.ll_stritem_nonneg(i))) + c = s.ll_stritem_nonneg(i) + if ord(c) > 127: + raise UnicodeDecodeError + res.ll_append_char(cast_primitive(UniChar, c)) return res.ll_build() def ll_unichr2unicode(ch): Modified: pypy/branch/pypy-rpython-unicode/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/rstr.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/rstr.py Fri Nov 9 12:29:14 2007 @@ -24,8 +24,7 @@ raise TypeError("Cannot do toupper on unicode string") def rtype_method_lower(self, hop): - raise TypeError("Cannot do toupper on unicode string") - + raise TypeError("Cannot do tolower on unicode string") class __extend__(annmodel.SomeString): def rtyper_makerepr(self, rtyper): @@ -253,9 +252,14 @@ return hop.inputconst(hop.r_result, hop.s_result.const) repr = hop.args_r[0].repr v_str = hop.inputarg(repr, 0) - hop.exception_cannot_occur() + hop.exception_is_here() return hop.gendirectcall(self.ll.ll_str2unicode, v_str) + def rtype_method_decode(self, hop): + v_self = hop.inputarg(self, 0) + hop.exception_is_here() + return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it string_repr = hop.args_r[0].repr Modified: pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py (original) +++ pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Fri Nov 9 12:29:14 2007 @@ -100,6 +100,25 @@ assert self.interpret(f, [38]) == f(38) assert self.interpret(f, [138]) == f(138) + def test_unicode_decode(self): + def f(x): + y = 'xxx' + return (y + chr(x)).decode('ascii') + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + + def test_unicode_decode_error(self): + def f(x): + y = 'xxx' + try: + x = (y + chr(x)).decode('ascii') + return len(x) + except UnicodeDecodeError: + return -1 + + assert self.interpret(f, [38]) == f(38) + assert self.interpret(f, [138]) == f(138) + def test_unichar_const(self): def fn(c): From fijal at codespeak.net Fri Nov 9 13:53:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Nov 2007 13:53:05 +0100 (CET) Subject: [pypy-svn] r48466 - in pypy/dist/pypy: annotation annotation/test objspace objspace/flow/test objspace/std/test rpython rpython/lltypesystem rpython/module rpython/ootypesystem rpython/ootypesystem/test rpython/test translator/test Message-ID: <20071109125305.B1B3781E1@code0.codespeak.net> Author: fijal Date: Fri Nov 9 13:53:02 2007 New Revision: 48466 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/signature.py pypy/dist/pypy/annotation/test/test_annrpython.py pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/objspace/descroperation.py pypy/dist/pypy/objspace/flow/test/test_objspace.py pypy/dist/pypy/objspace/std/test/test_unicodeobject.py pypy/dist/pypy/rpython/extfunctable.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/module/support.py pypy/dist/pypy/rpython/ootypesystem/ooregistry.py pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/rpython/ootypesystem/test/test_ooann.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/test/test_rstr.py pypy/dist/pypy/rpython/test/test_runicode.py pypy/dist/pypy/rpython/test/tool.py pypy/dist/pypy/translator/test/test_geninterp.py Log: (fijal, cfbolz, antocuni) Merge branch of rpython-unicode ------------------------------------------------------------------------ r48463 | cfbolz | 2007-11-09 11:26:22 +0100 (Fri, 09 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py unicode.decode doesn't make any sense! it should be unicode.encode ------------------------------------------------------------------------ r48461 | cfbolz | 2007-11-09 10:27:42 +0100 (Fri, 09 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py a failing test about unicode conversion raising an exception ------------------------------------------------------------------------ r48458 | fijal | 2007-11-08 23:37:41 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Yay! Turn on ootypesystem tests, as they're passing ------------------------------------------------------------------------ r48457 | fijal | 2007-11-08 23:24:18 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py Some more tests passing ------------------------------------------------------------------------ r48456 | cfbolz | 2007-11-08 22:55:54 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py more tests passing ------------------------------------------------------------------------ r48455 | cfbolz | 2007-11-08 22:50:47 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/rbuiltin.py at least this is needed ------------------------------------------------------------------------ r48454 | cfbolz | 2007-11-08 22:47:04 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py skip this for now ------------------------------------------------------------------------ r48453 | cfbolz | 2007-11-08 22:46:01 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py remove this too ------------------------------------------------------------------------ r48452 | fijal | 2007-11-08 22:39:48 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Revert 48449 ------------------------------------------------------------------------ r48451 | cfbolz | 2007-11-08 22:38:11 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py remove unicode and unichr from the list of base space operations ------------------------------------------------------------------------ r48450 | fijal | 2007-11-08 22:38:00 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py Remove unichr as spaceop ------------------------------------------------------------------------ r48449 | fijal | 2007-11-08 22:34:54 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/flow/specialcase.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Remove unichr as spaceop ------------------------------------------------------------------------ r48448 | fijal | 2007-11-08 22:14:18 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Create proper str2unicode ------------------------------------------------------------------------ r48447 | fijal | 2007-11-08 22:07:25 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Remove unichr() and unicode() from descroperatoin ------------------------------------------------------------------------ r48444 | fijal | 2007-11-08 21:19:53 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py Squash few failures, now 7 out of 13 tests are failing ------------------------------------------------------------------------ r48439 | fijal | 2007-11-08 20:55:03 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py that was easy :) ------------------------------------------------------------------------ r48435 | cfbolz | 2007-11-08 20:50:14 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/translator/test/test_geninterp.py test for geninterp unicode behaviour ------------------------------------------------------------------------ r48433 | antocuni | 2007-11-08 20:43:50 +0100 (Thu, 08 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/llinterp.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ooregistry.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py more work towards ootype unicode ------------------------------------------------------------------------ r48428 | antocuni | 2007-11-08 20:16:06 +0100 (Thu, 08 Nov 2007) | 4 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py fix this test; the problem was the _defl method of AbstractString, which shouldn't have been there. ------------------------------------------------------------------------ r48427 | fijal | 2007-11-08 19:04:22 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/std/test/test_unicodeobject.py Skipped test that should work ------------------------------------------------------------------------ r48426 | fijal | 2007-11-08 18:43:51 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Fix test ------------------------------------------------------------------------ r48425 | antocuni | 2007-11-08 18:39:23 +0100 (Thu, 08 Nov 2007) | 5 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/test/test_ooann.py M /pypy/branch/pypy-rpython-unicode/rpython/test/tool.py first step to add Unicode support to ootype; it's not completed because of a blocker I could not solve yet; see the failing test in ootypesystem/test/test_ooann.py ------------------------------------------------------------------------ r48418 | fijal | 2007-11-08 16:08:03 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py unicode.decode('ascii') ------------------------------------------------------------------------ r48417 | fijal | 2007-11-08 16:02:00 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py One more case... ------------------------------------------------------------------------ r48416 | fijal | 2007-11-08 15:52:44 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py ll_str for unicode ------------------------------------------------------------------------ r48415 | fijal | 2007-11-08 15:39:32 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Test str(unicodeconstant) is constant propagated and works out of the box. ------------------------------------------------------------------------ r48414 | fijal | 2007-11-08 14:31:36 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py This is no longer needed I think ------------------------------------------------------------------------ r48411 | fijal | 2007-11-08 14:20:32 +0100 (Thu, 08 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Shuffle tests around, so unichar tests are in test_runicode now, not in test_rstr. ------------------------------------------------------------------------ r48410 | antocuni | 2007-11-08 14:20:11 +0100 (Thu, 08 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py remove outdated comment ------------------------------------------------------------------------ r48408 | cfbolz | 2007-11-08 13:46:19 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/ootype.py introduce Unicode. check in to have Anto take over. ------------------------------------------------------------------------ r48388 | fijal | 2007-11-08 01:55:43 +0100 (Thu, 08 Nov 2007) | 6 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/binaryop.py M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py All string tests, which are decided to be supported by now, passes with unicode (for lltypesystem). There are few todos: * fix the if typeOf(...) evil hacks * think about unicode mod sth * think about str(some_unicode) ------------------------------------------------------------------------ r48386 | fijal | 2007-11-08 01:43:13 +0100 (Thu, 08 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py First round of simplifications and getting rid of ugly typeOf switch ------------------------------------------------------------------------ r48374 | fijal | 2007-11-07 23:19:32 +0100 (Wed, 07 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py * fix test * a bit of code simplification ------------------------------------------------------------------------ r48372 | fijal | 2007-11-07 22:05:57 +0100 (Wed, 07 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/ootypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Support for most methods which I think should be supported for unicode (right now). Still roughly 12 tests missing. ------------------------------------------------------------------------ r48367 | fijal | 2007-11-07 18:00:44 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/module/support.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py M /pypy/branch/pypy-rpython-unicode/rpython/test/tool.py Whack, whack.. another tests are passing ------------------------------------------------------------------------ r48366 | fijal | 2007-11-07 18:00:28 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Allow unichr * integer ------------------------------------------------------------------------ r48365 | fijal | 2007-11-07 16:23:07 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py One test passing, one skipped ------------------------------------------------------------------------ r48364 | fijal | 2007-11-07 16:15:08 +0100 (Wed, 07 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Eventual future support for unichar.isxxx, right now disabled (+ disable test) ------------------------------------------------------------------------ r48363 | fijal | 2007-11-07 16:14:37 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Remove unnecessary method (it's on SomeObject anyway) ------------------------------------------------------------------------ r48362 | fijal | 2007-11-07 16:04:57 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py unichr -> unicode translation ------------------------------------------------------------------------ r48360 | fijal | 2007-11-07 14:48:26 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Iteration over unicode strings works. ------------------------------------------------------------------------ r48359 | fijal | 2007-11-07 14:46:16 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/model.py M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Fix thing that iteration over unicode returned chars ------------------------------------------------------------------------ r48357 | fijal | 2007-11-07 14:22:06 +0100 (Wed, 07 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/model.py M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Make test pass (thanks carl, that was stupid) ------------------------------------------------------------------------ r48356 | fijal | 2007-11-07 14:02:11 +0100 (Wed, 07 Nov 2007) | 4 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/model.py M /pypy/branch/pypy-rpython-unicode/annotation/signature.py M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py * Relax somewhat return values from method_xxx of SomeString, as SomeChar should return string, not self * Try to make the same working on SomeUnicode, does not work ------------------------------------------------------------------------ r48341 | fijal | 2007-11-06 17:10:38 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Expose string methods to unicode strings ------------------------------------------------------------------------ r48336 | fijal | 2007-11-06 13:24:09 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py simplify code, as mixing unicode and strings are forbidden ------------------------------------------------------------------------ r48335 | fijal | 2007-11-06 13:19:17 +0100 (Tue, 06 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/binaryop.py M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py Forbid mixing unicode and string (this leaves some desgin flaw around as mixing of unichr and chr is ok) ------------------------------------------------------------------------ r48334 | fijal | 2007-11-06 12:42:26 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py Simple version works as well ------------------------------------------------------------------------ r48333 | fijal | 2007-11-06 12:39:37 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py Make test more mix unicode and str ------------------------------------------------------------------------ r48332 | fijal | 2007-11-06 12:38:02 +0100 (Tue, 06 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/rpython/lltypesystem/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_rstr.py M /pypy/branch/pypy-rpython-unicode/rpython/test/test_runicode.py Some support for unicode strings. This is work in progress, not all tests are really testing what they should. ------------------------------------------------------------------------ r48329 | fijal | 2007-11-06 10:51:36 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/binaryop.py Ooops, one annotation too far ------------------------------------------------------------------------ r48328 | fijal | 2007-11-06 01:08:34 +0100 (Tue, 06 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/annotation/binaryop.py M /pypy/branch/pypy-rpython-unicode/annotation/test/test_annrpython.py M /pypy/branch/pypy-rpython-unicode/annotation/unaryop.py Proper support for unicode() and unichr() operations ------------------------------------------------------------------------ r48327 | fijal | 2007-11-06 00:54:49 +0100 (Tue, 06 Nov 2007) | 3 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/objspace/descroperation.py Dummy descroperations, because otherwise something complains about lack of those ------------------------------------------------------------------------ r48326 | fijal | 2007-11-05 21:07:56 +0100 (Mon, 05 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Constant folding for unicode as well ------------------------------------------------------------------------ r48325 | fijal | 2007-11-05 21:05:56 +0100 (Mon, 05 Nov 2007) | 2 lines Changed paths: M /pypy/branch/pypy-rpython-unicode/interpreter/baseobjspace.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/operation.py M /pypy/branch/pypy-rpython-unicode/objspace/flow/test/test_objspace.py Support for constant-folding of unichr ------------------------------------------------------------------------ r48315 | fijal | 2007-11-05 12:25:54 +0100 (Mon, 05 Nov 2007) | 2 lines Changed paths: A /pypy/branch/pypy-rpython-unicode (from /pypy/dist/pypy:48314) Create new branch for rpython unicode support Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Fri Nov 9 13:53:02 2007 @@ -426,13 +426,18 @@ return SomeChar() -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint), - pairtype(SomeChar, SomeUnicodeCodePoint), +class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeCodePoint, SomeChar)): + def union((uchr1, uchr2)): + return SomeUnicodeCodePoint() +class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): def union((uchr1, uchr2)): return SomeUnicodeCodePoint() + def add((chr1, chr2)): + return SomeUnicodeString() + class __extend__(pairtype(SomeString, SomeObject)): def mod((str, args)): @@ -610,10 +615,11 @@ lst1.listdef.resize() delitem.can_only_throw = [] -class __extend__(pairtype(SomeString, SomeSlice)): +class __extend__(pairtype(SomeString, SomeSlice), + pairtype(SomeUnicodeString, SomeSlice)): def getitem((str1, slic)): - return SomeString() + return str1.basestringclass() getitem.can_only_throw = [] class __extend__(pairtype(SomeString, SomeInteger)): @@ -651,23 +657,30 @@ getitem_idx_key = getitem_idx - # uncomment if we really want to support that - #def mul((str1, int2)): # xxx do we want to support this - # getbookkeeper().count("str_mul", str1, int2) - # return SomeString() + def mul((str1, int2)): # xxx do we want to support this + getbookkeeper().count("str_mul", str1, int2) + return SomeUnicodeString() -class __extend__(pairtype(SomeInteger, SomeString)): +class __extend__(pairtype(SomeInteger, SomeString), + pairtype(SomeInteger, SomeUnicodeString)): def mul((int1, str2)): # xxx do we want to support this getbookkeeper().count("str_mul", str2, int1) - return SomeString() + return str2.basestringclass() -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString), +class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), + pairtype(SomeUnicodeString, SomeUnicodeCodePoint), pairtype(SomeUnicodeString, SomeUnicodeString)): def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_None or - str2.can_be_None) + return SomeUnicodeString(can_be_None=str1.can_be_none() or + str2.can_be_none()) + + def add((str1, str2)): + # propagate const-ness to help getattr(obj, 'prefix' + const_name) + result = SomeUnicodeString() + if str1.is_immutable_constant() and str2.is_immutable_constant(): + result.const = str1.const + str2.const + return result class __extend__(pairtype(SomeInteger, SomeList)): Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Fri Nov 9 13:53:02 2007 @@ -220,14 +220,16 @@ class SomeChar(SomeString): "Stands for an object known to be a string of length 1." -class SomeUnicodeCodePoint(SomeObject): +class SomeUnicodeCodePoint(SomeUnicodeString): "Stands for an object known to be a unicode codepoint." - knowntype = unicode - immutable = True - def can_be_none(self): return False +SomeString.basestringclass = SomeString +SomeString.basecharclass = SomeChar +SomeUnicodeString.basestringclass = SomeUnicodeString +SomeUnicodeString.basecharclass = SomeUnicodeCodePoint + class SomeList(SomeObject): "Stands for a homogenous list of any length." knowntype = list Modified: pypy/dist/pypy/annotation/signature.py ============================================================================== --- pypy/dist/pypy/annotation/signature.py (original) +++ pypy/dist/pypy/annotation/signature.py Fri Nov 9 13:53:02 2007 @@ -3,7 +3,7 @@ from pypy.annotation.model import SomeBool, SomeInteger, SomeString,\ SomeFloat, SomeList, SomeDict, s_None, SomeExternalObject,\ SomeObject, SomeInstance, SomeTuple, lltype_to_annotation,\ - unionof + unionof, SomeUnicodeString from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -80,6 +80,8 @@ return SomeFloat() elif issubclass(t, str): # py.lib uses annotated str subclasses return SomeString() + elif t is unicode: + return SomeUnicodeString() elif t is list: return SomeList(MOST_GENERAL_LISTDEF) elif t is dict: Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Fri Nov 9 13:53:02 2007 @@ -2883,7 +2883,7 @@ def test_unicode(self): def g(n): if n > 0: - return "xxx" + return unichr(1234) else: return u"x\xe4x" @@ -2906,6 +2906,62 @@ s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeUnicodeString) + def test_unicode_add(self): + def f(x): + return unicode(x) + unichr(1234) + + def g(x): + return unichr(x) + unichr(2) + + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeUnicodeString) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_unicode_startswith(self): + def f(x): + return u'xxxx'.replace(x, u'z') + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_unicode_buildtypes(self): + def f(x): + return x + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_replace_annotations(self): + def f(x): + return 'a'.replace(x, 'b') + + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeString) + + def f(x): + return u'a'.replace(x, u'b') + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode]) + assert isinstance(s, annmodel.SomeUnicodeString) + + def test_unicode_char(self): + def f(x, i): + for c in x: + if c == i: + return c + return 'x' + + a = self.RPythonAnnotator() + s = a.build_types(f, [unicode, str]) + assert isinstance(s, annmodel.SomeUnicodeCodePoint) + def g(n): return [0,1,2,n] Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Fri Nov 9 13:53:02 2007 @@ -9,7 +9,7 @@ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ SomeCTypesObject, s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable, SomeWeakRef + SomeGenericCallable, SomeWeakRef, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -25,7 +25,7 @@ 'iter', 'next', 'invert', 'type', 'issubtype', 'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct', 'ord', 'int', 'float', 'long', 'id', - 'neg_ovf', 'abs_ovf', 'hint']) + 'neg_ovf', 'abs_ovf', 'hint', 'unicode', 'unichr']) for opname in UNARY_OPERATIONS: missing_operation(SomeObject, opname) @@ -103,6 +103,10 @@ getbookkeeper().count('str', obj) return SomeString() + def unicode(obj): + getbookkeeper().count('unicode', obj) + return SomeUnicodeString() + def repr(obj): getbookkeeper().count('repr', obj) return SomeString() @@ -201,7 +205,6 @@ def invert(self): return SomeInteger(knowntype=self.knowntype) - invert.can_only_throw = [] def pos(self): @@ -402,7 +405,8 @@ return s_Bool -class __extend__(SomeString): +class __extend__(SomeString, + SomeUnicodeString): def method_startswith(str, frag): return s_Bool @@ -420,27 +424,29 @@ return SomeInteger(nonneg=True) def method_strip(str, chr): - return SomeString() + return str.basestringclass() def method_lstrip(str, chr): - return SomeString() + return str.basestringclass() def method_rstrip(str, chr): - return SomeString() + return str.basestringclass() def method_join(str, s_list): getbookkeeper().count("str_join", str) s_item = s_list.listdef.read_item() if isinstance(s_item, SomeImpossibleValue): + if isinstance(str, SomeUnicodeString): + return immutablevalue(u"") return immutablevalue("") - return SomeString() + return str.basestringclass() def iter(str): return SomeIterator(str) iter.can_only_throw = [] def getanyitem(str): - return SomeChar() + return str.basecharclass() def ord(str): return SomeInteger(nonneg=True) @@ -450,17 +456,36 @@ def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) - return getbookkeeper().newlist(SomeString()) + return getbookkeeper().newlist(str.basestringclass()) def method_replace(str, s1, s2): - return SomeString() + return str.basestringclass() - def method_lower(str): +class __extend__(SomeUnicodeString): + def method_encode(uni, s_enc): + if not s_enc.is_constant(): + raise TypeError("Non-constant encoding not supported") + enc = s_enc.const + if enc != 'ascii': + raise TypeError("Encoding %s not supported for unicode" % (enc,)) return SomeString() + method_encode.can_only_throw = [UnicodeEncodeError] +class __extend__(SomeString): def method_upper(str): return SomeString() + def method_lower(str): + return SomeString() + + def method_decode(str, s_enc): + if not s_enc.is_constant(): + raise TypeError("Non-constant encoding not supported") + enc = s_enc.const + if enc != 'ascii': + raise TypeError("Encoding %s not supported for strings" % (enc,)) + return SomeUnicodeString() + method_decode.can_only_throw = [UnicodeDecodeError] class __extend__(SomeChar): @@ -485,12 +510,6 @@ def method_isupper(chr): return s_Bool -class __extend__(SomeUnicodeCodePoint): - - def ord(uchr): - return SomeInteger(nonneg=True) - - class __extend__(SomeIterator): def iter(itr): Modified: pypy/dist/pypy/objspace/descroperation.py ============================================================================== --- pypy/dist/pypy/objspace/descroperation.py (original) +++ pypy/dist/pypy/objspace/descroperation.py Fri Nov 9 13:53:02 2007 @@ -592,7 +592,7 @@ setattr(DescrOperation,_name,_impl_maker(_symbol,_specialnames)) elif _name not in ['is_', 'id','type','issubtype', # not really to be defined in DescrOperation - 'ord']: + 'ord', 'unichr', 'unicode']: raise Exception, "missing def for operation %s" % _name Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Fri Nov 9 13:53:02 2007 @@ -712,6 +712,21 @@ return x[s] graph = self.codetest(myfunc) + def test_unichr_constfold(self): + py.test.skip("not working") + def myfunc(): + return unichr(1234) + graph = self.codetest(myfunc) + assert graph.startblock.exits[0].target is graph.returnblock + + def test_unicode_constfold(self): + py.test.skip("not working for now") + def myfunc(): + return unicode("1234") + graph = self.codetest(myfunc) + assert graph.startblock.exits[0].target is graph.returnblock + + def test_getitem(self): def f(c, x): try: Modified: pypy/dist/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_unicodeobject.py Fri Nov 9 13:53:02 2007 @@ -264,3 +264,17 @@ assert unicode(None) == u'None' assert unicode(123) == u'123' assert unicode([2, 3]) == u'[2, 3]' + + def test_call_unicode(self): + skip("does not work") + class X: + def __unicode__(self): + return u'x' + + try: + unicode(X(), 'ascii') + except TypeError, t: + assert 'need string or buffer' in str(t) + else: + raise Exception("DID NOT RAISE") + Modified: pypy/dist/pypy/rpython/extfunctable.py ============================================================================== --- pypy/dist/pypy/rpython/extfunctable.py (original) +++ pypy/dist/pypy/rpython/extfunctable.py Fri Nov 9 13:53:02 2007 @@ -196,5 +196,7 @@ IndexError : True, AssertionError : True, RuntimeError : True, + UnicodeDecodeError: True, + UnicodeEncodeError: True, } Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Nov 9 13:53:02 2007 @@ -1086,6 +1086,9 @@ def op_oostring(self, obj, base): return ootype.oostring(obj, base) + def op_oounicode(self, obj, base): + return ootype.oounicode(obj, base) + def op_ooparse_int(self, s, base): try: return ootype.ooparse_int(s, base) Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Fri Nov 9 13:53:02 2007 @@ -8,12 +8,13 @@ from pypy.rpython.rmodel import inputconst, IntegerRepr from pypy.rpython.rstr import AbstractStringRepr,AbstractCharRepr,\ AbstractUniCharRepr, AbstractStringIteratorRepr,\ - AbstractLLHelpers + AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.lltypesystem import ll_str from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, UniChar, Ptr, malloc, \ - Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive - + Bool, Void, GcArray, nullptr, pyobjectptr, cast_primitive, typeOf,\ + staticAdtMethod, GcForwardReference +from pypy.rpython.rmodel import Repr # ____________________________________________________________ # @@ -24,19 +25,43 @@ # chars: array of Char # } -STR = GcStruct('rpy_string', ('hash', Signed), - ('chars', Array(Char, hints={'immutable': True, - 'isrpystring': True}))) -UNICODE = GcStruct('rpy_unicode', ('hash', Signed), - ('chars', Array(UniChar, hints={'immutable': True}))) +STR = GcForwardReference() +UNICODE = GcForwardReference() + +def new_malloc(TP): + def mallocstr(length): + debug_assert(length >= 0, "negative string length") + r = malloc(TP, length) + if not we_are_translated() or not malloc_zero_filled: + r.hash = 0 + return r + mallocstr._annspecialcase_ = 'specialize:semierased' + return mallocstr + +mallocstr = new_malloc(STR) +mallocunicode = new_malloc(UNICODE) + +def emptystrfun(): + return emptystr + +def emptyunicodefun(): + return emptyunicode + +STR.become(GcStruct('rpy_string', ('hash', Signed), + ('chars', Array(Char, hints={'immutable': True, + 'isrpystring': True})), + adtmeths={'malloc' : staticAdtMethod(mallocstr), + 'empty' : staticAdtMethod(emptystrfun)})) +UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed), + ('chars', Array(UniChar, hints={'immutable': True})), + adtmeths={'malloc' : staticAdtMethod(mallocunicode), + 'empty' : staticAdtMethod(emptyunicodefun)} + )) SIGNED_ARRAY = GcArray(Signed) CONST_STR_CACHE = WeakValueDictionary() +CONST_UNICODE_CACHE = WeakValueDictionary() -class BaseStringRepr(AbstractStringRepr): - def __init__(self, *args): - AbstractStringRepr.__init__(self, *args) - self.ll = LLHelpers - +class BaseLLStringRepr(Repr): def convert_const(self, value): if value is None: return nullptr(self.lowleveltype.TO) @@ -44,20 +69,21 @@ if not isinstance(value, self.basetype): raise TyperError("not a str: %r" % (value,)) try: - return CONST_STR_CACHE[value] + return self.CACHE[value] except KeyError: p = self.malloc(len(value)) for i in range(len(value)): p.chars[i] = cast_primitive(self.base, value[i]) p.hash = 0 self.ll.ll_strhash(p) # precompute the hash - CONST_STR_CACHE[value] = p + self.CACHE[value] = p return p def make_iterator_repr(self): - return string_iterator_repr + return self.iterator_repr def can_ll_be_null(self, s_value): + # XXX unicode if self is string_repr: return s_value.can_be_none() else: @@ -71,28 +97,44 @@ v_items = hop.gendirectcall(LIST.ll_items, v_lst) return v_length, v_items -class StringRepr(BaseStringRepr): +class StringRepr(BaseLLStringRepr, AbstractStringRepr): lowleveltype = Ptr(STR) basetype = str base = Char + CACHE = CONST_STR_CACHE def __init__(self, *args): - BaseStringRepr.__init__(self, *args) + AbstractStringRepr.__init__(self, *args) + self.ll = LLHelpers self.malloc = mallocstr -class UnicodeRepr(BaseStringRepr): +class UnicodeRepr(BaseLLStringRepr, AbstractUnicodeRepr): lowleveltype = Ptr(UNICODE) basetype = basestring base = UniChar + CACHE = CONST_UNICODE_CACHE def __init__(self, *args): - BaseStringRepr.__init__(self, *args) + AbstractUnicodeRepr.__init__(self, *args) + self.ll = LLHelpers self.malloc = mallocunicode + def ll_str(self, s): + # XXX crazy that this is here, but I don't want to break + # rmodel logic + lgt = len(s.chars) + result = mallocstr(lgt) + for i in range(lgt): + c = s.chars[i] + if ord(c) > 127: + raise UnicodeEncodeError("character not in ascii range") + result.chars[i] = cast_primitive(Char, c) + return result + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char -class UniCharRepr(AbstractUniCharRepr): +class UniCharRepr(AbstractUniCharRepr, UnicodeRepr): lowleveltype = UniChar class __extend__(pairtype(PyObjRepr, AbstractStringRepr)): @@ -119,19 +161,6 @@ resulttype=pyobj_repr, _callable= lambda v: pyobjectptr(''.join(v.chars))) -def new_malloc(TP): - def mallocstr(length): - debug_assert(length >= 0, "negative string length") - r = malloc(TP, length) - if not we_are_translated() or not malloc_zero_filled: - r.hash = 0 - return r - mallocstr._annspecialcase_ = 'specialize:semierased' - return mallocstr - -mallocstr = new_malloc(STR) -mallocunicode = new_malloc(UNICODE) - # ____________________________________________________________ # # Low-level methods. These can be run for testing, but are meant to @@ -163,9 +192,13 @@ class LLHelpers(AbstractLLHelpers): def ll_char_mul(ch, times): + if typeOf(ch) is Char: + malloc = mallocstr + else: + malloc = mallocunicode if times < 0: times = 0 - newstr = mallocstr(times) + newstr = malloc(times) j = 0 while j < times: newstr.chars[j] = ch @@ -183,10 +216,23 @@ ll_stritem_nonneg._annenforceargs_ = [None, int] def ll_chr2str(ch): - s = mallocstr(1) + if typeOf(ch) is Char: + malloc = mallocstr + else: + malloc = mallocunicode + s = malloc(1) s.chars[0] = ch return s + def ll_str2unicode(str): + lgt = len(str.chars) + s = mallocunicode(lgt) + for i in range(lgt): + if ord(str.chars[i]) > 127: + raise UnicodeDecodeError + s.chars[i] = cast_primitive(UniChar, str.chars[i]) + return s + def ll_strhash(s): # unlike CPython, there is no reason to avoid to return -1 # but our malloc initializes the memory to zero, so we use zero as the @@ -204,7 +250,7 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - newstr = mallocstr(len1 + len2) + newstr = s1.malloc(len1 + len2) j = 0 while j < len1: newstr.chars[j] = s1.chars[j] @@ -219,7 +265,7 @@ def ll_strip(s, ch, left, right): s_len = len(s.chars) if s_len == 0: - return emptystr + return s.empty() lpos = 0 rpos = s_len - 1 if left: @@ -229,7 +275,7 @@ while lpos < rpos and s.chars[rpos] == ch: rpos -= 1 r_len = rpos - lpos + 1 - result = mallocstr(r_len) + result = s.malloc(r_len) i = 0 j = lpos while i < r_len: @@ -242,9 +288,9 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - return emptystr + return s.empty() i = 0 - result = mallocstr(s_len) + result = s.malloc(s_len) while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': @@ -257,9 +303,9 @@ s_chars = s.chars s_len = len(s_chars) if s_len == 0: - return emptystr + return s.empty() i = 0 - result = mallocstr(s_len) + result = s.malloc(s_len) while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': @@ -273,13 +319,13 @@ s_len = len(s_chars) num_items = length if num_items == 0: - return emptystr + return s.empty() itemslen = 0 i = 0 while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen + s_len * (num_items - 1)) + result = s.malloc(itemslen + s_len * (num_items - 1)) res_chars = result.chars res_index = 0 i = 0 @@ -529,7 +575,11 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen) + if typeOf(items).TO.OF.TO == STR: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(itemslen) res_chars = result.chars res_index = 0 i = 0 @@ -546,7 +596,11 @@ def ll_join_chars(length, chars): num_chars = length - result = mallocstr(num_chars) + if typeOf(chars).TO.OF == Char: + malloc = mallocstr + else: + malloc = mallocunicode + result = malloc(num_chars) res_chars = result.chars i = 0 while i < num_chars: @@ -556,7 +610,7 @@ def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) - newstr = mallocstr(len1 - start) + newstr = s1.malloc(len1 - start) j = 0 while start < len1: newstr.chars[j] = s1.chars[start] @@ -571,7 +625,7 @@ if start == 0: return s1 stop = len(s1.chars) - newstr = mallocstr(stop - start) + newstr = s1.malloc(stop - start) j = 0 while start < stop: newstr.chars[j] = s1.chars[start] @@ -581,7 +635,7 @@ def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 - newstr = mallocstr(newlen) + newstr = s1.malloc(newlen) j = 0 while j < newlen: newstr.chars[j] = s1.chars[j] @@ -604,7 +658,7 @@ resindex = 0 while j < strlen: if chars[j] == c: - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = s.malloc(j - i) newchars = item.chars k = i while k < j: @@ -613,7 +667,7 @@ resindex += 1 i = j + 1 j += 1 - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = s.malloc(j - i) newchars = item.chars k = i while k < j: @@ -625,7 +679,7 @@ def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) - newstr = mallocstr(length) + newstr = s.malloc(length) src = s.chars dst = newstr.chars j = 0 @@ -782,22 +836,42 @@ unichar_repr.ll = LLHelpers unicode_repr = UnicodeRepr() emptystr = string_repr.convert_const("") +emptyunicode = unicode_repr.convert_const(u'') StringRepr.repr = string_repr UnicodeRepr.repr = unicode_repr +UniCharRepr.repr = unicode_repr +UniCharRepr.char_repr = unichar_repr +UnicodeRepr.char_repr = unichar_repr +CharRepr.char_repr = char_repr +StringRepr.char_repr = char_repr + +class BaseStringIteratorRepr(AbstractStringIteratorRepr): -class StringIteratorRepr(AbstractStringIteratorRepr): + def __init__(self): + self.ll_striter = ll_striter + self.ll_strnext = ll_strnext +class StringIteratorRepr(BaseStringIteratorRepr): + lowleveltype = Ptr(GcStruct('stringiter', ('string', string_repr.lowleveltype), ('index', Signed))) - def __init__(self): - self.ll_striter = ll_striter - self.ll_strnext = ll_strnext +class UnicodeIteratorRepr(BaseStringIteratorRepr): + + lowleveltype = Ptr(GcStruct('unicodeiter', + ('string', unicode_repr.lowleveltype), + ('index', Signed))) def ll_striter(string): - iter = malloc(string_iterator_repr.lowleveltype.TO) + if typeOf(string) == string_repr.lowleveltype: + TP = string_repr.iterator_repr.lowleveltype.TO + elif typeOf(string) == unicode_repr.lowleveltype: + TP = unicode_repr.iterator_repr.lowleveltype.TO + else: + raise TypeError("Unknown string type %s" % (typeOf(string),)) + iter = malloc(TP) iter.string = string iter.index = 0 return iter @@ -810,8 +884,8 @@ iter.index = index + 1 return chars[index] -string_iterator_repr = StringIteratorRepr() - +string_repr.iterator_repr = StringIteratorRepr() +unicode_repr.iterator_repr = UnicodeIteratorRepr() # these should be in rclass, but circular imports prevent (also it's # not that insane that a string constant is built in this file). Modified: pypy/dist/pypy/rpython/module/support.py ============================================================================== --- pypy/dist/pypy/rpython/module/support.py (original) +++ pypy/dist/pypy/rpython/module/support.py Fri Nov 9 13:53:02 2007 @@ -29,6 +29,16 @@ return p to_rstr = staticmethod(to_rstr) + def to_runicode(s): + from pypy.rpython.lltypesystem.rstr import UNICODE, mallocunicode + if s is None: + return lltype.nullptr(UNICODE) + p = mallocunicode(len(s)) + for i in range(len(s)): + p.chars[i] = s[i] + return p + to_runicode = staticmethod(to_runicode) + def from_rstr(rs): if not rs: # null pointer return None Modified: pypy/dist/pypy/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ooregistry.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ooregistry.py Fri Nov 9 13:53:02 2007 @@ -24,6 +24,19 @@ vlist = hop.inputargs(hop.args_r[0], ootype.Signed) return hop.genop('oostring', vlist, resulttype = ootype.String) +class Entry_oounicode(ExtRegistryEntry): + _about_ = ootype.oounicode + + def compute_result_annotation(self, obj_s, base_s): + assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) + assert isinstance(base_s, annmodel.SomeInteger) + return annmodel.SomeOOInstance(ootype.Unicode) + + def specialize_call(self, hop): + assert isinstance(hop.args_s[0],annmodel.SomeUnicodeCodePoint) + vlist = hop.inputargs(hop.args_r[0], ootype.Signed) + return hop.genop('oounicode', vlist, resulttype = ootype.Unicode) + class Entry_ootype_string(ExtRegistryEntry): _type_ = ootype._string @@ -72,11 +85,13 @@ def compute_result_annotation(self, str_s): assert isinstance(str_s, annmodel.SomeOOInstance)\ - and str_s.ootype is ootype.String + and (str_s.ootype is ootype.String or + str_s.ootype is ootype.Unicode) return annmodel.SomeInteger() def specialize_call(self, hop): assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)\ - and hop.args_s[0].ootype is ootype.String + and (hop.args_s[0].ootype is ootype.String or + hop.args_s[0].ootype is ootype.Unicode) vlist = hop.inputargs(hop.args_r[0]) return hop.genop('oohash', vlist, resulttype=ootype.Signed) Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Fri Nov 9 13:53:02 2007 @@ -322,16 +322,14 @@ return set() -# WARNING: the name 'String' is rebound at the end of file -class String(BuiltinADTType): - SELFTYPE_T = object() +class AbstractString(BuiltinADTType): def __init__(self): self._null = _null_string(self) generic_types = { self.SELFTYPE_T: self } self._GENERIC_METHODS = frozendict({ - "ll_stritem_nonneg": Meth([Signed], Char), + "ll_stritem_nonneg": Meth([Signed], self.CHAR), "ll_strlen": Meth([], Signed), "ll_strconcat": Meth([self.SELFTYPE_T], self.SELFTYPE_T), "ll_streq": Meth([self.SELFTYPE_T], Bool), @@ -341,47 +339,74 @@ "ll_find": Meth([self.SELFTYPE_T, Signed, Signed], Signed), "ll_rfind": Meth([self.SELFTYPE_T, Signed, Signed], Signed), "ll_count": Meth([self.SELFTYPE_T, Signed, Signed], Signed), - "ll_find_char": Meth([Char, Signed, Signed], Signed), - "ll_rfind_char": Meth([Char, Signed, Signed], Signed), - "ll_count_char": Meth([Char, Signed, Signed], Signed), - "ll_strip": Meth([Char, Bool, Bool], self.SELFTYPE_T), + "ll_find_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_rfind_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_count_char": Meth([self.CHAR, Signed, Signed], Signed), + "ll_strip": Meth([self.CHAR, Bool, Bool], self.SELFTYPE_T), "ll_upper": Meth([], self.SELFTYPE_T), "ll_lower": Meth([], self.SELFTYPE_T), "ll_substring": Meth([Signed, Signed], self.SELFTYPE_T), # ll_substring(start, count) - "ll_split_chr": Meth([Char], List(self.SELFTYPE_T)), - "ll_contains": Meth([Char], Bool), - "ll_replace_chr_chr": Meth([Char, Char], self.SELFTYPE_T), + "ll_split_chr": Meth([self.CHAR], List(self.SELFTYPE_T)), + "ll_contains": Meth([self.CHAR], Bool), + "ll_replace_chr_chr": Meth([self.CHAR, self.CHAR], self.SELFTYPE_T), }) self._setup_methods(generic_types) + def _example(self): + return self._defl() + + def _get_interp_class(self): + return _string + + def _specialize(self, generic_types): + return self + +# WARNING: the name 'String' is rebound at the end of file +class String(AbstractString): + SELFTYPE_T = object() + CHAR = Char + + # TODO: should it return _null or ''? + def _defl(self): + return make_string('') + def _enforce(self, value): + # XXX share this with Unicode? TYPE = typeOf(value) - if TYPE == Char: + if TYPE == self.CHAR: return make_string(value) else: return BuiltinADTType._enforce(self, value) + +# WARNING: the name 'Unicode' is rebound at the end of file +class Unicode(AbstractString): + SELFTYPE_T = object() + CHAR = UniChar + # TODO: should it return _null or ''? def _defl(self): - return make_string("") - def _example(self): - return self._defl() + return make_unicode(u'') + + def _enforce(self, value): + TYPE = typeOf(value) + if TYPE == self.CHAR: + return make_unicode(value) + else: + return BuiltinADTType._enforce(self, value) + - def _get_interp_class(self): - return _string - def _specialize(self, generic_types): - return self # WARNING: the name 'StringBuilder' is rebound at the end of file class StringBuilder(BuiltinADTType): - def __init__(self): + def __init__(self, STRINGTP, CHARTP): self._null = _null_string_builder(self) self._GENERIC_METHODS = frozendict({ "ll_allocate": Meth([Signed], Void), - "ll_append_char": Meth([Char], Void), - "ll_append": Meth([String], Void), - "ll_build": Meth([], String), + "ll_append_char": Meth([CHARTP], Void), + "ll_append": Meth([STRINGTP], Void), + "ll_build": Meth([], STRINGTP), }) self._setup_methods({}) @@ -837,6 +862,10 @@ assert isinstance(value, str) return _string(String, value) +def make_unicode(value): + assert isinstance(value, unicode) + return _string(Unicode, value) + def make_instance(INSTANCE): inst = _instance(INSTANCE) if STATICNESS: @@ -1049,6 +1078,14 @@ def __cmp__(self, other): return cmp(self._str, other._str) + def make_string(self, value): + if self._TYPE is String: + return make_string(value) + elif self._TYPE is Unicode: + return make_unicode(value) + else: + assert False, 'Unknown type %s' % self._TYPE + def ll_stritem_nonneg(self, i): # NOT_RPYTHON s = self._str @@ -1061,7 +1098,7 @@ def ll_strconcat(self, s): # NOT_RPYTHON - return make_string(self._str + s._str) + return self.make_string(self._str + s._str) def ll_streq(self, s): # NOT_RPYTON @@ -1110,24 +1147,24 @@ s = s.lstrip(ch) if right: s = s.rstrip(ch) - return make_string(s) + return self.make_string(s) def ll_upper(self): # NOT_RPYTHON - return make_string(self._str.upper()) + return self.make_string(self._str.upper()) def ll_lower(self): # NOT_RPYTHON - return make_string(self._str.lower()) + return self.make_string(self._str.lower()) def ll_substring(self, start, count): # NOT_RPYTHON - return make_string(self._str[start:start+count]) + return self.make_string(self._str[start:start+count]) def ll_split_chr(self, ch): # NOT_RPYTHON - res = _list(List(String)) - res._list = [make_string(s) for s in self._str.split(ch)] + res = _list(List(self._TYPE)) + res._list = [self.make_string(s) for s in self._str.split(ch)] return res def ll_contains(self, ch): @@ -1136,7 +1173,7 @@ def ll_replace_chr_chr(self, ch1, ch2): # NOT_RPYTHON - return make_string(self._str.replace(ch1, ch2)) + return self.make_string(self._str.replace(ch1, ch2)) class _null_string(_null_mixin(_string), _string): def __init__(self, STRING): @@ -1154,7 +1191,7 @@ # do nothing def ll_append_char(self, ch): - assert isinstance(ch, str) and len(ch) == 1 + assert isinstance(ch, basestring) and len(ch) == 1 self._buf.append(ch) def ll_append(self, s): @@ -1162,7 +1199,10 @@ self._buf.append(s._str) def ll_build(self): - return make_string(''.join(self._buf)) + if self._TYPE is StringBuilder: + return make_string(''.join(self._buf)) + else: + return make_unicode(u''.join(self._buf)) class _null_string_builder(_null_mixin(_string_builder), _string_builder): def __init__(self, STRING_BUILDER): @@ -1501,7 +1541,8 @@ return inst._identityhash() def oohash(inst): - assert typeOf(inst) is String # for now only strings are supported + assert typeOf(inst) is String or typeOf(inst) is Unicode + # for now only strings and unicode are supported return hash(inst._str) def oostring(obj, base): @@ -1520,6 +1561,17 @@ obj = '<%s object>' % obj._inst._TYPE._name return make_string(str(obj)) +def oounicode(obj, base): + """ + Convert an unichar into an unicode string. + + base must be -1, for consistency with oostring. + """ + assert base == -1 + assert isinstance(obj, unicode) + assert len(obj) == 1 + return make_unicode(obj) + def ooparse_int(s, base): return int(s._str, base) @@ -1548,6 +1600,10 @@ ROOT = Instance('Root', None, _is_root=True) String = String() -StringBuilder = StringBuilder() +Unicode = Unicode() +UnicodeBuilder = StringBuilder(Unicode, UniChar) +StringBuilder = StringBuilder(String, Char) +String.builder = StringBuilder +Unicode.builder = UnicodeBuilder WeakReference = WeakReference() dead_wref = new(WeakReference) Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Fri Nov 9 13:53:02 2007 @@ -1,34 +1,19 @@ +from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rpython.rstr import AbstractStringRepr,AbstractCharRepr,\ AbstractUniCharRepr, AbstractStringIteratorRepr,\ - AbstractLLHelpers + AbstractLLHelpers, AbstractUnicodeRepr from pypy.rpython.rmodel import IntegerRepr -from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar +from pypy.rpython.lltypesystem.lltype import Ptr, Char, UniChar, typeOf,\ + cast_primitive from pypy.rpython.ootypesystem import ootype +from pypy.rpython.rmodel import Repr # TODO: investigate if it's possible and it's worth to concatenate a # String and a Char directly without passing to Char-->String # conversion -class StringRepr(AbstractStringRepr): - """ - Some comments about the state of ootype strings at the end of Tokyo sprint - - What was accomplished: - - The rstr module was split in an lltype and ootype version. - - There is the beginnings of a String type in ootype. - - The runtime representation of Strings is a subclass of the builtin str. - The idea is that this saves us from boilerplate code implementing the - builtin str methods. - - Nothing more was done because of lack of time and paralysis in the face - of too many problems. Among other things, to write any meaningful tests - we first need conversion from Chars to Strings (because - test_llinterp.interpret won't accept strings as arguments). We will need a - new low-level operation (convert_char_to_oostring or some such) for this. - """ - - lowleveltype = ootype.String +class BaseOOStringRepr(Repr): def __init__(self, *args): AbstractStringRepr.__init__(self, *args) @@ -36,13 +21,16 @@ def convert_const(self, value): if value is None: - return ootype.String._null - if not isinstance(value, str): + return self.lowleveltype._null + if not isinstance(value, self.basetype): raise TyperError("not a str: %r" % (value,)) - return ootype.make_string(value) + return self.make_string(value) + + def make_string(self, value): + raise NotImplementedError def make_iterator_repr(self): - return string_iterator_repr + return self.string_iterator_repr def _list_length_items(self, hop, v_lst, LIST): # ootypesystem list has a different interface that @@ -53,17 +41,64 @@ return c_length, v_lst +class StringRepr(BaseOOStringRepr, AbstractStringRepr): + lowleveltype = ootype.String + basetype = str + + def make_string(self, value): + return ootype.make_string(value) + +class UnicodeRepr(BaseOOStringRepr, AbstractUnicodeRepr): + lowleveltype = ootype.Unicode + basetype = basestring + + def make_string(self, value): + return ootype.make_unicode(value) + + def ll_str(self, value): + sb = ootype.new(ootype.StringBuilder) + lgt = value.ll_strlen() + sb.ll_allocate(lgt) + for i in range(lgt): + c = value.ll_stritem_nonneg(i) + if ord(c) > 127: + raise UnicodeEncodeError("%d > 127, not ascii" % ord(c)) + sb.ll_append_char(cast_primitive(Char, c)) + return sb.ll_build() + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char -class UniCharRepr(AbstractUniCharRepr): +class UniCharRepr(AbstractUniCharRepr, UnicodeRepr): lowleveltype = UniChar + +class __extend__(pairtype(UniCharRepr, UnicodeRepr)): + def convert_from_to((r_from, r_to), v, llops): + rstr = llops.rtyper.type_system.rstr + if r_from == unichar_repr and r_to == unicode_repr: + return llops.gendirectcall(r_from.ll.ll_unichr2unicode, v) + return NotImplemented + class LLHelpers(AbstractLLHelpers): def ll_chr2str(ch): return ootype.oostring(ch, -1) + def ll_str2unicode(s): + res = ootype.new(ootype.UnicodeBuilder) + lgt = s.ll_strlen() + res.ll_allocate(lgt) + for i in range(lgt): + c = s.ll_stritem_nonneg(i) + if ord(c) > 127: + raise UnicodeDecodeError + res.ll_append_char(cast_primitive(UniChar, c)) + return res.ll_build() + + def ll_unichr2unicode(ch): + return ootype.oounicode(ch, -1) + def ll_strhash(s): return ootype.oohash(s) @@ -73,7 +108,10 @@ def ll_char_mul(ch, times): if times < 0: times = 0 - buf = ootype.new(ootype.StringBuilder) + if typeOf(ch) == Char: + buf = ootype.new(ootype.StringBuilder) + else: + buf = ootype.new(ootype.UnicodeBuilder) buf.ll_allocate(times) i = 0 while i 2: v_start = hop.inputarg(Signed, arg=2) @@ -118,13 +134,13 @@ return self.rtype_method_find(hop, reverse=True) def rtype_method_count(self, hop): - rstr = hop.rtyper.type_system.rstr - v_str = hop.inputarg(rstr.string_repr, arg=0) + rstr = hop.args_r[0].repr + v_str = hop.inputarg(rstr.repr, arg=0) if hop.args_r[1] == rstr.char_repr: v_value = hop.inputarg(rstr.char_repr, arg=1) llfn = self.ll.ll_count_char else: - v_value = hop.inputarg(rstr.string_repr, arg=1) + v_value = hop.inputarg(rstr.repr, arg=1) llfn = self.ll.ll_count if hop.nb_args > 2: v_start = hop.inputarg(Signed, arg=2) @@ -142,8 +158,8 @@ return hop.gendirectcall(llfn, v_str, v_value, v_start, v_end) def rtype_method_strip(self, hop, left=True, right=True): - rstr = hop.rtyper.type_system.rstr - v_str = hop.inputarg(rstr.string_repr, arg=0) + rstr = hop.args_r[0].repr + v_str = hop.inputarg(rstr.repr, arg=0) v_char = hop.inputarg(rstr.char_repr, arg=1) v_left = hop.inputconst(Bool, left) v_right = hop.inputconst(Bool, right) @@ -156,13 +172,13 @@ return self.rtype_method_strip(hop, left=False, right=True) def rtype_method_upper(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_upper, v_str) def rtype_method_lower(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_lower, v_str) @@ -174,17 +190,17 @@ def rtype_method_join(self, hop): hop.exception_cannot_occur() - rstr = hop.rtyper.type_system.rstr + rstr = hop.args_r[0] if hop.s_result.is_constant(): - return inputconst(rstr.string_repr, hop.s_result.const) + return inputconst(rstr.repr, hop.s_result.const) r_lst = hop.args_r[1] if not isinstance(r_lst, hop.rtyper.type_system.rlist.BaseListRepr): raise TyperError("string.join of non-list: %r" % r_lst) - v_str, v_lst = hop.inputargs(rstr.string_repr, r_lst) + v_str, v_lst = hop.inputargs(rstr.repr, r_lst) v_length, v_items = self._list_length_items(hop, v_lst, r_lst.lowleveltype) if hop.args_s[0].is_constant() and hop.args_s[0].const == '': - if r_lst.item_repr == rstr.string_repr: + if r_lst.item_repr == rstr.repr: llfn = self.ll.ll_join_strs elif r_lst.item_repr == rstr.char_repr: llfn = self.ll.ll_join_chars @@ -192,15 +208,15 @@ raise TyperError("''.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_length, v_items) else: - if r_lst.item_repr == rstr.string_repr: + if r_lst.item_repr == rstr.repr: llfn = self.ll.ll_join else: raise TyperError("sep.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_str, v_length, v_items) def rtype_method_split(self, hop): - rstr = hop.rtyper.type_system.rstr - v_str, v_chr = hop.inputargs(rstr.string_repr, rstr.char_repr) + rstr = hop.args_r[0].repr + v_str, v_chr = hop.inputargs(rstr.repr, rstr.char_repr) try: list_type = hop.r_result.lowleveltype.TO except AttributeError: @@ -210,16 +226,16 @@ return hop.gendirectcall(self.ll.ll_split_chr, cLIST, v_str, v_chr) def rtype_method_replace(self, hop): - rstr = hop.rtyper.type_system.rstr + rstr = hop.args_r[0].repr if not (hop.args_r[1] == rstr.char_repr and hop.args_r[2] == rstr.char_repr): raise TyperError, 'replace only works for char args' - v_str, v_c1, v_c2 = hop.inputargs(rstr.string_repr, rstr.char_repr, rstr.char_repr) + v_str, v_c1, v_c2 = hop.inputargs(rstr.repr, rstr.char_repr, rstr.char_repr) hop.exception_cannot_occur() return hop.gendirectcall(self.ll.ll_replace_chr_chr, v_str, v_c1, v_c2) def rtype_int(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr if hop.nb_args == 1: v_str, = hop.inputargs(string_repr) c_base = inputconst(Signed, 10) @@ -231,9 +247,22 @@ hop.exception_is_here() return hop.gendirectcall(self.ll.ll_int, v_str, v_base) + def rtype_unicode(self, hop): + if hop.args_s[0].is_constant(): + return hop.inputconst(hop.r_result, hop.s_result.const) + repr = hop.args_r[0].repr + v_str = hop.inputarg(repr, 0) + hop.exception_is_here() + return hop.gendirectcall(self.ll.ll_str2unicode, v_str) + + def rtype_method_decode(self, hop): + v_self = hop.inputarg(self, 0) + hop.exception_is_here() + return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) hop.exception_is_here() return hop.gendirectcall(self.ll.ll_float, v_str) @@ -241,6 +270,11 @@ def ll_str(self, s): return s +class __extend__(AbstractUnicodeRepr): + def rtype_method_encode(self, hop): + v_self = hop.inputarg(self, 0) + hop.exception_is_here() + return hop.gendirectcall(self.ll_str, v_self) class __extend__(pairtype(AbstractStringRepr, Repr)): def rtype_mod((r_str, _), hop): @@ -280,73 +314,69 @@ class __extend__(pairtype(AbstractStringRepr, AbstractSliceRepr)): def rtype_getitem((r_str, r_slic), hop): - rstr = hop.rtyper.type_system.rstr + string_repr = r_str.repr rslice = hop.rtyper.type_system.rslice if r_slic == rslice.startonly_slice_repr: - v_str, v_start = hop.inputargs(rstr.string_repr, rslice.startonly_slice_repr) + v_str, v_start = hop.inputargs(string_repr, rslice.startonly_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice_startonly, v_str, v_start) if r_slic == rslice.startstop_slice_repr: - v_str, v_slice = hop.inputargs(rstr.string_repr, rslice.startstop_slice_repr) + v_str, v_slice = hop.inputargs(string_repr, rslice.startstop_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice, v_str, v_slice) if r_slic == rslice.minusone_slice_repr: - v_str, v_ignored = hop.inputargs(rstr.string_repr, rslice.minusone_slice_repr) + v_str, v_ignored = hop.inputargs(string_repr, rslice.minusone_slice_repr) return hop.gendirectcall(r_str.ll.ll_stringslice_minusone, v_str) raise TyperError(r_slic) - class __extend__(pairtype(AbstractStringRepr, AbstractStringRepr)): def rtype_add((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + str1_repr = r_str1.repr + str2_repr = r_str2.repr if hop.s_result.is_constant(): - return hop.inputconst(string_repr, hop.s_result.const) - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + return hop.inputconst(str1_repr, hop.s_result.const) + v_str1, v_str2 = hop.inputargs(str1_repr, str2_repr) return hop.gendirectcall(r_str1.ll.ll_strconcat, v_str1, v_str2) rtype_inplace_add = rtype_add def rtype_eq((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) return hop.gendirectcall(r_str1.ll.ll_streq, v_str1, v_str2) def rtype_ne((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_streq, v_str1, v_str2) return hop.genop('bool_not', [vres], resulttype=Bool) def rtype_lt((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_lt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_le((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_le', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_ge((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_ge', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) def rtype_gt((r_str1, r_str2), hop): - string_repr = hop.rtyper.type_system.rstr.string_repr - v_str1, v_str2 = hop.inputargs(string_repr, string_repr) + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) vres = hop.gendirectcall(r_str1.ll.ll_strcmp, v_str1, v_str2) return hop.genop('int_gt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) -class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr)): +class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr), + pairtype(AbstractUnicodeRepr, AbstractUniCharRepr)): def rtype_contains((r_str, r_chr), hop): - rstr = hop.rtyper.type_system.rstr - v_str, v_chr = hop.inputargs(rstr.string_repr, rstr.char_repr) + string_repr = r_str.repr + char_repr = r_chr.char_repr + v_str, v_chr = hop.inputargs(string_repr, char_repr) return hop.gendirectcall(r_str.ll.ll_contains, v_str, v_chr) class __extend__(pairtype(AbstractStringRepr, AbstractTupleRepr)): @@ -362,7 +392,8 @@ return r_str.ll.do_stringformat(hop, sourcevars) -class __extend__(AbstractCharRepr): +class __extend__(AbstractCharRepr, + AbstractUniCharRepr): def convert_const(self, value): if not isinstance(value, str) or len(value) != 1: @@ -388,13 +419,13 @@ return hop.inputconst(Bool, True) def rtype_ord(_, hop): - rstr = hop.rtyper.type_system.rstr - vlist = hop.inputargs(rstr.char_repr) + repr = hop.args_r[0].char_repr + vlist = hop.inputargs(repr) return hop.genop('cast_char_to_int', vlist, resulttype=Signed) def _rtype_method_isxxx(_, llfn, hop): - rstr = hop.rtyper.type_system.rstr - vlist = hop.inputargs(rstr.char_repr) + repr = hop.args_r[0].char_repr + vlist = hop.inputargs(repr) hop.exception_cannot_occur() return hop.gendirectcall(llfn, vlist[0]) @@ -411,18 +442,20 @@ def rtype_method_islower(self, hop): return self._rtype_method_isxxx(self.ll.ll_char_islower, hop) -class __extend__(pairtype(AbstractCharRepr, IntegerRepr)): +class __extend__(pairtype(AbstractCharRepr, IntegerRepr), + pairtype(AbstractUniCharRepr, IntegerRepr)): def rtype_mul((r_chr, r_int), hop): - rstr = hop.rtyper.type_system.rstr - v_char, v_int = hop.inputargs(rstr.char_repr, Signed) + char_repr = r_chr.char_repr + v_char, v_int = hop.inputargs(char_repr, Signed) return hop.gendirectcall(r_chr.ll.ll_char_mul, v_char, v_int) rtype_inplace_mul = rtype_mul -class __extend__(pairtype(IntegerRepr, AbstractCharRepr)): +class __extend__(pairtype(IntegerRepr, AbstractCharRepr), + pairtype(IntegerRepr, AbstractUniCharRepr)): def rtype_mul((r_int, r_chr), hop): - rstr = hop.rtyper.type_system.rstr - v_int, v_char = hop.inputargs(Signed, rstr.char_repr) + char_repr = r_chr.char_repr + v_int, v_char = hop.inputargs(Signed, char_repr) return hop.gendirectcall(r_chr.ll.ll_char_mul, v_char, v_int) rtype_inplace_mul = rtype_mul @@ -492,10 +525,12 @@ # # _________________________ Conversions _________________________ -class __extend__(pairtype(AbstractCharRepr, AbstractStringRepr)): +class __extend__(pairtype(AbstractCharRepr, AbstractStringRepr), + pairtype(AbstractUniCharRepr, AbstractUnicodeRepr)): def convert_from_to((r_from, r_to), v, llops): rstr = llops.rtyper.type_system.rstr - if r_from == rstr.char_repr and r_to == rstr.string_repr: + if (r_from == rstr.char_repr and r_to == rstr.string_repr) or\ + (r_from == rstr.unichar_repr and r_to == rstr.unicode_repr): return llops.gendirectcall(r_from.ll.ll_chr2str, v) return NotImplemented @@ -519,7 +554,7 @@ class AbstractStringIteratorRepr(IteratorRepr): def newiter(self, hop): - string_repr = hop.rtyper.type_system.rstr.string_repr + string_repr = hop.args_r[0].repr v_str, = hop.inputargs(string_repr) return hop.gendirectcall(self.ll_striter, v_str) @@ -572,7 +607,7 @@ c = ord(ch) return 65 <= c <= 90 - def ll_char_islower(ch): + def ll_char_islower(ch): c = ord(ch) return 97 <= c <= 122 Modified: pypy/dist/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rstr.py (original) +++ pypy/dist/pypy/rpython/test/test_rstr.py Fri Nov 9 13:53:02 2007 @@ -14,26 +14,31 @@ assert parse('%s') == [('s',)] assert parse("name '%s' is not defined") == ["name '", ("s",), "' is not defined"] -class BaseTestRstr(BaseRtypingTest): - +class AbstractTestRstr(BaseRtypingTest): def test_simple(self): + const = self.const def fn(i): - s = 'hello' + s = const('hello') return s[i] for i in range(5): res = self.interpret(fn, [i]) - assert res == 'hello'[i] + expected = fn(i) + assert res == expected + assert res.__class__ is expected.__class__ def test_implicit_index_error(self): + const = self.const def fn(i): - s = 'hello' + s = const('hello') try: return s[i] except IndexError: - return '*' + return const('*') for i in range(-5, 5): res = self.interpret(fn, [i]) - assert res == 'hello'[i] + expected = fn(i) + assert res == expected + assert res.__class__ is expected.__class__ res = self.interpret(fn, [5]) assert res == '*' res = self.interpret(fn, [6]) @@ -42,8 +47,9 @@ assert res == '*' def test_nonzero(self): + const = self.const def fn(i, j): - s = ['', 'xx'][j] + s = [const(''), const('xx')][j] if i < 0: s = None if i > -2: @@ -56,9 +62,10 @@ assert res is fn(i, j) def test_concat(self): + const = self.const def fn(i, j): - s1 = ['', 'a', 'ab'] - s2 = ['', 'x', 'xy'] + s1 = [const(''), const('a'), const('ab')] + s2 = [const(''), const('x'), const('xy')] return s1[i] + s2[j] for i in range(3): for j in range(3): @@ -66,8 +73,9 @@ assert self.ll_to_string(res) == fn(i, j) def test_iter(self): + const = self.const def fn(i): - s = ['', 'a', 'hello'][i] + s = [const(''), const('a'), const('hello')][i] i = 0 for c in s: if c != s[i]: @@ -82,15 +90,17 @@ assert res is True def test_char_constant(self): + const = self.const def fn(s): - return s + '.' - res = self.interpret(fn, ['x']) + return s + const('.') + res = self.interpret(fn, [const('x')]) res = self.ll_to_string(res) assert len(res) == 2 - assert res[0] == 'x' - assert res[1] == '.' + assert res[0] == const('x') + assert res[1] == const('.') def test_char_isxxx(self): + constchar = self.constchar def fn(s): return (s.isspace() | s.isdigit() << 1 | @@ -99,93 +109,52 @@ s.isupper() << 4 | s.islower() << 5) for i in range(128): - ch = chr(i) + ch = constchar(i) res = self.interpret(fn, [ch]) assert res == fn(ch) def test_char_compare(self): - res = self.interpret(lambda c1, c2: c1 == c2, ['a', 'b']) + const = self.const + res = self.interpret(lambda c1, c2: c1 == c2, [const('a'), + const('b')]) assert res is False - res = self.interpret(lambda c1, c2: c1 == c2, ['a', 'a']) + res = self.interpret(lambda c1, c2: c1 == c2, [const('a'), + const('a')]) assert res is True - res = self.interpret(lambda c1, c2: c1 <= c2, ['z', 'a']) + res = self.interpret(lambda c1, c2: c1 <= c2, [const('z'), + const('a')]) assert res is False def test_char_mul(self): + const = self.const def fn(c, mul): s = c * mul res = 0 for i in range(len(s)): - res = res*10 + ord(s[i]) - ord('0') + res = res*10 + ord(const(s[i])) - ord(const('0')) c2 = c c2 *= mul res = 10 * res + (c2 == s) return res - res = self.interpret(fn, ['3', 5]) + res = self.interpret(fn, [const('3'), 5]) assert res == 333331 - res = self.interpret(fn, ['5', 3]) + res = self.interpret(fn, [const('5'), 3]) assert res == 5551 - def test_unichar_const(self): - def fn(c): - return c - assert self.interpret(fn, [u'\u03b1']) == u'\u03b1' - - def test_unichar_eq(self): - def fn(c1, c2): - return c1 == c2 - assert self.interpret(fn, [u'\u03b1', u'\u03b1']) == True - assert self.interpret(fn, [u'\u03b1', u'\u03b2']) == False - - def test_unichar_ord(self): - def fn(c): - return ord(c) - assert self.interpret(fn, [u'\u03b1']) == ord(u'\u03b1') - - def test_unichar_hash(self): - def fn(c): - d = {c: 42} - return d[c] - assert self.interpret(fn, [u'\u03b1']) == 42 - - def test_convert_char_to_unichar(self): - def g(c): - return ord(c) - def fn(n): - if n < 0: - c = unichr(-n) - else: - c = chr(n) - return g(c) - assert self.interpret(fn, [65]) == 65 - assert self.interpret(fn, [-5555]) == 5555 - - def test_char_unichar_eq(self): - def fn(c1, c2): - return c1 == c2 - assert self.interpret(fn, [u'(', '(']) == True - assert self.interpret(fn, [u'\u1028', '(']) == False - assert self.interpret(fn, ['(', u'(']) == True - assert self.interpret(fn, ['(', u'\u1028']) == False - - def test_char_unichar_eq_2(self): - def fn(c1): - return c1 == 'X' - assert self.interpret(fn, [u'(']) == False - assert self.interpret(fn, [u'\u1058']) == False - assert self.interpret(fn, [u'X']) == True - def test_is_none(self): + const = self.const def fn(i): - s1 = ['foo', None][i] + s1 = [const('foo'), None][i] return s1 is None assert self.interpret(fn, [0]) == False assert self.interpret(fn, [1]) == True def test_str_compare(self): + const = self.const def fn(i, j): - s1 = ['one', 'two', None] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar', None] + s1 = [const('one'), const('two'), None] + s2 = [const('one'), const('two'), const('o'), + const('on'), const('twos'), const('foobar'), None] return s1[i] == s2[j] for i in range(3): for j in range(7): @@ -193,8 +162,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] != s2[j] for i in range(2): for j in range(6): @@ -202,8 +171,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] < s2[j] for i in range(2): for j in range(6): @@ -211,8 +180,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] <= s2[j] for i in range(2): for j in range(6): @@ -220,8 +189,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] >= s2[j] for i in range(2): for j in range(6): @@ -229,8 +198,8 @@ assert res is fn(i, j) def fn(i, j): - s1 = ['one', 'two'] - s2 = ['one', 'two', 'o', 'on', 'twos', 'foobar'] + s1 = [const('one'), const('two')] + s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] > s2[j] for i in range(2): for j in range(6): @@ -238,9 +207,10 @@ assert res is fn(i, j) def test_startswith(self): + const = self.const def fn(i, j): - s1 = ['', 'one', 'two'] - s2 = ['', 'one', 'two', 'o', 'on', 'ne', 'e', 'twos', 'foobar', 'fortytwo'] + s1 = [const(''), const('one'), const('two')] + s2 = [const(''), const('one'), const('two'), const('o'), const('on'), const('ne'), const('e'), const('twos'), const('foobar'), const('fortytwo')] return s1[i].startswith(s2[j]) for i in range(3): for j in range(10): @@ -248,9 +218,10 @@ assert res is fn(i, j) def test_endswith(self): + const = self.const def fn(i, j): - s1 = ['', 'one', 'two'] - s2 = ['', 'one', 'two', 'o', 'on', 'ne', 'e', 'twos', 'foobar', 'fortytwo'] + s1 = [const(''), const('one'), const('two')] + s2 = [const(''), const('one'), const('two'), const('o'), const('on'), const('ne'), const('e'), const('twos'), const('foobar'), const('fortytwo')] return s1[i].endswith(s2[j]) for i in range(3): for j in range(10): @@ -258,9 +229,10 @@ assert res is fn(i, j) def test_find(self): + const = self.const def fn(i, j): - s1 = ['one two three', 'abc abcdab abcdabcdabde'] - s2 = ['one', 'two', 'abcdab', 'one tou', 'abcdefgh', 'fortytwo', ''] + s1 = [const('one two three'), const('abc abcdab abcdabcdabde')] + s2 = [const('one'), const('two'), const('abcdab'), const('one tou'), const('abcdefgh'), const('fortytwo'), const('')] return s1[i].find(s2[j]) for i in range(2): for j in range(7): @@ -268,80 +240,89 @@ assert res == fn(i, j) def test_find_with_start(self): + const = self.const def fn(i): assert i >= 0 - return 'ababcabc'.find('abc', i) + return const('ababcabc').find(const('abc'), i) for i in range(9): res = self.interpret(fn, [i]) assert res == fn(i) def test_find_with_start_end(self): + const = self.const def fn(i, j): assert i >= 0 assert j >= 0 - return 'ababcabc'.find('abc', i, j) + return const('ababcabc').find(const('abc'), i, j) for (i, j) in [(1,7), (2,6), (3,7), (3,8)]: res = self.interpret(fn, [i, j]) assert res == fn(i, j) def test_find_empty_string(self): + const = self.const def f(i): assert i >= 0 - s = "abc" - x = s.find('') - x+= s.find('', i)*10 - x+= s.find('', i, i)*100 - x+= s.find('', i, i+1)*1000 + s = const("abc") + x = s.find(const('')) + x+= s.find(const(''), i)*10 + x+= s.find(const(''), i, i)*100 + x+= s.find(const(''), i, i+1)*1000 return x for i in range(5): res = self.interpret(f, [i]) assert res == f(i) def test_rfind(self): + const = self.const def fn(): - return 'aaa'.rfind('a') + 'aaa'.rfind('a', 1) + 'aaa'.rfind('a', 1, 2) + return const('aaa').rfind(const('a')) + const('aaa').rfind(const('a'), 1) + const('aaa').rfind(const('a'), 1, 2) res = self.interpret(fn, []) assert res == 2 + 2 + 1 def test_rfind_empty_string(self): + const = self.const def f(i): assert i >= 0 - s = "abc" - x = s.find('') - x+= s.find('', i)*10 - x+= s.find('', i, i)*100 - x+= s.find('', i, i+1)*1000 + s = const("abc") + x = s.find(const('')) + x+= s.find(const(''), i)*10 + x+= s.find(const(''), i, i)*100 + x+= s.find(const(''), i, i+1)*1000 return x for i in range(5): res = self.interpret(f, [i]) assert res == f(i) def test_find_char(self): + const = self.const def fn(ch): - pos1 = 'aiuwraz 483'.find(ch) - pos2 = 'aiuwraz 483'.rfind(ch) + pos1 = const('aiuwraz 483').find(ch) + pos2 = const('aiuwraz 483').rfind(ch) return pos1 + (pos2*100) - for ch in 'a ?3': + for ch in const('a ?3'): res = self.interpret(fn, [ch]) assert res == fn(ch) def test_strip(self): + const = self.const def both(): - return '!ab!'.strip('!') + return const('!ab!').strip(const('!')) def left(): - return '!ab!'.lstrip('!') + return const('!ab!').lstrip(const('!')) def right(): - return '!ab!'.rstrip('!') + return const('!ab!').rstrip(const('!')) res = self.interpret(both, []) - assert self.ll_to_string(res) == 'ab' + assert self.ll_to_string(res) == const('ab') res = self.interpret(left, []) - assert self.ll_to_string(res) == 'ab!' + assert self.ll_to_string(res) == const('ab!') res = self.interpret(right, []) - assert self.ll_to_string(res) == '!ab' + assert self.ll_to_string(res) == const('!ab') def test_upper(self): - strings = ['', ' ', 'upper', 'UpPeR', ',uppEr,'] - for i in range(256): strings.append(chr(i)) + const = self.const + constchar = self.constchar + strings = [const(''), const(' '), const('upper'), const('UpPeR'), const(',uppEr,')] + for i in range(256): strings.append(constchar(i)) def fn(i): return strings[i].upper() for i in range(len(strings)): @@ -349,7 +330,8 @@ assert self.ll_to_string(res) == fn(i) def test_lower(self): - strings = ['', ' ', 'lower', 'LoWeR', ',lowEr,'] + const = self.const + strings = [const(''), const(' '), const('lower'), const('LoWeR'), const(',lowEr,')] for i in range(256): strings.append(chr(i)) def fn(i): return strings[i].lower() @@ -358,21 +340,22 @@ assert self.ll_to_string(res) == fn(i) def test_join(self): - res = self.interpret(lambda: ''.join([]), []) + const = self.const + res = self.interpret(lambda: const('').join([]), []) assert self.ll_to_string(res) == "" - res = self.interpret(lambda: ''.join(['a', 'b', 'c']), []) + res = self.interpret(lambda: const('').join([const('a'), const('b'), const('c')]), []) assert self.ll_to_string(res) == "abc" - res = self.interpret(lambda: ''.join(['abc', 'de', 'fghi']), []) + res = self.interpret(lambda: const('').join([const('abc'), const('de'), const('fghi')]), []) assert self.ll_to_string(res) == "abcdefghi" - res = self.interpret(lambda: '.'.join(['abc', 'def']), []) - assert self.ll_to_string(res) == 'abc.def' + res = self.interpret(lambda: const('.').join([const('abc'), const('def')]), []) + assert self.ll_to_string(res) == const('abc.def') def fn(i, j): - s1 = [ '', ',', ' and '] - s2 = [ [], ['foo'], ['bar', 'baz', 'bazz']] + s1 = [ const(''), const(','), const(' and ')] + s2 = [ [], [const('foo')], [const('bar'), const('baz'), const('bazz')]] return s1[i].join(s2[j]) for i in range(3): for j in range(3): @@ -380,9 +363,9 @@ assert self.ll_to_string(res) == fn(i, j) def fn(i, j): - s1 = [ '', ',', ' and '] - s2 = [ [], ['foo'], ['bar', 'baz', 'bazz']] - s2[1].extend(['x']) + s1 = [ const(''), const(','), const(' and ')] + s2 = [ [], [const('foo')], [const('bar'), const('baz'), const('bazz')]] + s2[1].extend([const('x')]) return s1[i].join(s2[j]) for i in range(3): for j in range(3): @@ -390,73 +373,77 @@ assert self.ll_to_string(res) == fn(i, j) def test_str_slice(self): + const = self.const def fn(): - s = 'hello' + s = const('hello') s1 = s[:3] s2 = s[3:] s3 = s[3:10] - return s1+s2 == s and s2+s1 == 'lohel' and s1+s3 == s + return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s res = self.interpret(fn, ()) assert res def test_str_slice_minusone(self): + const = self.const def fn(): - s = 'hello' - z = 'h' + s = const('hello') + z = const('h') return s[:-1]+z[:-1] res = self.interpret(fn, ()) - assert self.ll_to_string(res) == 'hell' + assert self.ll_to_string(res) == const('hell') def test_strformat(self): + const = self.const def percentS(s): - return "before %s after" % (s,) + return const("before %s after") % (s,) - res = self.interpret(percentS, ['1']) - assert self.ll_to_string(res) == 'before 1 after' + res = self.interpret(percentS, [const('1')]) + assert self.ll_to_string(res) == const('before 1 after') def percentD(i): return "bing %d bang" % (i,) res = self.interpret(percentD, [23]) - assert self.ll_to_string(res) == 'bing 23 bang' + assert self.ll_to_string(res) == const('bing 23 bang') def percentX(i): - return "bing %x bang" % (i,) + return const("bing %x bang") % (i,) res = self.interpret(percentX, [23]) - assert self.ll_to_string(res) == 'bing 17 bang' + assert self.ll_to_string(res) == const('bing 17 bang') res = self.interpret(percentX, [-123]) - assert self.ll_to_string(res) == 'bing -7b bang' + assert self.ll_to_string(res) == const('bing -7b bang') def percentO(i): - return "bing %o bang" % (i,) + return const("bing %o bang") % (i,) res = self.interpret(percentO, [23]) - assert self.ll_to_string(res) == 'bing 27 bang' + assert self.ll_to_string(res) == const('bing 27 bang') res = self.interpret(percentO, [-123]) - assert self.ll_to_string(res) == 'bing -173 bang' + assert self.ll_to_string(res) == const('bing -173 bang') def moreThanOne(s, d, x, o): - return "string: %s decimal: %d hex: %x oct: %o" % (s, d, x, o) + return const("string: %s decimal: %d hex: %x oct: %o") % (s, d, x, o) - args = 'a', 2, 3, 4 + args = const('a'), 2, 3, 4 res = self.interpret(moreThanOne, list(args)) assert self.ll_to_string(res) == moreThanOne(*args) def test_strformat_nontuple(self): + const = self.const def percentD(i): - return "before %d after" % i + return const("before %d after") % i res = self.interpret(percentD, [1]) - assert self.ll_to_string(res) == 'before 1 after' + assert self.ll_to_string(res) == const('before 1 after') def percentS(i): - return "before %s after" % i + return const("before %s after") % i - res = self.interpret(percentS, ['D']) - assert self.ll_to_string(res) == 'before D after' + res = self.interpret(percentS, [const('D')]) + assert self.ll_to_string(res) == const('before D after') def test_strformat_instance(self): class C: @@ -532,52 +519,58 @@ assert res == expected def test_split(self): + const = self.const def fn(i): - s = ['', '0.1.2.4.8', '.1.2', '1.2.', '.1.2.4.'][i] - l = s.split('.') + s = [const(''), const('0.1.2.4.8'), const('.1.2'), const('1.2.'), const('.1.2.4.')][i] + l = s.split(const('.')) sum = 0 for num in l: if len(num): - sum += ord(num) - ord('0') + sum += ord(num) - ord(const('0')) return sum + len(l) * 100 for i in range(5): res = self.interpret(fn, [i]) assert res == fn(i) def test_contains(self): + const = self.const + constchar = self.constchar def fn(i): - s = 'Hello world' - return chr(i) in s + s = const('Hello world') + return constchar(i) in s for i in range(256): res = self.interpret(fn, [i])#, view=i==42) assert res == fn(i) def test_replace(self): + const = self.const def fn(c1, c2): - s = 'abbccc' + s = const('abbccc') s = s.replace(c1, c2) res = 0 for c in s: if c == c2: res += 1 return res - res = self.interpret(fn, ['a', 'c']) + res = self.interpret(fn, [const('a'), const('c')]) assert res == 4 - res = self.interpret(fn, ['c', 'b']) + res = self.interpret(fn, [const('c'), const('b')]) assert res == 5 def test_replace_TyperError(self): + const = self.const def fn(): - s = 'abbccc' - s = s.replace('a', 'baz') + s = const('abbccc') + s = s.replace(const('a'), const('baz')) raises(TyperError, self.interpret, fn, ()) def fn(): - s = 'abbccc' - s = s.replace('abb', 'c') + s = const('abbccc') + s = s.replace(const('abb'), const('c')) raises(TyperError, self.interpret, fn, ()) def test_int(self): - s1 = [ '42', '01001', 'abc', 'ABC', '4aBc', ' 12ef ', '+42', 'foo', '42foo', '42.1', '', '+ 42'] + const = self.const + s1 = [ const('42'), const('01001'), const('abc'), const('ABC'), const('4aBc'), const(' 12ef '), const('+42'), const('foo'), const('42foo'), const('42.1'), const(''), const('+ 42')] def fn(i, base): s = s1[i] res = int(s, base) @@ -593,7 +586,8 @@ assert res == expected def test_int_valueerror(self): - s1 = ['42g', '?', '+', '+ '] + const = self.const + s1 = [const('42g'), const('?'), const('+'), const('+ ')] def fn(i): try: return int(s1[i]) @@ -607,7 +601,8 @@ assert res == -654 def test_float(self): - f = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] + const = self.const + f = [const(''), const(' '), const('0'), const('1'), const('-1.5'), const('1.5E2'), const('2.5e-1'), const(' 0 '), const('?')] def fn(i): s = f[i] return float(s) @@ -622,34 +617,38 @@ assert res == expected def test_char_mul_n(self): + const = self.const def f(c, n): return c*n - res = self.interpret(f, ['a', 4]) + res = self.interpret(f, [const('a'), 4]) assert self.ll_to_string(res) == 'a'*4 - res = self.interpret(f, ['a', 0]) + res = self.interpret(f, [const('a'), 0]) assert self.ll_to_string(res) == "" def test_char_mul_negative(self): + const = self.const def f(c): return c * -3 - res = self.interpret(f, ['a']) + res = self.interpret(f, [const('a')]) assert self.ll_to_string(res) == '' def test_n_mul_char(self): + const = self.const def f(c, n): return n*c - res = self.interpret(f, ['a', 4]) + res = self.interpret(f, [const('a'), 4]) assert self.ll_to_string(res) == 'a'*4 - res = self.interpret(f, ['a', 0]) + res = self.interpret(f, [const('a'), 0]) assert self.ll_to_string(res) == "" def test_hash(self): + const = self.const def fn(i): if i == 0: - s = '' + s = const('') else: - s = "xxx" + s = const("xxx") return hash(s) res = self.interpret(fn, [0]) assert res == self.EMPTY_STRING_HASH @@ -657,45 +656,52 @@ assert typeOf(res) == Signed def test_call_str_on_string(self): + const = self.const def fn(i): - s = "x" * i - return str(s) + s = const("x") * i + return const(s) res = self.interpret(fn, [3]) assert self.ll_to_string(res) == 'xxx' def test_count_char(self): + const = self.const def fn(i): - s = "".join(["abcasd"] * i) - return s.count("a") + s.count("a", 2) + s.count("b", 1, 6) + s = const("").join([const("abcasd")] * i) + return s.count(const("a")) + s.count(const("a"), 2) + \ + s.count(const("b"), 1, 6) res = self.interpret(fn, [4]) assert res == 8 + 7 + 1 def test_count(self): + const = self.const def fn(i): - s = "".join(["abcabsd"] * i) + s = const("").join([const("abcabsd")] * i) one = i / i # confuse the annotator - return (s.count("abc") + "abcde".count("") + - "abcda".count("a" * one)) + return (s.count(const("abc")) + const("abcde").count(const("")) + + const("abcda").count(const("a") * one)) res = self.interpret(fn, [4]) assert res == 4 + 6 + 2 def test_count_overlapping_occurences(self): + const = self.const def fn(): - return 'ababa'.count('aba') + return const('ababa').count(const('aba')) res = self.interpret(fn, []) assert res == 1 def test_hlstr(self): + const = self.const from pypy.rpython.annlowlevel import hlstr def f(s): - return "*"+hlstr(s)+"*" == "*abba*" + return const("*")+const(hlstr(s))+const("*") == const("*abba*") - res = self.interpret(f, [self.string_to_ll("abba")]) + res = self.interpret(f, [self.string_to_ll(const("abba"))]) assert res def test_getitem_exc(self): + const = self.const def f(x): - s = "z" + s = const("z") return s[x] res = self.interpret(f, [0]) @@ -708,13 +714,13 @@ assert False def f(x): - s = "z" + s = const("z") try: return s[x] except IndexError: - return 'X' + return const('X') except Exception: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -722,11 +728,11 @@ assert res == 'X' def f(x): - s = "z" + s = const("z") try: return s[x] except Exception: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -734,11 +740,11 @@ assert res == ' ' def f(x): - s = "z" + s = const("z") try: return s[x] except ValueError: - return ' ' + return const(' ') res = self.interpret(f, [0]) assert res == 'z' @@ -750,10 +756,11 @@ assert False def test_fold_concat(self): + const = self.const def g(tail): - return "head"+tail + return const("head")+tail def f(): - return g("tail") + return g(const("tail")) from pypy import conftest t, typer, fgraph = self.gengraph(f, [], backendopt=True) @@ -779,6 +786,10 @@ res = interpret(g, [-2]) assert res._obj.value == 42 +class BaseTestRstr(AbstractTestRstr): + const = str + constchar = chr + class TestLLtype(BaseTestRstr, LLRtypeMixin): EMPTY_STRING_HASH = -1 Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Fri Nov 9 13:53:02 2007 @@ -1,19 +1,193 @@ -from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin +from pypy.rpython.test.tool import LLRtypeMixin, OORtypeMixin +from pypy.rpython.test.test_rstr import AbstractTestRstr +import py -class BaseTestRUnicode(BaseRtypingTest): - def test_simple(self): - def f(n): - if n % 2 == 0: - x = 'xxx' - else: - x = u'x\u221Ex' - return x[n] - - for i in range(0, 3): - res = self.interpret(f, [i]) - assert res == f(i) +# ====> test_rstr.py + +class BaseTestRUnicode(AbstractTestRstr): + const = unicode + constchar = unichr + + def test_unicode_explicit_conv(self): + def f(x): + return unicode(x) + + for v in ['x', u'x']: + res = self.interpret(f, [v]) + assert self.ll_to_unicode(res) == v + + def f(x): + if x > 1: + y = const('yxx') + else: + y = const('xx') + return unicode(y) + + const = str + assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + + def f(x): + if x > 1: + y = const('yxx') + else: + y = const('xx') + return unicode(y) + + # a copy, because llinterp caches functions + + const = unicode + assert self.ll_to_unicode(self.interpret(f, [1])) == f(1) + + def test_str_unicode_const(self): + def f(): + return str(u'xxx') + + assert self.ll_to_string(self.interpret(f, [])) == 'xxx' + + def test_conversion_errors(self): + py.test.skip("do we want this test to pass?") + def f(x): + if x: + string = '\x80\x81' + uni = u'\x80\x81' + else: + string = '\x82\x83' + uni = u'\x83\x84\x84' + try: + str(uni) + except UnicodeEncodeError: + pass + else: + return -1 + try: + unicode(string) + except UnicodeDecodeError: + return len(string) + len(uni) + else: + return -2 + assert f(True) == 4 + assert f(False) == 5 + res = self.interpret(f, [True]) + assert res == 4 + + + def test_str_unicode_nonconst(self): + def f(x): + y = u'xxx' + unichr(x) + return str(y) + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + self.interpret_raises(UnicodeEncodeError, f, [1234]) + + def test_unicode_encode(self): + def f(x): + y = u'xxx' + return (y + unichr(x)).encode('ascii') + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + + def test_unicode_encode_error(self): + def f(x): + y = u'xxx' + try: + x = (y + unichr(x)).encode('ascii') + return len(x) + except UnicodeEncodeError: + return -1 + + assert self.interpret(f, [38]) == f(38) + assert self.interpret(f, [138]) == f(138) + + def test_unicode_decode(self): + def f(x): + y = 'xxx' + return (y + chr(x)).decode('ascii') + + assert self.ll_to_string(self.interpret(f, [38])) == f(38) + + def test_unicode_decode_error(self): + def f(x): + y = 'xxx' + try: + x = (y + chr(x)).decode('ascii') + return len(x) + except UnicodeDecodeError: + return -1 + + assert self.interpret(f, [38]) == f(38) + assert self.interpret(f, [138]) == f(138) + + + def test_unichar_const(self): + def fn(c): + return c + assert self.interpret(fn, [u'\u03b1']) == u'\u03b1' + + def test_unichar_eq(self): + def fn(c1, c2): + return c1 == c2 + assert self.interpret(fn, [u'\u03b1', u'\u03b1']) == True + assert self.interpret(fn, [u'\u03b1', u'\u03b2']) == False + + def test_unichar_ord(self): + def fn(c): + return ord(c) + assert self.interpret(fn, [u'\u03b1']) == ord(u'\u03b1') + + def test_unichar_hash(self): + def fn(c): + d = {c: 42} + return d[c] + assert self.interpret(fn, [u'\u03b1']) == 42 + + def test_convert_char_to_unichar(self): + def g(c): + return ord(c) + def fn(n): + if n < 0: + c = unichr(-n) + else: + c = chr(n) + return g(c) + assert self.interpret(fn, [65]) == 65 + assert self.interpret(fn, [-5555]) == 5555 + + def test_char_unichar_eq(self): + def fn(c1, c2): + return c1 == c2 + assert self.interpret(fn, [u'(', '(']) == True + assert self.interpret(fn, [u'\u1028', '(']) == False + assert self.interpret(fn, ['(', u'(']) == True + assert self.interpret(fn, ['(', u'\u1028']) == False + + def test_char_unichar_eq_2(self): + def fn(c1): + return c1 == 'X' + assert self.interpret(fn, [u'(']) == False + assert self.interpret(fn, [u'\u1058']) == False + assert self.interpret(fn, [u'X']) == True + + def unsupported(self): + py.test.skip("not supported") + + test_char_isxxx = unsupported + test_upper = unsupported + test_lower = unsupported + test_strformat = unsupported + test_strformat_instance = unsupported + test_strformat_nontuple = unsupported + test_percentformat_instance = unsupported + test_percentformat_tuple = unsupported + test_percentformat_list = unsupported + test_int = unsupported + test_int_valueerror = unsupported + test_float = unsupported + test_hlstr = unsupported class TestLLtype(BaseTestRUnicode, LLRtypeMixin): - pass + EMPTY_STRING_HASH = -1 + +class TestOOtype(BaseTestRUnicode, OORtypeMixin): + EMPTY_STRING_HASH = 0 Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Fri Nov 9 13:53:02 2007 @@ -34,10 +34,17 @@ def ll_to_string(self, s): return ''.join(s.chars) + def ll_to_unicode(self, s): + return u''.join(s.chars) + def string_to_ll(self, s): from pypy.rpython.module.support import LLSupport return LLSupport.to_rstr(s) + def unicode_to_ll(self, s): + from pypy.rpython.module.support import LLSupport + return LLSupport.to_runicode(s) + def ll_to_list(self, l): r = [] items = l.ll_items() @@ -75,6 +82,8 @@ def ll_to_string(self, s): return s._str + ll_to_unicode = ll_to_string + def string_to_ll(self, s): from pypy.rpython.module.support import OOSupport return OOSupport.to_rstr(s) Modified: pypy/dist/pypy/translator/test/test_geninterp.py ============================================================================== --- pypy/dist/pypy/translator/test/test_geninterp.py (original) +++ pypy/dist/pypy/translator/test/test_geninterp.py Fri Nov 9 13:53:02 2007 @@ -44,6 +44,9 @@ def import_sys_func(): import sys return sys.__name__ + + def unicode_test(x): + return unicode(x, 'ascii') """ def setup_class(cls): @@ -275,3 +278,9 @@ fn = self.build_interpfunc(snippet.t_neg_long) result = fn() assert result == -132L and type(result) is long + + def test_unicode_with_encoding(self): + fn = self.build_interpfunc(snippet.unicode_test) + result = fn("abc") + assert result == u"abc" and type(result) is unicode + From fijal at codespeak.net Fri Nov 9 14:02:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Nov 2007 14:02:10 +0100 (CET) Subject: [pypy-svn] r48467 - pypy/branch/unicode-objspace Message-ID: <20071109130210.1AD5981E2@code0.codespeak.net> Author: fijal Date: Fri Nov 9 14:02:09 2007 New Revision: 48467 Added: pypy/branch/unicode-objspace/ - copied from r48466, pypy/dist/ Log: Create a new branch for unicode as an objspace operation. From antocuni at codespeak.net Fri Nov 9 14:02:15 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Nov 2007 14:02:15 +0100 (CET) Subject: [pypy-svn] r48468 - pypy/dist/pypy/config Message-ID: <20071109130215.BEDDA81E7@code0.codespeak.net> Author: antocuni Date: Fri Nov 9 14:02:15 2007 New Revision: 48468 Modified: pypy/dist/pypy/config/translationoption.py Log: raisingop2direct_call works with ootype Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Nov 9 14:02:15 2007 @@ -17,7 +17,7 @@ ChoiceOption("type_system", "Type system to use when RTyping", ["lltype", "ootype"], cmdline=None, requires={ - "ootype": [("translation.backendopt.raisingop2direct_call", False), + "ootype": [ ("translation.backendopt.constfold", False), ("translation.backendopt.heap2stack", False), ("translation.backendopt.clever_malloc_removal", False), From ericvrp at codespeak.net Fri Nov 9 14:05:26 2007 From: ericvrp at codespeak.net (ericvrp at codespeak.net) Date: Fri, 9 Nov 2007 14:05:26 +0100 (CET) Subject: [pypy-svn] r48469 - pypy/dist/pypy/translator/llvm Message-ID: <20071109130526.DF9EC80DE@code0.codespeak.net> Author: ericvrp Date: Fri Nov 9 14:05:26 2007 New Revision: 48469 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py Log: small fix to make most llvm tests pass not only on Darwin but on linux too. Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Fri Nov 9 14:05:26 2007 @@ -128,8 +128,8 @@ self.cmds.append("gcc -O3 %s.o %s %s -lm -bundle -o %s.so" % (base, gc_libs_path, gc_libs, base)) else: - gc_libs_path = '-static' - XXX + gc_libs_path = '-shared' + self.cmds.append("gcc -O3 %s.o %s %s -pipe -o %s.so" % (base, gc_libs_path, gc_libs, base)) try: self.execute_cmds() From antocuni at codespeak.net Fri Nov 9 14:05:55 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Nov 2007 14:05:55 +0100 (CET) Subject: [pypy-svn] r48470 - pypy/dist/pypy/translator/cli/test Message-ID: <20071109130555.6921A8147@code0.codespeak.net> Author: antocuni Date: Fri Nov 9 14:05:54 2007 New Revision: 48470 Modified: pypy/dist/pypy/translator/cli/test/test_op.py Log: a failing test that has been around my wc for ages Modified: pypy/dist/pypy/translator/cli/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_op.py (original) +++ pypy/dist/pypy/translator/cli/test/test_op.py Fri Nov 9 14:05:54 2007 @@ -1,7 +1,19 @@ +import sys from pypy.translator.cli.test.runtest import CliTest from pypy.translator.oosupport.test_template.operations import BaseTestOperations +from pypy.rlib.rarithmetic import ovfcheck # ====> ../../oosupport/test_template/operations.py class TestOperations(CliTest, BaseTestOperations): - pass + def test_int_div_overflow(self): + import py + py.test.skip('fixme!') + def fn(x, y): + try: + return x//y + except OverflowError: + return 42 + res = self.interpret(fn, [-sys.maxint-1, -1]) + assert res == 42 + From antocuni at codespeak.net Fri Nov 9 17:32:27 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Nov 2007 17:32:27 +0100 (CET) Subject: [pypy-svn] r48474 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/ootypesystem rpython/ootypesystem/test rpython/test translator/cli translator/cli/src translator/cli/src/stub translator/cli/test translator/oosupport Message-ID: <20071109163227.372E381E0@code0.codespeak.net> Author: antocuni Date: Fri Nov 9 17:32:25 2007 New Revision: 48474 Added: pypy/dist/pypy/translator/cli/test/test_unicode.py (contents, props changed) Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/ootypesystem/ooregistry.py pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/rpython/ootypesystem/test/test_oostring.py pypy/dist/pypy/rpython/test/test_runicode.py pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/cli/ilgenerator.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/cli/prebuiltnodes.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/src/stub/main.il pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/oosupport/constant.py Log: - use oounicode to convert from string to unicode - add support for unicode to cli backend Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Nov 9 17:32:25 2007 @@ -1087,7 +1087,10 @@ return ootype.oostring(obj, base) def op_oounicode(self, obj, base): - return ootype.oounicode(obj, base) + try: + return ootype.oounicode(obj, base) + except UnicodeDecodeError: + self.make_llexception() def op_ooparse_int(self, s, base): try: Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Fri Nov 9 17:32:25 2007 @@ -468,8 +468,9 @@ 'ooidentityhash': LLOp(oo=True, sideeffects=False), 'oostring': LLOp(oo=True, sideeffects=False), 'ooparse_int': LLOp(oo=True, canraise=(ValueError,)), - 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), + 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), 'oohash': LLOp(oo=True, sideeffects=False), + 'oounicode': LLOp(oo=True, canraise=(UnicodeDecodeError,)), # _____ read frame var support ___ 'get_frame_base': LLOp(sideeffects=False), Modified: pypy/dist/pypy/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ooregistry.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ooregistry.py Fri Nov 9 17:32:25 2007 @@ -28,12 +28,15 @@ _about_ = ootype.oounicode def compute_result_annotation(self, obj_s, base_s): - assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) + assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) or \ + (isinstance(obj_s, annmodel.SomeOOInstance) + and obj_s.ootype in (ootype.String, ootype.Unicode)) assert isinstance(base_s, annmodel.SomeInteger) return annmodel.SomeOOInstance(ootype.Unicode) def specialize_call(self, hop): - assert isinstance(hop.args_s[0],annmodel.SomeUnicodeCodePoint) + assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint, + annmodel.SomeOOInstance)) vlist = hop.inputargs(hop.args_r[0], ootype.Signed) return hop.genop('oounicode', vlist, resulttype = ootype.Unicode) Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Fri Nov 9 17:32:25 2007 @@ -1563,14 +1563,21 @@ def oounicode(obj, base): """ - Convert an unichar into an unicode string. + Convert: + - an unichar into an unicode string OR + - a string into an unicode string base must be -1, for consistency with oostring. """ assert base == -1 - assert isinstance(obj, unicode) - assert len(obj) == 1 - return make_unicode(obj) + if isinstance(obj, unicode): + assert len(obj) == 1 + return make_unicode(obj) + elif isinstance(obj, _string): + s = unicode(obj._str) + return make_unicode(s) + else: + assert False def ooparse_int(s, base): return int(s._str, base) Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Fri Nov 9 17:32:25 2007 @@ -86,15 +86,7 @@ return ootype.oostring(ch, -1) def ll_str2unicode(s): - res = ootype.new(ootype.UnicodeBuilder) - lgt = s.ll_strlen() - res.ll_allocate(lgt) - for i in range(lgt): - c = s.ll_stritem_nonneg(i) - if ord(c) > 127: - raise UnicodeDecodeError - res.ll_append_char(cast_primitive(UniChar, c)) - return res.ll_build() + return ootype.oounicode(s, -1) def ll_unichr2unicode(ch): return ootype.oounicode(ch, -1) Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_oostring.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_oostring.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_oostring.py Fri Nov 9 17:32:25 2007 @@ -1,3 +1,4 @@ +import py from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.rstr import string_repr from pypy.rpython.test.test_llinterp import interpret @@ -26,3 +27,17 @@ res = interpret(f, [], type_system='ootype') assert res._str == 'foo' + +def test_oounicode(): + u = ootype.oounicode(u'a', -1) + assert isinstance(u, ootype._string) + assert ootype.typeOf(u) is ootype.Unicode + + s = ootype.make_string('a string') + u = ootype.oounicode(s, -1) + assert isinstance(u, ootype._string) + assert ootype.typeOf(u) is ootype.Unicode + + s = ootype.make_string('non-ascii string: \xe0') + py.test.raises(UnicodeDecodeError, ootype.oounicode, s, -1) + Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Fri Nov 9 17:32:25 2007 @@ -186,6 +186,10 @@ test_float = unsupported test_hlstr = unsupported + def test_find_empty_string(self): + py.test.skip("We should think how to solve this problem") + test_rfind_empty_string = test_find_empty_string + class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Fri Nov 9 17:32:25 2007 @@ -132,6 +132,8 @@ ootype.Class: types.type, ootype.String: types.string, ootype.StringBuilder: types.string_builder, + ootype.Unicode: types.string, + ootype.UnicodeBuilder: types.string_builder, ootype.WeakReference: types.weakref, # maps generic types to their ordinal Modified: pypy/dist/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/dist/pypy/translator/cli/ilgenerator.py (original) +++ pypy/dist/pypy/translator/cli/ilgenerator.py Fri Nov 9 17:32:25 2007 @@ -397,7 +397,7 @@ ilasm.opcode('ldc.i4', str(value)) elif TYPE in (ootype.SignedLongLong, ootype.UnsignedLongLong): ilasm.opcode('ldc.i8', str(value)) - elif TYPE is ootype.String: + elif TYPE in (ootype.String, ootype.Unicode): if value._str is None: ilasm.opcode('ldnull') else: Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Fri Nov 9 17:32:25 2007 @@ -55,7 +55,7 @@ generator.load(arg) # XXX: very hackish, need refactoring - if this.concretetype is ootype.String: + if this.concretetype in (ootype.String, ootype.Unicode): # special case for string: don't use methods, but plain functions METH = this.concretetype._METHODS[method_name] cts = generator.cts @@ -106,6 +106,17 @@ generator.load(op.args[1]) generator.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) +class _OOUnicode(MicroInstruction): + def render(self, generator, op): + from pypy.objspace.flow.model import Constant + ARGTYPE = op.args[0].concretetype + argtype = generator.cts.lltype_to_cts(ARGTYPE) + v_base = op.args[1] + assert v_base.value == -1, "The second argument of oounicode must be -1" + + generator.load(op.args[0]) + generator.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + class _NewCustomDict(MicroInstruction): def render(self, generator, op): DICT = op.args[0].value @@ -226,6 +237,7 @@ IndirectCall = _IndirectCall() RuntimeNew = _RuntimeNew() OOString = _OOString() +OOUnicode = _OOUnicode() NewCustomDict = _NewCustomDict() #CastWeakAdrToPtr = _CastWeakAdrToPtr() Box = _Box() Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Fri Nov 9 17:32:25 2007 @@ -1,7 +1,7 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, OOString, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive + TypeOf, CastPrimitive, OOUnicode from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive from pypy.translator.cli.cts import WEAKREF @@ -50,6 +50,7 @@ 'ooidentityhash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oohash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oostring': [OOString], + 'oounicode': [OOUnicode], 'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': [PushAllArgs, 'call float64 [pypylib]pypy.runtime.Utils::OOParseFloat(string)'], 'oonewcustomdict': [NewCustomDict], Modified: pypy/dist/pypy/translator/cli/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/cli/prebuiltnodes.py Fri Nov 9 17:32:25 2007 @@ -25,10 +25,14 @@ def raise_ZeroDivisionError(): raise ZeroDivisionError +def raise_UnicodeDecodeError(): + raise UnicodeDecodeError + HELPERS = [(raise_RuntimeError, []), (raise_OverflowError, []), (raise_ValueError, []), (raise_ZeroDivisionError, []), + (raise_UnicodeDecodeError, []), ] def _build_helpers(translator, db): Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Nov 9 17:32:25 2007 @@ -28,6 +28,9 @@ } } + public static string ToPython_unicode(string x) { return "u" + ToPython(x); } + public static string ToPython_unicode(char x) { return "u" + ToPython(x); } + public static string ToPython(object x) { if (x == null) return "None"; @@ -103,6 +106,24 @@ return b.ToString(); } + private static void check_ascii(char ch) + { + if ((int)ch > 127) + Helpers.raise_UnicodeDecodeError(); + } + + public static string OOUnicode(char ch) + { + return ch.ToString(); + } + + public static string OOUnicode(string s) + { + foreach(char ch in s) + check_ascii(ch); + return s; + } + public static int OOParseInt(string s, int base_) { return Convert.ToInt32(s, base_); Modified: pypy/dist/pypy/translator/cli/src/stub/main.il ============================================================================== --- pypy/dist/pypy/translator/cli/src/stub/main.il (original) +++ pypy/dist/pypy/translator/cli/src/stub/main.il Fri Nov 9 17:32:25 2007 @@ -58,5 +58,13 @@ throw ret } + + .method public static void raise_UnicodeDecodeError() il managed + { + ldstr "This is only a stub, it should not be called" + newobj instance void class [mscorlib]System.ApplicationException::.ctor(string) + throw + ret + } } } Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Fri Nov 9 17:32:25 2007 @@ -28,6 +28,12 @@ def format_object(TYPE, cts, ilasm): if TYPE is ootype.Void: ilasm.opcode('ldstr "None"') + elif TYPE in (ootype.Unicode, ootype.UniChar): + # the CLI type for Unicode is the very same as for + # ootype.String, so we can't rely on overloading to + # distinguish + type_ = cts.lltype_to_cts(TYPE) + ilasm.call('string class [pypylib]pypy.test.Result::ToPython_unicode(%s)' % type_) else: if isinstance(TYPE, (ootype.BuiltinType, ootype.Instance, ootype.StaticMethod)) and TYPE is not ootype.String: type_ = 'object' @@ -296,6 +302,9 @@ def ll_to_string(self, s): return s + def ll_to_unicode(self, s): + return s + def ll_to_list(self, l): return l Added: pypy/dist/pypy/translator/cli/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/test/test_unicode.py Fri Nov 9 17:32:25 2007 @@ -0,0 +1,20 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_runicode import BaseTestRUnicode + +# ====> ../../../rpython/test/test_runicode.py + +class TestCliUnicode(CliTest, BaseTestRUnicode): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("CLI interpret doesn't support unicode for input arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_getitem_exc(self): + py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Fri Nov 9 17:32:25 2007 @@ -32,7 +32,7 @@ PRIMITIVE_TYPES = set([ootype.Void, ootype.Bool, ootype.Char, ootype.UniChar, ootype.Float, ootype.Signed, ootype.Unsigned, - ootype.String, ootype.SignedLongLong, + ootype.String, ootype.Unicode, ootype.SignedLongLong, ootype.UnsignedLongLong]) def is_primitive(TYPE): From antocuni at codespeak.net Fri Nov 9 18:00:54 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Nov 2007 18:00:54 +0100 (CET) Subject: [pypy-svn] r48475 - in pypy/branch/unicode-objspace/pypy: rpython rpython/lltypesystem rpython/ootypesystem rpython/ootypesystem/test rpython/test translator/cli translator/cli/src translator/cli/src/stub translator/cli/test translator/oosupport Message-ID: <20071109170054.BE9C08168@code0.codespeak.net> Author: antocuni Date: Fri Nov 9 18:00:53 2007 New Revision: 48475 Added: pypy/branch/unicode-objspace/pypy/translator/cli/test/test_unicode.py - copied unchanged from r48474, pypy/dist/pypy/translator/cli/test/test_unicode.py Modified: pypy/branch/unicode-objspace/pypy/rpython/llinterp.py pypy/branch/unicode-objspace/pypy/rpython/lltypesystem/lloperation.py pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ooregistry.py pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ootype.py pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/rstr.py pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/test/test_oostring.py pypy/branch/unicode-objspace/pypy/rpython/test/test_runicode.py pypy/branch/unicode-objspace/pypy/translator/cli/cts.py pypy/branch/unicode-objspace/pypy/translator/cli/ilgenerator.py pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py pypy/branch/unicode-objspace/pypy/translator/cli/prebuiltnodes.py pypy/branch/unicode-objspace/pypy/translator/cli/src/pypylib.cs pypy/branch/unicode-objspace/pypy/translator/cli/src/stub/main.il pypy/branch/unicode-objspace/pypy/translator/cli/test/runtest.py pypy/branch/unicode-objspace/pypy/translator/oosupport/constant.py Log: apply checkin 48474 also to this branch: svn merge svn+ssh://codespeak.net/svn/pypy/dist/pypy at 48473 svn+ssh://codespeak.net/svn/pypy/dist/pypy at 48474 Modified: pypy/branch/unicode-objspace/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/llinterp.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/llinterp.py Fri Nov 9 18:00:53 2007 @@ -1087,7 +1087,10 @@ return ootype.oostring(obj, base) def op_oounicode(self, obj, base): - return ootype.oounicode(obj, base) + try: + return ootype.oounicode(obj, base) + except UnicodeDecodeError: + self.make_llexception() def op_ooparse_int(self, s, base): try: Modified: pypy/branch/unicode-objspace/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/lltypesystem/lloperation.py Fri Nov 9 18:00:53 2007 @@ -468,8 +468,9 @@ 'ooidentityhash': LLOp(oo=True, sideeffects=False), 'oostring': LLOp(oo=True, sideeffects=False), 'ooparse_int': LLOp(oo=True, canraise=(ValueError,)), - 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), + 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), 'oohash': LLOp(oo=True, sideeffects=False), + 'oounicode': LLOp(oo=True, canraise=(UnicodeDecodeError,)), # _____ read frame var support ___ 'get_frame_base': LLOp(sideeffects=False), Modified: pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ooregistry.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ooregistry.py Fri Nov 9 18:00:53 2007 @@ -28,12 +28,15 @@ _about_ = ootype.oounicode def compute_result_annotation(self, obj_s, base_s): - assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) + assert isinstance(obj_s, annmodel.SomeUnicodeCodePoint) or \ + (isinstance(obj_s, annmodel.SomeOOInstance) + and obj_s.ootype in (ootype.String, ootype.Unicode)) assert isinstance(base_s, annmodel.SomeInteger) return annmodel.SomeOOInstance(ootype.Unicode) def specialize_call(self, hop): - assert isinstance(hop.args_s[0],annmodel.SomeUnicodeCodePoint) + assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint, + annmodel.SomeOOInstance)) vlist = hop.inputargs(hop.args_r[0], ootype.Signed) return hop.genop('oounicode', vlist, resulttype = ootype.Unicode) Modified: pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/ootype.py Fri Nov 9 18:00:53 2007 @@ -1563,14 +1563,21 @@ def oounicode(obj, base): """ - Convert an unichar into an unicode string. + Convert: + - an unichar into an unicode string OR + - a string into an unicode string base must be -1, for consistency with oostring. """ assert base == -1 - assert isinstance(obj, unicode) - assert len(obj) == 1 - return make_unicode(obj) + if isinstance(obj, unicode): + assert len(obj) == 1 + return make_unicode(obj) + elif isinstance(obj, _string): + s = unicode(obj._str) + return make_unicode(s) + else: + assert False def ooparse_int(s, base): return int(s._str, base) Modified: pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/rstr.py Fri Nov 9 18:00:53 2007 @@ -86,15 +86,7 @@ return ootype.oostring(ch, -1) def ll_str2unicode(s): - res = ootype.new(ootype.UnicodeBuilder) - lgt = s.ll_strlen() - res.ll_allocate(lgt) - for i in range(lgt): - c = s.ll_stritem_nonneg(i) - if ord(c) > 127: - raise UnicodeDecodeError - res.ll_append_char(cast_primitive(UniChar, c)) - return res.ll_build() + return ootype.oounicode(s, -1) def ll_unichr2unicode(ch): return ootype.oounicode(ch, -1) Modified: pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/test/test_oostring.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/test/test_oostring.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/ootypesystem/test/test_oostring.py Fri Nov 9 18:00:53 2007 @@ -1,3 +1,4 @@ +import py from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.rstr import string_repr from pypy.rpython.test.test_llinterp import interpret @@ -26,3 +27,17 @@ res = interpret(f, [], type_system='ootype') assert res._str == 'foo' + +def test_oounicode(): + u = ootype.oounicode(u'a', -1) + assert isinstance(u, ootype._string) + assert ootype.typeOf(u) is ootype.Unicode + + s = ootype.make_string('a string') + u = ootype.oounicode(s, -1) + assert isinstance(u, ootype._string) + assert ootype.typeOf(u) is ootype.Unicode + + s = ootype.make_string('non-ascii string: \xe0') + py.test.raises(UnicodeDecodeError, ootype.oounicode, s, -1) + Modified: pypy/branch/unicode-objspace/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rpython/test/test_runicode.py (original) +++ pypy/branch/unicode-objspace/pypy/rpython/test/test_runicode.py Fri Nov 9 18:00:53 2007 @@ -186,6 +186,10 @@ test_float = unsupported test_hlstr = unsupported + def test_find_empty_string(self): + py.test.skip("We should think how to solve this problem") + test_rfind_empty_string = test_find_empty_string + class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 Modified: pypy/branch/unicode-objspace/pypy/translator/cli/cts.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/cts.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/cts.py Fri Nov 9 18:00:53 2007 @@ -132,6 +132,8 @@ ootype.Class: types.type, ootype.String: types.string, ootype.StringBuilder: types.string_builder, + ootype.Unicode: types.string, + ootype.UnicodeBuilder: types.string_builder, ootype.WeakReference: types.weakref, # maps generic types to their ordinal Modified: pypy/branch/unicode-objspace/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/ilgenerator.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/ilgenerator.py Fri Nov 9 18:00:53 2007 @@ -397,7 +397,7 @@ ilasm.opcode('ldc.i4', str(value)) elif TYPE in (ootype.SignedLongLong, ootype.UnsignedLongLong): ilasm.opcode('ldc.i8', str(value)) - elif TYPE is ootype.String: + elif TYPE in (ootype.String, ootype.Unicode): if value._str is None: ilasm.opcode('ldnull') else: Modified: pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py Fri Nov 9 18:00:53 2007 @@ -55,7 +55,7 @@ generator.load(arg) # XXX: very hackish, need refactoring - if this.concretetype is ootype.String: + if this.concretetype in (ootype.String, ootype.Unicode): # special case for string: don't use methods, but plain functions METH = this.concretetype._METHODS[method_name] cts = generator.cts @@ -106,6 +106,17 @@ generator.load(op.args[1]) generator.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) +class _OOUnicode(MicroInstruction): + def render(self, generator, op): + from pypy.objspace.flow.model import Constant + ARGTYPE = op.args[0].concretetype + argtype = generator.cts.lltype_to_cts(ARGTYPE) + v_base = op.args[1] + assert v_base.value == -1, "The second argument of oounicode must be -1" + + generator.load(op.args[0]) + generator.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + class _NewCustomDict(MicroInstruction): def render(self, generator, op): DICT = op.args[0].value @@ -226,6 +237,7 @@ IndirectCall = _IndirectCall() RuntimeNew = _RuntimeNew() OOString = _OOString() +OOUnicode = _OOUnicode() NewCustomDict = _NewCustomDict() #CastWeakAdrToPtr = _CastWeakAdrToPtr() Box = _Box() Modified: pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py Fri Nov 9 18:00:53 2007 @@ -1,7 +1,7 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, OOString, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive + TypeOf, CastPrimitive, OOUnicode from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive from pypy.translator.cli.cts import WEAKREF @@ -50,6 +50,7 @@ 'ooidentityhash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oohash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oostring': [OOString], + 'oounicode': [OOUnicode], 'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': [PushAllArgs, 'call float64 [pypylib]pypy.runtime.Utils::OOParseFloat(string)'], 'oonewcustomdict': [NewCustomDict], Modified: pypy/branch/unicode-objspace/pypy/translator/cli/prebuiltnodes.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/prebuiltnodes.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/prebuiltnodes.py Fri Nov 9 18:00:53 2007 @@ -25,10 +25,14 @@ def raise_ZeroDivisionError(): raise ZeroDivisionError +def raise_UnicodeDecodeError(): + raise UnicodeDecodeError + HELPERS = [(raise_RuntimeError, []), (raise_OverflowError, []), (raise_ValueError, []), (raise_ZeroDivisionError, []), + (raise_UnicodeDecodeError, []), ] def _build_helpers(translator, db): Modified: pypy/branch/unicode-objspace/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/src/pypylib.cs Fri Nov 9 18:00:53 2007 @@ -28,6 +28,9 @@ } } + public static string ToPython_unicode(string x) { return "u" + ToPython(x); } + public static string ToPython_unicode(char x) { return "u" + ToPython(x); } + public static string ToPython(object x) { if (x == null) return "None"; @@ -103,6 +106,24 @@ return b.ToString(); } + private static void check_ascii(char ch) + { + if ((int)ch > 127) + Helpers.raise_UnicodeDecodeError(); + } + + public static string OOUnicode(char ch) + { + return ch.ToString(); + } + + public static string OOUnicode(string s) + { + foreach(char ch in s) + check_ascii(ch); + return s; + } + public static int OOParseInt(string s, int base_) { return Convert.ToInt32(s, base_); Modified: pypy/branch/unicode-objspace/pypy/translator/cli/src/stub/main.il ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/src/stub/main.il (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/src/stub/main.il Fri Nov 9 18:00:53 2007 @@ -58,5 +58,13 @@ throw ret } + + .method public static void raise_UnicodeDecodeError() il managed + { + ldstr "This is only a stub, it should not be called" + newobj instance void class [mscorlib]System.ApplicationException::.ctor(string) + throw + ret + } } } Modified: pypy/branch/unicode-objspace/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/test/runtest.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/test/runtest.py Fri Nov 9 18:00:53 2007 @@ -28,6 +28,12 @@ def format_object(TYPE, cts, ilasm): if TYPE is ootype.Void: ilasm.opcode('ldstr "None"') + elif TYPE in (ootype.Unicode, ootype.UniChar): + # the CLI type for Unicode is the very same as for + # ootype.String, so we can't rely on overloading to + # distinguish + type_ = cts.lltype_to_cts(TYPE) + ilasm.call('string class [pypylib]pypy.test.Result::ToPython_unicode(%s)' % type_) else: if isinstance(TYPE, (ootype.BuiltinType, ootype.Instance, ootype.StaticMethod)) and TYPE is not ootype.String: type_ = 'object' @@ -296,6 +302,9 @@ def ll_to_string(self, s): return s + def ll_to_unicode(self, s): + return s + def ll_to_list(self, l): return l Modified: pypy/branch/unicode-objspace/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/oosupport/constant.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/oosupport/constant.py Fri Nov 9 18:00:53 2007 @@ -32,7 +32,7 @@ PRIMITIVE_TYPES = set([ootype.Void, ootype.Bool, ootype.Char, ootype.UniChar, ootype.Float, ootype.Signed, ootype.Unsigned, - ootype.String, ootype.SignedLongLong, + ootype.String, ootype.Unicode, ootype.SignedLongLong, ootype.UnsignedLongLong]) def is_primitive(TYPE): From cfbolz at codespeak.net Fri Nov 9 19:02:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 19:02:05 +0100 (CET) Subject: [pypy-svn] r48476 - pypy/branch/unicode-objspace Message-ID: <20071109180205.7AA998208@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 19:02:04 2007 New Revision: 48476 Removed: pypy/branch/unicode-objspace/ Log: kill this branch, as it was created in a potentially confusing way. From cfbolz at codespeak.net Fri Nov 9 19:02:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Nov 2007 19:02:30 +0100 (CET) Subject: [pypy-svn] r48477 - pypy/branch/unicode-objspace Message-ID: <20071109180230.840408207@code0.codespeak.net> Author: cfbolz Date: Fri Nov 9 19:02:30 2007 New Revision: 48477 Added: pypy/branch/unicode-objspace/ - copied from r48476, pypy/dist/ Log: branch again From fijal at codespeak.net Fri Nov 9 22:38:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Nov 2007 22:38:16 +0100 (CET) Subject: [pypy-svn] r48478 - pypy/dist/pypy/translator/backendopt Message-ID: <20071109213816.4770F81EB@code0.codespeak.net> Author: fijal Date: Fri Nov 9 22:38:14 2007 New Revision: 48478 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py Log: This fixes translation, but I fail to write test for that :-( Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Fri Nov 9 22:38:14 2007 @@ -391,6 +391,8 @@ return fromcrep.constant return None result = 0 + import py + py.test.pdb() for block in graph.iterblocks(): seen_setvar = setop.args[0] in block.inputargs for op in block.operations: @@ -398,6 +400,8 @@ seen_setvar = True if not op.opname.startswith("malloc"): continue + if op.args[1].value['flavor'] != 'gc': + continue if adi.getstate(op.result).get_crep(checksingle=True) is not tocrep: continue TYPE = op.result.concretetype.TO From fijal at codespeak.net Fri Nov 9 22:47:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Nov 2007 22:47:29 +0100 (CET) Subject: [pypy-svn] r48479 - pypy/dist/pypy/translator/backendopt Message-ID: <20071109214729.8CA2B81FA@code0.codespeak.net> Author: fijal Date: Fri Nov 9 22:47:29 2007 New Revision: 48479 Modified: pypy/dist/pypy/translator/backendopt/coalloc.py Log: Don't check in the pdb Modified: pypy/dist/pypy/translator/backendopt/coalloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/coalloc.py (original) +++ pypy/dist/pypy/translator/backendopt/coalloc.py Fri Nov 9 22:47:29 2007 @@ -391,8 +391,6 @@ return fromcrep.constant return None result = 0 - import py - py.test.pdb() for block in graph.iterblocks(): seen_setvar = setop.args[0] in block.inputargs for op in block.operations: From cfbolz at codespeak.net Sat Nov 10 00:24:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 00:24:59 +0100 (CET) Subject: [pypy-svn] r48482 - in pypy/branch/unicode-objspace/pypy: interpreter module/__builtin__ module/_sre module/unicodedata objspace objspace/cpy objspace/cpy/test objspace/fake objspace/std objspace/std/test Message-ID: <20071109232459.1023A81B8@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 00:24:58 2007 New Revision: 48482 Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py pypy/branch/unicode-objspace/pypy/module/_sre/interp_sre.py pypy/branch/unicode-objspace/pypy/module/unicodedata/interp_ucd.py pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py pypy/branch/unicode-objspace/pypy/objspace/dump.py pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py pypy/branch/unicode-objspace/pypy/objspace/logic.py pypy/branch/unicode-objspace/pypy/objspace/std/default.py pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py pypy/branch/unicode-objspace/pypy/objspace/thunk.py Log: plotch. refactor the std object space to use rpython unicode objects to represent unicode objects. Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py Sat Nov 10 00:24:58 2007 @@ -1037,7 +1037,6 @@ # newtuple([w_1, w_2,...]) -> w_tuple # newlist([w_1, w_2,...]) -> w_list # newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) -# newunicode([i1, i2,...]) -> w_unicode from integers # newdict() -> empty w_dict # newslice(w_start,w_stop,w_step) -> w_slice # call_args(w_obj,Arguments()) -> w_result @@ -1049,7 +1048,7 @@ 'float_w', 'uint_w', 'bigint_w', - 'unichars_w', + 'unicode_w', 'interpclass_w', 'unwrap', 'is_true', @@ -1057,7 +1056,6 @@ 'newtuple', 'newlist', 'newstring', - 'newunicode', 'newdict', 'newslice', 'call_args', Modified: pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py (original) +++ pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py Sat Nov 10 00:24:58 2007 @@ -25,7 +25,7 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("unichr() arg out of range")) - return space.newunicode([c]) + return space.wrap(c) unichr.unwrap_spec = [ObjSpace, int] def len(space, w_obj): Modified: pypy/branch/unicode-objspace/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/unicode-objspace/pypy/module/_sre/interp_sre.py Sat Nov 10 00:24:58 2007 @@ -150,11 +150,11 @@ rsre.insert_sre_methods(locals(), 'unicode') def unwrap_object(self): - self.unichars = self.space.unichars_w(self.w_string) - return len(self.unichars) + self.unicode = self.space.unicode_w(self.w_string) + return len(self.unicode) def get_char_ord(self, p): - return ord(self.unichars[p]) + return ord(self.unicode[p]) class W_GenericState(W_State): Modified: pypy/branch/unicode-objspace/pypy/module/unicodedata/interp_ucd.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/unicodedata/interp_ucd.py (original) +++ pypy/branch/unicode-objspace/pypy/module/unicodedata/interp_ucd.py Sat Nov 10 00:24:58 2007 @@ -215,10 +215,10 @@ result[0] = ch if not composed: # If decomposed normalization we are done - return space.newunicode([unichr(i) for i in result[:j]]) + return space.wrap(u''.join([unichr(i) for i in result[:j]])) if j <= 1: - return space.newunicode([unichr(i) for i in result[:j]]) + return space.wrap(u''.join([unichr(i) for i in result[:j]])) current = result[0] starter_pos = 0 @@ -268,7 +268,7 @@ result[starter_pos] = current - return space.newunicode([unichr(i) for i in result[:next_insert]]) + return space.wrap(u''.join([unichr(i) for i in result[:next_insert]])) normalize.unwrap_spec = ['self', ObjSpace, str, W_Root] Modified: pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py Sat Nov 10 00:24:58 2007 @@ -71,6 +71,9 @@ return PyInt_FromLong(x) if isinstance(x, str): return PyString_FromStringAndSize(x, len(x)) + if isinstance(x, str): + # XXX fix me + raise NotImplementedError if isinstance(x, float): return PyFloat_FromDouble(x) if isinstance(x, r_uint): @@ -206,7 +209,7 @@ buf[i] = p[i] return buf.raw - def unichars_w(self, w_obj): + def unicode_w(self, w_obj): not_implemented_sorry def call_function(self, w_callable, *args_w): @@ -234,13 +237,6 @@ buf[i] = chr(self.int_w(bytes_w[i])) return PyString_FromStringAndSize(buf, length) - def newunicode(self, codes): - # XXX inefficient - lst = [PyUnicode_FromOrdinal(ord(code)) for code in codes] - w_lst = self.newlist(lst) - w_emptyunicode = PyUnicode_FromUnicode(None, 0) - return self.call_method(w_emptyunicode, 'join', w_lst) - def newint(self, intval): return PyInt_FromLong(intval) Modified: pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py Sat Nov 10 00:24:58 2007 @@ -54,23 +54,25 @@ w = space.newstring([space.wrap(65), space.wrap(66)]) assert space.str_w(w) == 'AB' -def test_newunicode(): +def test_wrapunicode(): + py.test.skip("fix me") space = CPyObjSpace() - w = space.newunicode([unichr(65), unichr(66)]) + w = space.wrap(unichr(65) + unichr(66)) assert space.is_w(space.type(w), space.w_unicode) for i in range(2): code = space.int_w(space.ord(space.getitem(w, space.wrap(i)))) assert code == 65+i def test_ord(): + py.test.skip("fix me") space = CPyObjSpace() w = space.wrap('A') assert space.int_w(space.ord(w)) == 65 w = space.wrap('\x00') assert space.int_w(space.ord(w)) == 0 - w = space.newunicode([unichr(65)]) + w = space.wrap(unichr(65)) assert space.int_w(space.ord(w)) == 65 - w = space.newunicode([unichr(0)]) + w = space.wrap(unichr(0)) assert space.int_w(space.ord(w)) == 0 def test_id(): Modified: pypy/branch/unicode-objspace/pypy/objspace/dump.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/dump.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/dump.py Sat Nov 10 00:24:58 2007 @@ -145,7 +145,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -154,7 +154,6 @@ 'newtuple': 0, 'newlist': 0, 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, @@ -166,7 +165,6 @@ 'newtuple': True, 'newlist': True, 'newstring': True, - 'newunicode': True, 'newdict': True, 'newslice': True, 'call_args': True, Modified: pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py Sat Nov 10 00:24:58 2007 @@ -23,7 +23,7 @@ uint_dummy = make_dummy(r_uint(42), r_uint(43)) str_dummy = make_dummy('foo', 'bar') bool_dummy = make_dummy(True, False) -unichars_dummy = make_dummy([u'a', u'b'], [u'c', u'd']) +unicode_dummy = make_dummy(u'abc', u'cde') bigint_dummy = make_dummy(rbigint([0]), rbigint([1])) class FakeObjSpace(ObjSpace): @@ -75,7 +75,7 @@ int_w = int_dummy uint_w = uint_dummy float_w = float_dummy - unichars_w = unichars_dummy + unicode_w = unicode_dummy bigint_w = bigint_dummy iter = make_dummy() type = make_dummy() @@ -89,7 +89,6 @@ call_args = make_dummy() new_interned_str = make_dummy() newstring = make_dummy() - newunicode = make_dummy() newint = make_dummy() newlong = make_dummy() newfloat = make_dummy() Modified: pypy/branch/unicode-objspace/pypy/objspace/logic.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/logic.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/logic.py Sat Nov 10 00:24:58 2007 @@ -49,7 +49,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -58,7 +58,6 @@ 'newtuple': 0, 'newlist': 0, 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, Modified: pypy/branch/unicode-objspace/pypy/objspace/std/default.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/default.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/default.py Sat Nov 10 00:24:58 2007 @@ -35,9 +35,9 @@ raise OperationError(space.w_TypeError, typed_unwrap_error_msg(space, "integer", w_obj)) -def unichars_w__ANY(space,w_obj): +def unicode_w__ANY(space,w_obj): raise OperationError(space.w_TypeError, - typed_unwrap_error_msg(space, "string", w_obj)) + typed_unwrap_error_msg(space, "unicode", w_obj)) def bigint_w__ANY(space,w_obj): raise OperationError(space.w_TypeError, Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Sat Nov 10 00:24:58 2007 @@ -176,10 +176,7 @@ if self.w_valuedict is None: raise OperationError(space.w_TypeError, space.wrap("format requires a mapping")) - if do_unicode: - w_key = space.newunicode(key) - else: - w_key = space.wrap(key) + w_key = space.wrap(key) return space.getitem(self.w_valuedict, w_key) def parse_fmt(self): @@ -301,7 +298,7 @@ if do_unicode: w_defaultencoding = space.call_function( space.sys.get('getdefaultencoding')) - w_s = space.call_method(space.newunicode([c]), + w_s = space.call_method(space.wrap(c), "encode", w_defaultencoding, space.wrap('replace')) @@ -371,7 +368,7 @@ else: if not got_unicode: w_value = space.call_function(space.w_unicode, w_value) - s = space.unichars_w(w_value) + s = space.unicode_w(w_value) self.std_wp(s) def fmt_r(self, w_value): @@ -389,7 +386,7 @@ elif space.is_true(space.isinstance(w_value, space.w_unicode)): if not do_unicode: raise NeedUnicodeFormattingError - lst = space.unichars_w(w_value) + lst = space.unicode_w(w_value) if len(lst) != 1: raise OperationError(space.w_TypeError, space.wrap("%c requires int or unichar")) @@ -442,10 +439,10 @@ else: return space.wrap(''.join(result)) else: - fmt = space.unichars_w(w_fmt) + fmt = space.unicode_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() - return space.newunicode(result) + return space.wrap(''.join(result)) def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): Modified: pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py Sat Nov 10 00:24:58 2007 @@ -400,7 +400,7 @@ from pypy.objspace.std.stringtype import wrapstr return wrapstr(self, x) if isinstance(x, unicode): - return W_UnicodeObject([unichr(ord(u)) for u in x]) # xxx + return W_UnicodeObject(x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, Wrappable): @@ -533,9 +533,6 @@ self.wrap("character code not in range(256)")) return self.wrap(''.join(chars)) - def newunicode(self, chars): - return W_UnicodeObject(chars) - def newseqiter(self, w_obj): return W_SeqIterObject(w_obj) @@ -653,7 +650,7 @@ str_w = StdObjSpaceMultiMethod('str_w', 1, []) # returns an unwrapped string float_w = StdObjSpaceMultiMethod('float_w', 1, []) # returns an unwrapped float uint_w = StdObjSpaceMultiMethod('uint_w', 1, []) # returns an unwrapped unsigned int (r_uint) - unichars_w = StdObjSpaceMultiMethod('unichars_w', 1, []) # returns an unwrapped list of unicode characters + unicode_w = StdObjSpaceMultiMethod('unicode_w', 1, []) # returns an unwrapped list of unicode characters bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint # NOTE: when adding more sometype_w() methods, you need to write a # stub in default.py to raise a space.w_TypeError Modified: pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py Sat Nov 10 00:24:58 2007 @@ -36,6 +36,22 @@ W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)] del i +def _decode_ascii(space, s): + try: + return s.decode("ascii") + except UnicodeDecodeError: + for i in range(len(s)): + if ord(s[i]) > 127: + break + raise OperationError( + space.w_UnicodeDecodeError, + space.wrap(("'ascii' codec can't decode byte %s in position %s:" + " ordinal not in range(128)") % (hex(ord(i)), i))) + +def unicode_w__String(space, w_self): + # XXX should this use the default encoding? + return _decode_ascii(space, w_self._value) + def _is_generic(space, w_self, fun): v = w_self._value Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 00:24:58 2007 @@ -21,9 +21,6 @@ check(u'a' + 'b', u'ab') check('a' + u'b', u'ab') - def test_hash(self): - assert hash(u'') == 0 - def test_join(self): def check(a, b): assert a == b @@ -278,3 +275,62 @@ else: raise Exception("DID NOT RAISE") + def test_startswith(self): + assert u'ab'.startswith(u'ab') is True + assert u'ab'.startswith(u'a') is True + assert u'ab'.startswith(u'') is True + assert u'x'.startswith(u'a') is False + assert u'x'.startswith(u'x') is True + assert u''.startswith(u'') is True + assert u''.startswith(u'a') is False + assert u'x'.startswith(u'xx') is False + assert u'y'.startswith(u'xx') is False + + def test_startswith_more(self): + assert u'ab'.startswith(u'a', 0) is True + assert u'ab'.startswith(u'a', 1) is False + assert u'ab'.startswith(u'b', 1) is True + assert u'abc'.startswith(u'bc', 1, 2) is False + assert u'abc'.startswith(u'c', -1, 4) is True + + def test_startswith_tuples(self): + assert u'hello'.startswith((u'he', u'ha')) + assert not u'hello'.startswith((u'lo', u'llo')) + assert u'hello'.startswith((u'hellox', u'hello')) + assert not u'hello'.startswith(()) + assert u'helloworld'.startswith((u'hellowo', u'rld', u'lowo'), 3) + assert not u'helloworld'.startswith((u'hellowo', u'ello', u'rld'), 3) + assert u'hello'.startswith((u'lo', u'he'), 0, -1) + assert not u'hello'.startswith((u'he', u'hel'), 0, 1) + assert u'hello'.startswith((u'he', u'hel'), 0, 2) + raises(TypeError, u'hello'.startswith, (42,)) + + def test_endswith(self): + assert u'ab'.endswith(u'ab') is True + assert u'ab'.endswith(u'b') is True + assert u'ab'.endswith(u'') is True + assert u'x'.endswith(u'a') is False + assert u'x'.endswith(u'x') is True + assert u''.endswith(u'') is True + assert u''.endswith(u'a') is False + assert u'x'.endswith(u'xx') is False + assert u'y'.endswith(u'xx') is False + + def test_endswith_more(self): + assert u'abc'.endswith(u'ab', 0, 2) is True + assert u'abc'.endswith(u'bc', 1) is True + assert u'abc'.endswith(u'bc', 2) is False + assert u'abc'.endswith(u'b', -3, -1) is True + + def test_endswith_tuple(self): + assert not u'hello'.endswith((u'he', u'ha')) + assert u'hello'.endswith((u'lo', u'llo')) + assert u'hello'.endswith((u'hellox', u'hello')) + assert not u'hello'.endswith(()) + assert u'helloworld'.endswith((u'hellowo', u'rld', u'lowo'), 3) + assert not u'helloworld'.endswith((u'hellowo', u'ello', u'rld'), 3, -1) + assert u'hello'.endswith((u'hell', u'ell'), 0, -1) + assert not u'hello'.endswith((u'he', u'hel'), 0, 1) + assert u'hello'.endswith((u'he', u'hell'), 0, 4) + raises(TypeError, u'hello'.endswith, (42,)) + Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 00:24:58 2007 @@ -7,23 +7,25 @@ from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.module.unicodedata import unicodedb_3_2_0 as unicodedb +from pypy.tool.sourcetools import func_with_new_name from pypy.objspace.std.formatting import mod_format class W_UnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef - def __init__(w_self, unicodechars): - w_self._value = unicodechars - w_self.w_hash = None + def __init__(w_self, unistr): + assert isinstance(unistr, unicode) + w_self._value = unistr + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%r)" % (w_self.__class__.__name__, w_self._value) def unwrap(w_self, space): - # For faked functions taking unicodearguments. - # Remove when we no longer need faking. - return u''.join(w_self._value) + # for testing + return w_self._value +W_UnicodeObject.EMPTY = W_UnicodeObject(u'') registerimplementation(W_UnicodeObject) @@ -63,7 +65,7 @@ def str_w__Unicode(space, w_uni): return space.str_w(space.str(w_uni)) -def unichars_w__Unicode(space, w_uni): +def unicode_w__Unicode(space, w_uni): return w_uni._value def str__Unicode(space, w_uni): @@ -75,11 +77,7 @@ def lt__Unicode_Unicode(space, w_left, w_right): left = w_left._value right = w_right._value - for i in range(min(len(left), len(right))): - if left[i] != right[i]: - return space.newbool(ord(left[i]) < ord(right[i])) - # NB. 'unichar < unichar' is not RPython at the moment - return space.newbool(len(left) < len(right)) + return space.newbool(left < right) def ord__Unicode(space, w_uni): if len(w_uni._value) != 1: @@ -107,108 +105,58 @@ contains__Rope_Unicode = contains__String_Unicode -def _find(self, sub, start, end): - if len(sub) == 0: - return start - if start >= end: - return -1 - for i in range(start, end - len(sub) + 1): - for j in range(len(sub)): - if self[i + j] != sub[j]: - break - else: - return i - return -1 - -def _rfind(self, sub, start, end): - if len(sub) == 0: - return end - if end - start < len(sub): - return -1 - for i in range(end - len(sub), start - 1, -1): - for j in range(len(sub)): - if self[i + j] != sub[j]: - break - else: - return i - return -1 - def contains__Unicode_Unicode(space, w_container, w_item): item = w_item._value container = w_container._value - return space.newbool(_find(container, item, 0, len(container)) >= 0) + return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - list = space.unpackiterable(w_list) + l = space.unpackiterable(w_list) delim = w_self._value totlen = 0 - if len(list) == 0: - return W_UnicodeObject([]) - if (len(list) == 1 and - space.is_w(space.type(list[0]), space.w_unicode)): - return list[0] - - values_list = [None] * len(list) - values_list[0] = [u'\0'] - for i in range(len(list)): - item = list[i] + if len(l) == 0: + return W_UnicodeObject.EMPTY + if (len(l) == 1 and + space.is_w(space.type(l[0]), space.w_unicode)): + return l[0] + + values_list = [] + for i in range(len(l)): + item = l[i] if space.is_true(space.isinstance(item, space.w_unicode)): - pass + item = item._value elif space.is_true(space.isinstance(item, space.w_str)): - item = space.call_function(space.w_unicode, item) + item = space.unicode_w(item) else: w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - assert isinstance(item, W_UnicodeObject) - item = item._value - totlen += len(item) - values_list[i] = item - totlen += len(delim) * (len(values_list) - 1) - if len(values_list) == 1: - return W_UnicodeObject(values_list[0]) - # Allocate result - result = [u'\0'] * totlen - first = values_list[0] - for i in range(len(first)): - result[i] = first[i] - offset = len(first) - for i in range(1, len(values_list)): - item = values_list[i] - # Add delimiter - for j in range(len(delim)): - result[offset + j] = delim[j] - offset += len(delim) - # Add item from values_list - for j in range(len(item)): - result[offset + j] = item[j] - offset += len(item) - return W_UnicodeObject(result) - + values_list.append(item) + return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): - if w_uni.w_hash is None: - # hrmpf - chars = w_uni._value - if len(chars) == 0: + s = w_uni._value + if space.config.objspace.std.withrope: + # be compatible with the special ropes hash + # XXX no caching + if len(s) == 0: return space.wrap(0) - if space.config.objspace.std.withrope: - x = 0 - for c in chars: - x = intmask((1000003 * x) + ord(c)) - x <<= 1 - x ^= len(chars) - x ^= ord(chars[0]) - h = intmask(x) - else: - x = ord(chars[0]) << 7 - for c in chars: - x = intmask((1000003 * x) ^ ord(c)) - h = intmask(x ^ len(chars)) - if h == -1: - h = -2 - w_uni.w_hash = space.wrap(h) - return w_uni.w_hash + x = 0 + for c in s: + x = intmask((1000003 * x) + ord(c)) + x <<= 1 + x ^= len(s) + x ^= ord(s[0]) + h = intmask(x) + return space.wrap(h) + if we_are_translated(): + x = hash(s) # to use the hash cache in rpython strings + else: + from pypy.rlib.rarithmetic import _hash_string + x = _hash_string(s) # to make sure we get the same hash as rpython + # (otherwise translation will freeze W_DictObjects where we can't find + # the keys any more!) + return space.wrap(x) def len__Unicode(space, w_uni): return space.wrap(len(w_uni._value)) @@ -223,7 +171,7 @@ exc = space.call_function(space.w_IndexError, space.wrap("unicode index out of range")) raise OperationError(space.w_IndexError, exc) - return W_UnicodeObject([uni[ival]]) + return W_UnicodeObject(uni[ival]) def getitem__Unicode_Slice(space, w_uni, w_slice): uni = w_uni._value @@ -235,7 +183,7 @@ assert start >= 0 and stop >= 0 r = uni[start:stop] else: - r = [uni[start + i*step] for i in range(sl)] + r = u"".join([uni[start + i*step] for i in range(sl)]) return W_UnicodeObject(r) def mul__Unicode_ANY(space, w_uni, w_times): @@ -245,18 +193,13 @@ if e.match(space, space.w_TypeError): raise FailedToImplement raise - chars = w_uni._value - charlen = len(chars) - if times <= 0 or charlen == 0: - return W_UnicodeObject([]) - if times == 1: - return space.call_function(space.w_unicode, w_uni) - if charlen == 1: - return W_UnicodeObject([w_uni._value[0]] * times) - + uni = w_uni._value + length = len(uni) + if times <= 0 or length == 0: + return W_UnicodeObject.EMPTY try: - result_size = ovfcheck(charlen * times) - result = chars * times + result_size = ovfcheck(length * times) + result = uni * times except (OverflowError, MemoryError): raise OperationError(space.w_OverflowError, space.wrap('repeated string is too long')) return W_UnicodeObject(result) @@ -267,53 +210,23 @@ def _isspace(uchar): return unicodedb.isspace(ord(uchar)) -def unicode_isspace__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isspace(ord(uchar)): +def make_generic(funcname): + def func(space, w_self): + v = w_self._value + if len(v) == 0: return space.w_False - return space.w_True - -def unicode_isalpha__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isalpha(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isalnum__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isalnum(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isdecimal__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isdecimal(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isdigit__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isdigit(ord(uchar)): - return space.w_False - return space.w_True + for idx in range(len(v)): + if not getattr(unicodedb, funcname)(ord(v[idx])): + return space.w_False + return space.w_True + return func_with_new_name(func, "unicode_%s__Unicode" % (funcname, )) -def unicode_isnumeric__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isnumeric(ord(uchar)): - return space.w_False - return space.w_True +unicode_isspace__Unicode = make_generic("isspace") +unicode_isalpha__Unicode = make_generic("isalpha") +unicode_isalnum__Unicode = make_generic("isalnum") +unicode_isdecimal__Unicode = make_generic("isdecimal") +unicode_isdigit__Unicode = make_generic("isdigit") +unicode_isnumeric__Unicode = make_generic("isnumeric") def unicode_islower__Unicode(space, w_unicode): cased = False @@ -423,12 +336,12 @@ def unicode_capitalize__Unicode(space, w_self): input = w_self._value if len(input) == 0: - return W_UnicodeObject([]) + return W_UnicodeObject.EMPTY result = [u'\0'] * len(input) result[0] = unichr(unicodedb.toupper(ord(input[0]))) for i in range(1, len(input)): result[i] = unichr(unicodedb.tolower(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_title__Unicode(space, w_self): input = w_self._value @@ -436,7 +349,7 @@ return w_self result = [u'\0'] * len(input) - previous_is_cased = 0 + previous_is_cased = False for i in range(len(input)): unichar = ord(input[i]) if previous_is_cased: @@ -444,21 +357,21 @@ else: result[i] = unichr(unicodedb.totitle(unichar)) previous_is_cased = unicodedb.iscased(unichar) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_lower__Unicode(space, w_self): input = w_self._value result = [u'\0'] * len(input) for i in range(len(input)): result[i] = unichr(unicodedb.tolower(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_upper__Unicode(space, w_self): input = w_self._value result = [u'\0'] * len(input) for i in range(len(input)): result[i] = unichr(unicodedb.toupper(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_swapcase__Unicode(space, w_self): input = w_self._value @@ -471,7 +384,7 @@ result[i] = unichr(unicodedb.tolower(unichar)) else: result[i] = input[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def _normalize_index(length, index): if index < 0: @@ -516,7 +429,7 @@ def _to_unichar_w(space, w_char): try: - w_unichar = unicodetype.unicode_from_object(space, w_char) + unistr = space.unicode_w(w_char) except OperationError, e: if e.match(space, space.w_TypeError): msg = 'The fill character cannot be converted to Unicode' @@ -524,10 +437,9 @@ else: raise - if space.int_w(space.len(w_unichar)) != 1: + if len(unistr) != 1: raise OperationError(space.w_TypeError, space.wrap('The fill character must be exactly one character long')) - unichar = unichr(space.int_w(space.ord(w_unichar))) - return unichar + return unistr[0] def unicode_center__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -540,7 +452,7 @@ result = [fillchar] * width for i in range(len(self)): result[leftpad + i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_ljust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -552,7 +464,7 @@ result = [fillchar] * width for i in range(len(self)): result[i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_rjust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -564,13 +476,13 @@ result = [fillchar] * width for i in range(len(self)): result[padding + i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_zfill__Unicode_ANY(space, w_self, w_width): self = w_self._value width = space.int_w(w_width) if len(self) == 0: - return W_UnicodeObject([u'0'] * width) + return W_UnicodeObject(u'0' * width) padding = width - len(self) if padding <= 0: return space.call_function(space.w_unicode, w_self) @@ -581,7 +493,7 @@ if self[0] in (u'+', u'-'): result[0] = self[0] result[padding] = u'0' - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_splitlines__Unicode_ANY(space, w_self, w_keepends): self = w_self._value @@ -617,21 +529,21 @@ start = _normalize_index(len(self), space.int_w(w_start)) end = _normalize_index(len(self), space.int_w(w_end)) substr = w_substr._value - return space.wrap(_find(self, substr, start, end)) + return space.wrap(self.find(substr, start, end)) def unicode_rfind__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self = w_self._value start = _normalize_index(len(self), space.int_w(w_start)) end = _normalize_index(len(self), space.int_w(w_end)) substr = w_substr._value - return space.wrap(_rfind(self, substr, start, end)) + return space.wrap(self.rfind(substr, start, end)) def unicode_index__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self = w_self._value start = _normalize_index(len(self), space.int_w(w_start)) end = _normalize_index(len(self), space.int_w(w_end)) substr = w_substr._value - index = _find(self, substr, start, end) + index = self.find(substr, start, end) if index < 0: raise OperationError(space.w_ValueError, space.wrap('substring not found')) @@ -642,7 +554,7 @@ start = _normalize_index(len(self), space.int_w(w_start)) end = _normalize_index(len(self), space.int_w(w_end)) substr = w_substr._value - index = _rfind(self, substr, start, end) + index = self.rfind(substr, start, end) if index < 0: raise OperationError(space.w_ValueError, space.wrap('substring not found')) @@ -653,15 +565,7 @@ start = _normalize_index(len(self), space.int_w(w_start)) end = _normalize_index(len(self), space.int_w(w_end)) substr = w_substr._value - count = 0 - while start <= end: - index = _find(self, substr, start, end) - if index < 0: - break - start = index + 1 - count += 1 - return space.wrap(count) - + return space.wrap(self.count(substr, start, end)) def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit): self = w_self._value @@ -703,18 +607,8 @@ if delim_len == 0: raise OperationError(space.w_ValueError, space.wrap('empty separator')) - parts = [] - start = 0 - end = len(self) - while maxsplit != 0: - index = _find(self, delim, start, end) - if index < 0: - break - parts.append(W_UnicodeObject(self[start:index])) - start = index + delim_len - maxsplit -= 1 - parts.append(W_UnicodeObject(self[start:])) - return space.newlist(parts) + parts = _split_with(self, delim, maxsplit) + return space.newlist([W_UnicodeObject(part) for part in parts]) def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit): @@ -764,7 +658,7 @@ start = 0 end = len(self) while maxsplit != 0: - index = _rfind(self, delim, 0, end) + index = self.rfind(delim, 0, end) if index < 0: break parts.append(W_UnicodeObject(self[index+delim_len:end])) @@ -774,31 +668,47 @@ parts.reverse() return space.newlist(parts) -def _split(space, self, maxsplit): +def _split_into_chars(self, maxsplit): if maxsplit == 0: - return [W_UnicodeObject(self)] + return [self] index = 0 end = len(self) - parts = [W_UnicodeObject([])] + parts = [u''] maxsplit -= 1 while maxsplit != 0: if index >= end: break - parts.append(W_UnicodeObject([self[index]])) + parts.append(self[index]) index += 1 maxsplit -= 1 - parts.append(W_UnicodeObject(self[index:])) + parts.append(self[index:]) + return parts + +def _split_with(self, with_, maxsplit): + parts = [] + start = 0 + end = len(self) + length = len(with_) + while maxsplit != 0: + index = self.find(with_, start, end) + if index < 0: + break + parts.append(self[start:index]) + start = index + length + maxsplit -= 1 + parts.append(self[start:]) return parts def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old, w_new, w_maxsplit): if len(w_old._value): - w_parts = space.call_method(w_self, 'split', w_old, w_maxsplit) + parts = _split_with(w_self._value, w_old._value, + space.int_w(w_maxsplit)) else: self = w_self._value maxsplit = space.int_w(w_maxsplit) - w_parts = space.newlist(_split(space, self, maxsplit)) - return space.call_method(w_new, 'join', w_parts) + parts = _split_into_chars(self, maxsplit) + return W_UnicodeObject(w_new._value.join(parts)) app = gateway.applevel(r''' @@ -877,13 +787,13 @@ def unicode_startswith__Unicode_Tuple_ANY_ANY(unistr, prefixes, start, end): for prefix in prefixes: - if unistr.startswith(prefix): + if unistr.startswith(prefix, start, end): return True return False def unicode_endswith__Unicode_Tuple_ANY_ANY(unistr, suffixes, start, end): for suffix in suffixes: - if unistr.endswith(suffix): + if unistr.endswith(suffix, start, end): return True return False @@ -1016,7 +926,6 @@ i += 1 return space.wrap(''.join(result[:i])) -#repr__Unicode = app.interphook('repr__Unicode') # uncomment when repr code is moved to _codecs def mod__Unicode_ANY(space, w_format, w_values): return mod_format(space, w_format, w_values, do_unicode=True) @@ -1028,6 +937,10 @@ # str.strip(unicode) needs to convert self to unicode and call unicode.strip we # use the following magic to register strip_string_unicode as a String # multimethod. + +# XXX couldn't string and unicode _share_ the multimethods that make up their +# methods? + class str_methods: import stringtype W_UnicodeObject = W_UnicodeObject Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 00:24:58 2007 @@ -182,14 +182,11 @@ if not space.eq_w(w_encoding, space.wrap('ascii')): return unicode_from_object(space, w_str) s = space.str_w(w_str) - codelist = [] - for i in range(len(s)): - code = ord(s[i]) - if code >= 128: - # raising UnicodeDecodeError is messy, so "please crash for me" - return unicode_from_object(space, w_str) - codelist.append(unichr(code)) - return W_UnicodeObject(codelist) + try: + return W_UnicodeObject(s.decode("ascii")) + except UnicodeDecodeErrors: + # raising UnicodeDecodeErrors is messy, "please crash for me" + return unicode_from_object(space, w_str) def descr__new__(space, w_unicodetype, w_string='', w_encoding=None, w_errors=None): Modified: pypy/branch/unicode-objspace/pypy/objspace/thunk.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/thunk.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/thunk.py Sat Nov 10 00:24:58 2007 @@ -150,7 +150,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -159,7 +159,6 @@ 'newtuple': 0, 'newlist': 0, 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, From cfbolz at codespeak.net Sat Nov 10 00:40:56 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 00:40:56 +0100 (CET) Subject: [pypy-svn] r48483 - in pypy/branch/unicode-objspace/pypy: interpreter interpreter/test module/__builtin__ objspace objspace/cpy objspace/cpy/test objspace/fake objspace/std translator Message-ID: <20071109234056.2012B81E3@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 00:40:56 2007 New Revision: 48483 Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py pypy/branch/unicode-objspace/pypy/interpreter/test/test_objspace.py pypy/branch/unicode-objspace/pypy/interpreter/test/test_typedef.py pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py pypy/branch/unicode-objspace/pypy/objspace/dump.py pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py pypy/branch/unicode-objspace/pypy/objspace/logic.py pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py pypy/branch/unicode-objspace/pypy/objspace/thunk.py pypy/branch/unicode-objspace/pypy/translator/geninterplevel.py Log: get rid of newstring as well Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py Sat Nov 10 00:40:56 2007 @@ -1036,7 +1036,6 @@ # is_true(w_x) -> True or False # newtuple([w_1, w_2,...]) -> w_tuple # newlist([w_1, w_2,...]) -> w_list -# newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) # newdict() -> empty w_dict # newslice(w_start,w_stop,w_step) -> w_slice # call_args(w_obj,Arguments()) -> w_result @@ -1055,7 +1054,6 @@ 'is_w', 'newtuple', 'newlist', - 'newstring', 'newdict', 'newslice', 'call_args', Modified: pypy/branch/unicode-objspace/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/test/test_objspace.py Sat Nov 10 00:40:56 2007 @@ -9,23 +9,6 @@ # -- it's more to test that it's *there* class TestObjSpace: - def test_newstring(self): - w = self.space.wrap - s = 'abc' - chars_w = [w(ord(c)) for c in s] - assert self.space.eq_w(w(s), self.space.newstring(chars_w)) - - def test_newstring_fail(self): - w = self.space.wrap - s = 'abc' - not_chars_w = [w(c) for c in s] - self.space.raises_w(self.space.w_TypeError, - self.space.newstring, - not_chars_w) - self.space.raises_w(self.space.w_ValueError, - self.space.newstring, - [w(-1)]) - def test_newlist(self): w = self.space.wrap l = range(10) Modified: pypy/branch/unicode-objspace/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/test/test_typedef.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/test/test_typedef.py Sat Nov 10 00:40:56 2007 @@ -5,6 +5,7 @@ class AppTestTraceBackAttributes: def test_newstring(self): + # XXX why is this called newstring? import sys def f(): raise TypeError, "hello" Modified: pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py (original) +++ pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py Sat Nov 10 00:40:56 2007 @@ -14,8 +14,12 @@ def chr(space, w_ascii): "Return a string of one character with the given ascii code." - w_character = space.newstring([w_ascii]) - return w_character + try: + char = __builtin__.chr(space.int_w(w_ascii)) + except ValueError: # chr(out-of-range) + raise OperationError(space.w_ValueError, + space.wrap("character code not in range(256)")) + return space.wrap(char) def unichr(space, code): "Return a Unicode string of one character with the given ordinal." Modified: pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py Sat Nov 10 00:40:56 2007 @@ -230,13 +230,6 @@ PyString_InternInPlace(byref(w_s)) return w_s - def newstring(self, bytes_w): - length = len(bytes_w) - buf = ctypes.create_string_buffer(length) - for i in range(length): - buf[i] = chr(self.int_w(bytes_w[i])) - return PyString_FromStringAndSize(buf, length) - def newint(self, intval): return PyInt_FromLong(intval) Modified: pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py Sat Nov 10 00:40:56 2007 @@ -49,9 +49,9 @@ w2 = space.wrap(11) raises_w(space, space.w_TypeError, space.sub, w1, w2) -def test_newstring(): +def test_wrapstring(): space = CPyObjSpace() - w = space.newstring([space.wrap(65), space.wrap(66)]) + w = space.wrap('AB') assert space.str_w(w) == 'AB' def test_wrapunicode(): Modified: pypy/branch/unicode-objspace/pypy/objspace/dump.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/dump.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/dump.py Sat Nov 10 00:40:56 2007 @@ -153,7 +153,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, @@ -164,7 +163,6 @@ 'wrap': True, 'newtuple': True, 'newlist': True, - 'newstring': True, 'newdict': True, 'newslice': True, 'call_args': True, Modified: pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py Sat Nov 10 00:40:56 2007 @@ -88,7 +88,6 @@ str_w = str_dummy call_args = make_dummy() new_interned_str = make_dummy() - newstring = make_dummy() newint = make_dummy() newlong = make_dummy() newfloat = make_dummy() Modified: pypy/branch/unicode-objspace/pypy/objspace/logic.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/logic.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/logic.py Sat Nov 10 00:40:56 2007 @@ -57,7 +57,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, Modified: pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py Sat Nov 10 00:40:56 2007 @@ -525,14 +525,6 @@ def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) - def newstring(self, chars_w): - try: - chars = [chr(self.int_w(w_c)) for w_c in chars_w] - except ValueError: # chr(out-of-range) - raise OperationError(self.w_ValueError, - self.wrap("character code not in range(256)")) - return self.wrap(''.join(chars)) - def newseqiter(self, w_obj): return W_SeqIterObject(w_obj) Modified: pypy/branch/unicode-objspace/pypy/objspace/thunk.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/thunk.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/thunk.py Sat Nov 10 00:40:56 2007 @@ -158,7 +158,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, Modified: pypy/branch/unicode-objspace/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/geninterplevel.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/geninterplevel.py Sat Nov 10 00:40:56 2007 @@ -177,7 +177,7 @@ # special constructors: self.has_listarg = {} - for name in "newtuple newlist newstring".split(): + for name in "newtuple newlist".split(): self.has_listarg[name] = name # catching all builtins in advance, to avoid problems From cfbolz at codespeak.net Sat Nov 10 00:50:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 00:50:27 +0100 (CET) Subject: [pypy-svn] r48484 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071109235027.1C54081E5@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 00:50:26 2007 New Revision: 48484 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: partition, rpartition go from applevel to interplevel Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 00:50:26 2007 @@ -769,21 +769,6 @@ type(retval).__name__) return retval -# XXX: These should probably be written on interplevel - -def unicode_partition__Unicode_Unicode(unistr, unisub): - pos = unistr.find(unisub) - if pos == -1: - return (unistr, u'', u'') - else: - return (unistr[:pos], unisub, unistr[pos+len(unisub):]) - -def unicode_rpartition__Unicode_Unicode(unistr, unisub): - pos = unistr.rfind(unisub) - if pos == -1: - return (u'', u'', unistr) - else: - return (unistr[:pos], unisub, unistr[pos+len(unisub):]) def unicode_startswith__Unicode_Tuple_ANY_ANY(unistr, prefixes, start, end): for prefix in prefixes: @@ -799,14 +784,43 @@ ''') + + unicode_expandtabs__Unicode_ANY = app.interphook('unicode_expandtabs__Unicode_ANY') unicode_translate__Unicode_ANY = app.interphook('unicode_translate__Unicode_ANY') unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') -unicode_partition__Unicode_Unicode = app.interphook('unicode_partition__Unicode_Unicode') -unicode_rpartition__Unicode_Unicode = app.interphook('unicode_rpartition__Unicode_Unicode') unicode_startswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_startswith__Unicode_Tuple_ANY_ANY') unicode_endswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_endswith__Unicode_Tuple_ANY_ANY') +def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): + unistr = w_unistr._value + unisub = w_unisub._value + if not unisub: + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) + pos = unistr.find(unisub) + if pos == -1: + return space.newtuple([w_unistr, W_UnicodeObject.EMPTY, + W_UnicodeObject.EMPTY]) + else: + return space.newtuple([space.wrap(unistr[:pos]), w_unisub, + space.wrap(unistr[pos+len(unisub):])]) + +def unicode_rpartition__Unicode_Unicode(space, w_unistr, w_unisub): + unistr = w_unistr._value + unisub = w_unisub._value + if not unisub: + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) + pos = unistr.rfind(unisub) + if pos == -1: + return space.newtuple([W_UnicodeObject.EMPTY, + W_UnicodeObject.EMPTY, w_unistr]) + else: + return space.newtuple([space.wrap(unistr[:pos]), w_unisub, + space.wrap(unistr[pos+len(unisub):])]) + + # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): hexdigits = "0123456789abcdef" From cfbolz at codespeak.net Sat Nov 10 01:26:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 01:26:00 +0100 (CET) Subject: [pypy-svn] r48485 - in pypy/branch/unicode-objspace/pypy/objspace/std: . test Message-ID: <20071110002600.A5F0681D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 01:25:59 2007 New Revision: 48485 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_index.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: move more stuff to interplevel. also fix indexing behaviour + tests Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_index.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_index.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_index.py Sat Nov 10 01:25:59 2007 @@ -223,15 +223,50 @@ SeqTestCase.setup_method(self, method) self.w_seq = self.space.newtuple([self.space.wrap(x) for x in (0,10,20,30,40,50)]) -class AppTest_StringTestCase(SeqTestCase): +class StringTestCase(object): + def test_startswith(self): + self.o.ind = 1 + assert self.const('abc').startswith(self.const('b'), self.o) + self.o.ind = 2 + assert not self.const('abc').startswith(self.const('abc'), 0, self.o) + + def test_endswith(self): + self.o.ind = 1 + assert self.const('abc').endswith(self.const('a'), 0, self.o) + self.o.ind = 2 + assert not self.const('abc').endswith(self.const('abc'), 0, self.o) + + def test_index(self): + self.o.ind = 3 + assert self.const('abcabc').index(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').index(self.const('abc'), self.o) == 3 + assert self.const('abcabc').rindex(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').rindex(self.const('abc'), self.o) == 3 + + def test_find(self): + self.o.ind = 3 + assert self.const('abcabc').find(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').find(self.const('abc'), self.o) == 3 + assert self.const('abcabc').rfind(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').rfind(self.const('abc'), self.o) == 3 + + def test_count(self): + self.o.ind = 3 + assert self.const('abcabc').count(self.const('abc'), 0, self.o) == 1 + assert self.const('abcabc').count(self.const('abc'), self.o) == 1 + + +class AppTest_StringTestCase(SeqTestCase, StringTestCase): def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap("this is a test") + self.w_const = self.space.w_str -class AppTest_UnicodeTestCase(SeqTestCase): +class AppTest_UnicodeTestCase(SeqTestCase, StringTestCase): def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap(u"this is a test") + self.w_const = self.space.w_unicode class AppTest_XRangeTestCase: Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 01:25:59 2007 @@ -4,6 +4,7 @@ from pypy.objspace.std.ropeobject import W_RopeObject from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.sliceobject import W_SliceObject +from pypy.objspace.std import slicetype from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.module.unicodedata import unicodedb_3_2_0 as unicodedb @@ -395,11 +396,18 @@ index = length return index -def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): +def _convert_idx_params(space, w_self, w_start, w_end): self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + start = slicetype.adapt_bound(space, len(self), w_start) + end = slicetype.adapt_bound(space, len(self), w_end) + + assert start >= 0 + assert end >= 0 + return (self, start, end) + +def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value substr_len = len(substr) @@ -412,9 +420,7 @@ return space.w_True def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value substr_len = len(substr) @@ -427,6 +433,23 @@ return space.w_False return space.w_True + +def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for prefix in space.unpacktuple(w_prefixes): + if unistr.startswith(prefix, start, end): + return True + return False + +def unicode_endswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for suffix in space.unpacktuple(w_suffixes): + if unistr.endswith(suffix, start, end): + return True + return False + def _to_unichar_w(space, w_char): try: unistr = space.unicode_w(w_char) @@ -525,23 +548,17 @@ return space.newlist(lines) def unicode_find__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value return space.wrap(self.find(substr, start, end)) def unicode_rfind__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value return space.wrap(self.rfind(substr, start, end)) def unicode_index__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value index = self.find(substr, start, end) if index < 0: @@ -550,9 +567,7 @@ return space.wrap(index) def unicode_rindex__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value index = self.rfind(substr, start, end) if index < 0: @@ -561,9 +576,7 @@ return space.wrap(index) def unicode_count__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value return space.wrap(self.count(substr, start, end)) @@ -770,17 +783,6 @@ return retval -def unicode_startswith__Unicode_Tuple_ANY_ANY(unistr, prefixes, start, end): - for prefix in prefixes: - if unistr.startswith(prefix, start, end): - return True - return False - -def unicode_endswith__Unicode_Tuple_ANY_ANY(unistr, suffixes, start, end): - for suffix in suffixes: - if unistr.endswith(suffix, start, end): - return True - return False ''') @@ -789,8 +791,6 @@ unicode_expandtabs__Unicode_ANY = app.interphook('unicode_expandtabs__Unicode_ANY') unicode_translate__Unicode_ANY = app.interphook('unicode_translate__Unicode_ANY') unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') -unicode_startswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_startswith__Unicode_Tuple_ANY_ANY') -unicode_endswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_endswith__Unicode_Tuple_ANY_ANY') def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): unistr = w_unistr._value From cfbolz at codespeak.net Sat Nov 10 01:28:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 01:28:40 +0100 (CET) Subject: [pypy-svn] r48486 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110002840.41B3D81E3@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 01:28:38 2007 New Revision: 48486 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: whoops, that was nonsense Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 01:28:38 2007 @@ -437,18 +437,20 @@ def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.unpacktuple(w_prefixes): + prefix = space.unicode_w(w_prefix) if unistr.startswith(prefix, start, end): - return True - return False + return space.w_True + return space.w_False def unicode_endswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.unpacktuple(w_suffixes): + suffix = space.unicode_w(w_suffix) if unistr.endswith(suffix, start, end): - return True - return False + return space.w_True + return space.w_False def _to_unichar_w(space, w_char): try: From cfbolz at codespeak.net Sat Nov 10 02:13:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 02:13:47 +0100 (CET) Subject: [pypy-svn] r48490 - in pypy/branch/unicode-objspace/pypy: interpreter module/sys module/sys/test objspace/std Message-ID: <20071110011347.70C348129@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 02:13:44 2007 New Revision: 48490 Added: pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py (contents, props changed) Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py pypy/branch/unicode-objspace/pypy/module/sys/__init__.py pypy/branch/unicode-objspace/pypy/module/sys/app.py pypy/branch/unicode-objspace/pypy/module/sys/test/test_sysmodule.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: attach the default-encoding to the space for easier accessibility. not sure this is a very nice solution, but I think it is way better than a global in an applevel implementation module in the sys mixed-module. Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py Sat Nov 10 02:13:44 2007 @@ -188,6 +188,7 @@ self.interned_strings = {} self.pending_actions = [] self.setoptions(**kw) + self.defaultencoding = "ascii" # if self.config.objspace.logbytecodes: # self.bytecodecounts = {} Modified: pypy/branch/unicode-objspace/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/__init__.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/__init__.py Sat Nov 10 02:13:44 2007 @@ -64,6 +64,9 @@ 'path_hooks' : 'space.wrap([])', 'path_importer_cache' : 'space.wrap({})', #'subversion' : added in Python 2.5 + + 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', + 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', } appleveldefs = { #'displayhook' : 'app.displayhook', @@ -75,8 +78,6 @@ 'pypy__exithandlers__' : 'app.pypy__exithandlers__', # internal 'getfilesystemencoding' : 'app.getfilesystemencoding', 'callstats' : 'app.callstats', - 'getdefaultencoding' : 'app.getdefaultencoding', - 'setdefaultencoding' : 'app.setdefaultencoding', } def setbuiltinmodule(self, w_module, name): Modified: pypy/branch/unicode-objspace/pypy/module/sys/app.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/app.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/app.py Sat Nov 10 02:13:44 2007 @@ -46,17 +46,3 @@ """Not implemented.""" return None -defaultencoding = 'ascii' - -def getdefaultencoding(): - """Return the current default string encoding used by the Unicode -implementation.""" - return defaultencoding - -def setdefaultencoding(encoding): - """Set the current default string encoding used by the Unicode -implementation.""" - global defaultencoding - import codecs - codecs.lookup(encoding) - defaultencoding = encoding Added: pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py ============================================================================== --- (empty file) +++ pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py Sat Nov 10 02:13:44 2007 @@ -0,0 +1,15 @@ + +def getdefaultencoding(space): + """Return the current default string encoding used by the Unicode +implementation.""" + return space.wrap(space.defaultencoding) + +def setdefaultencoding(space, w_encoding): + """Set the current default string encoding used by the Unicode +implementation.""" + encoding = space.str_w(w_encoding) + mod = space.getbuiltinmodule("_codecs") + w_lookup = space.getattr(mod, space.wrap("lookup")) + # check whether the encoding is there + space.call_function(w_lookup, w_encoding) + space.defaultencoding = encoding Modified: pypy/branch/unicode-objspace/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/test/test_sysmodule.py Sat Nov 10 02:13:44 2007 @@ -246,6 +246,16 @@ raises(TypeError, sys.getdefaultencoding, 42) # can't check more than the type, as the user might have changed it assert isinstance(sys.getdefaultencoding(), str) + + def test_getdefaultencoding(self): + encoding = sys.getdefaultencoding() + sys.setdefaultencoding("ascii") + try: + assert sys.getdefaultencoding() == 'ascii' + raises(UnicodeDecodeError, unicode, '\x80') + finally: + sys.setdefaultencoding(encoding) + # testing sys.settrace() is done in test_trace.py # testing sys.setprofile() is done in test_profile.py Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 02:13:44 2007 @@ -184,16 +184,15 @@ s = space.str_w(w_str) try: return W_UnicodeObject(s.decode("ascii")) - except UnicodeDecodeErrors: - # raising UnicodeDecodeErrors is messy, "please crash for me" + except UnicodeDecodeError: + # raising UnicodeDecodeError is messy, "please crash for me" return unicode_from_object(space, w_str) -def descr__new__(space, w_unicodetype, w_string='', w_encoding=None, w_errors=None): - # NB. the default value of w_string is really a *wrapped* empty string: +def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): + # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work from pypy.objspace.std.unicodeobject import W_UnicodeObject - w_obj = w_string w_obj_type = space.type(w_obj) if space.is_w(w_obj_type, space.w_unicode): From cfbolz at codespeak.net Sat Nov 10 02:26:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 02:26:22 +0100 (CET) Subject: [pypy-svn] r48491 - in pypy/branch/unicode-objspace/pypy/objspace/std: . test Message-ID: <20071110012622.0045981B5@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 02:26:22 2007 New Revision: 48491 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: move expandtabs from app to interplevel Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 02:26:22 2007 @@ -334,3 +334,23 @@ assert u'hello'.endswith((u'he', u'hell'), 0, 4) raises(TypeError, u'hello'.endswith, (42,)) + def test_expandtabs(self): + assert u'abc\rab\tdef\ng\thi'.expandtabs() == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(8) == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(4) == u'abc\rab def\ng hi' + assert u'abc\r\nab\tdef\ng\thi'.expandtabs(4) == u'abc\r\nab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs() == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(8) == u'abc\rab def\ng hi' + assert u'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == u'abc\r\nab\r\ndef\ng\r\nhi' + + s = u'xy\t' + assert s.expandtabs() =='xy ' + + s = u'\txy\t' + assert s.expandtabs() ==' xy ' + assert s.expandtabs(1) ==' xy ' + assert s.expandtabs(2) ==' xy ' + assert s.expandtabs(3) ==' xy ' + + assert u'xy'.expandtabs() =='xy' + assert u''.expandtabs() =='' Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 02:26:22 2007 @@ -699,7 +699,7 @@ parts.append(self[index:]) return parts -def _split_with(self, with_, maxsplit): +def _split_with(self, with_, maxsplit=-1): parts = [] start = 0 end = len(self) @@ -729,25 +729,6 @@ app = gateway.applevel(r''' import sys -def unicode_expandtabs__Unicode_ANY(self, tabsize): - parts = self.split(u'\t') - result = [ parts[0] ] - prevsize = 0 - for ch in parts[0]: - prevsize += 1 - if ch in (u"\n", u"\r"): - prevsize = 0 - for i in range(1, len(parts)): - pad = tabsize - prevsize % tabsize - result.append(u' ' * pad) - nextpart = parts[i] - result.append(nextpart) - prevsize = 0 - for ch in nextpart: - prevsize += 1 - if ch in (u"\n", u"\r"): - prevsize = 0 - return u''.join(result) def unicode_translate__Unicode_ANY(self, table): result = [] @@ -790,7 +771,6 @@ -unicode_expandtabs__Unicode_ANY = app.interphook('unicode_expandtabs__Unicode_ANY') unicode_translate__Unicode_ANY = app.interphook('unicode_translate__Unicode_ANY') unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') @@ -823,6 +803,29 @@ space.wrap(unistr[pos+len(unisub):])]) +def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize): + self = w_self._value + tabsize = space.int_w(w_tabsize) + parts = _split_with(self, u'\t') + result = [parts[0]] + prevsize = 0 + for ch in parts[0]: + prevsize += 1 + if ch == u"\n" or ch == u"\r": + prevsize = 0 + for i in range(1, len(parts)): + pad = tabsize - prevsize % tabsize + result.append(u' ' * pad) + nextpart = parts[i] + result.append(nextpart) + prevsize = 0 + for ch in nextpart: + prevsize += 1 + if ch in (u"\n", u"\r"): + prevsize = 0 + return space.wrap(u''.join(result)) + + # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): hexdigits = "0123456789abcdef" From cfbolz at codespeak.net Sat Nov 10 02:42:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 02:42:29 +0100 (CET) Subject: [pypy-svn] r48492 - in pypy/branch/unicode-objspace/pypy/objspace/std: . test Message-ID: <20071110014229.DC34881C6@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 02:42:28 2007 New Revision: 48492 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: move translate from app to interplevel. add tests. Why is the test coverage of unicode so terribly spotty? Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 02:42:28 2007 @@ -354,3 +354,14 @@ assert u'xy'.expandtabs() =='xy' assert u''.expandtabs() =='' + + def test_translate(self): + assert u'bbbc' == u'abababc'.translate({ord('a'):None}) + assert u'iiic' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i')}) + assert u'iiix' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):u'x'}) + assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''}) + assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''}) + assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'}) + + raises(TypeError, u'hello'.translate) + raises(TypeError, u'abababc'.translate, {ord('a'):''}) Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 02:42:28 2007 @@ -730,25 +730,6 @@ import sys -def unicode_translate__Unicode_ANY(self, table): - result = [] - for unichar in self: - try: - newval = table[ord(unichar)] - except KeyError: - result.append(unichar) - else: - if newval is None: - continue - elif isinstance(newval, int): - if newval < 0 or newval > sys.maxunicode: - raise TypeError("character mapping must be in range(0x%x)"%(sys.maxunicode + 1,)) - result.append(unichr(newval)) - elif isinstance(newval, unicode): - result.append(newval) - else: - raise TypeError("character mapping must return integer, None or unicode") - return ''.join(result) def unicode_encode__Unicode_ANY_ANY(unistr, encoding=None, errors=None): import codecs, sys @@ -826,6 +807,37 @@ return space.wrap(u''.join(result)) +def unicode_translate__Unicode_ANY(space, w_self, w_table): + self = w_self._value + w_sys = space.getbuiltinmodule('sys') + maxunicode = space.int_w(space.getattr(w_sys, space.wrap("maxunicode"))) + result = [] + for unichar in self: + try: + w_newval = space.getitem(w_table, space.wrap(ord(unichar))) + except OperationError, e: + if e.match(space, space.w_KeyError): + result.append(unichar) + else: + raise + else: + if space.is_w(w_newval, space.w_None): + continue + elif space.is_true(space.isinstance(w_newval, space.w_int)): + newval = space.int_w(w_newval) + if newval < 0 or newval > maxunicode: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must be in range(0x%x)" % (maxunicode + 1,))) + result.append(unichr(newval)) + elif space.is_true(space.isinstance(w_newval, space.w_unicode)): + result.append(space.unicode_w(w_newval)) + else: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must return integer, None or unicode")) + return W_UnicodeObject(''.join(result)) + # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): hexdigits = "0123456789abcdef" From cfbolz at codespeak.net Sat Nov 10 02:56:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 02:56:44 +0100 (CET) Subject: [pypy-svn] r48493 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110015644.82AE781D1@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 02:56:44 2007 New Revision: 48493 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: more applevel moving: this time in unicode.__new__ (!). Part 1 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 02:56:44 2007 @@ -729,8 +729,6 @@ app = gateway.applevel(r''' import sys - - def unicode_encode__Unicode_ANY_ANY(unistr, encoding=None, errors=None): import codecs, sys if encoding is None: @@ -745,9 +743,6 @@ raise TypeError("encoder did not return a string object (type=%s)" % type(retval).__name__) return retval - - - ''') Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 02:56:44 2007 @@ -157,29 +157,35 @@ type(retval).__name__) return retval -def unicode_from_object(obj): - if isinstance(obj, str): - res = obj - else: - try: - unicode_method = obj.__unicode__ - except AttributeError: - res = str(obj) - else: - res = unicode_method() - if isinstance(res, unicode): - return res - return unicode_from_encoded_object(res, None, "strict") ''') -unicode_from_object = app.interphook('unicode_from_object') unicode_from_encoded_object = app.interphook('unicode_from_encoded_object') +def unicode_from_object(space, w_obj): + if space.is_true(space.isinstance(w_obj, space.w_str)): + w_res = w_obj + else: + try: + # XXX should we have a space.unicode so we can go through + # descroperation? + w_unicode_method = space.getattr(w_obj, space.wrap("__unicode__")) + except OperationError, e: + if e.match(space, space.w_AttributeError): + w_res = space.str(w_obj) + else: + raise + else: + w_res = space.call(w_unicode_method) + if space.is_true(space.isinstance(w_res, space.w_unicode)): + return w_res + return unicode_from_encoded_object(w_res, space.w_None, + space.wrap("strict")) + def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack from pypy.objspace.std.unicodeobject import W_UnicodeObject - w_encoding = space.call_function(space.sys.get('getdefaultencoding')) - if not space.eq_w(w_encoding, space.wrap('ascii')): + encoding = space.defaultencoding + if encoding != 'ascii': return unicode_from_object(space, w_str) s = space.str_w(w_str) try: From cfbolz at codespeak.net Sat Nov 10 03:13:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 03:13:58 +0100 (CET) Subject: [pypy-svn] r48494 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110021358.6CE4681C4@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 03:13:55 2007 New Revision: 48494 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: part 2: no more applevel code in the __new__ Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 03:13:55 2007 @@ -142,24 +142,26 @@ # ____________________________________________________________ -app = gateway.applevel(''' -def unicode_from_encoded_object(obj, encoding, errors): - import codecs, sys +def unicode_from_encoded_object(space, w_obj, encoding, errors): + w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: - encoding = sys.getdefaultencoding() - decoder = codecs.getdecoder(encoding) + encoding = space.defaultencoding + w_decode = space.getattr(w_codecs, space.wrap("decode")) if errors is None: - retval, length = decoder(obj) + w_retval, w_length = space.unpacktuple( + space.call(w_decode, obj, space.wrap(encoding)), 2) else: - retval, length = decoder(obj, errors) - if not isinstance(retval, unicode): - raise TypeError("decoder did not return an unicode object (type=%s)" % - type(retval).__name__) + w_retval, w_length = space.unpacktuple( + space.call(w_decode, obj, space.wrap(encoding), + space.wrap(errors)), 2) + if not space.is_true(space.isinstance(w_retval, space.w_unicode)): + raise OperationError( + space.w_TypeError, + space.wrap( + "decoder did not return an unicode object (type=%s)" % + space.type(retval).getname('?'))) return retval - -''') -unicode_from_encoded_object = app.interphook('unicode_from_encoded_object') def unicode_from_object(space, w_obj): if space.is_true(space.isinstance(w_obj, space.w_str)): @@ -178,8 +180,7 @@ w_res = space.call(w_unicode_method) if space.is_true(space.isinstance(w_res, space.w_unicode)): return w_res - return unicode_from_encoded_object(w_res, space.w_None, - space.wrap("strict")) + return unicode_from_encoded_object(w_res, None, "strict") def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack @@ -218,7 +219,16 @@ else: w_value = unicode_from_object(space, w_obj) else: - w_value = unicode_from_encoded_object(space, w_obj, w_encoding, w_errors) + if space.is_w(w_encoding, space.w_None): + encoding = None + else: + encoding = space.str_w(w_encoding) + if space.is_w(w_errors, space.w_None): + errors = None + else: + errors = space.str_w(w_errors) + errors + w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) # help the annotator! also the ._value depends on W_UnicodeObject layout assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) From antocuni at codespeak.net Sat Nov 10 11:47:08 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 10 Nov 2007 11:47:08 +0100 (CET) Subject: [pypy-svn] r48495 - in pypy/branch/unicode-objspace/pypy/translator: cli jvm jvm/src/pypy jvm/test oosupport Message-ID: <20071110104708.417C881B0@code0.codespeak.net> Author: antocuni Date: Sat Nov 10 11:47:06 2007 New Revision: 48495 Added: pypy/branch/unicode-objspace/pypy/translator/jvm/test/test_unicode.py (contents, props changed) Modified: pypy/branch/unicode-objspace/pypy/translator/cli/function.py pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py pypy/branch/unicode-objspace/pypy/translator/jvm/builtin.py pypy/branch/unicode-objspace/pypy/translator/jvm/conftest.py pypy/branch/unicode-objspace/pypy/translator/jvm/database.py pypy/branch/unicode-objspace/pypy/translator/jvm/generator.py pypy/branch/unicode-objspace/pypy/translator/jvm/opcodes.py pypy/branch/unicode-objspace/pypy/translator/jvm/prebuiltnodes.py pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/Interlink.java pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/PyPy.java pypy/branch/unicode-objspace/pypy/translator/jvm/test/runtest.py pypy/branch/unicode-objspace/pypy/translator/oosupport/metavm.py Log: add unicode support to genjvm Modified: pypy/branch/unicode-objspace/pypy/translator/cli/function.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/function.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/function.py Sat Nov 10 11:47:06 2007 @@ -162,6 +162,17 @@ for link, lbl in cases.itervalues(): self.render_switch_case(link, lbl) + def call_oostring(self, ARGTYPE): + if isinstance(ARGTYPE, ootype.Instance): + argtype = self.cts.types.object + else: + argtype = self.cts.lltype_to_cts(ARGTYPE) + self.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) + + def call_oounicode(self, ARGTYPE): + argtype = self.cts.lltype_to_cts(ARGTYPE) + self.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + # Those parts of the generator interface that are function # specific Modified: pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py Sat Nov 10 11:47:06 2007 @@ -95,28 +95,6 @@ generator.call_signature('object [pypylib]pypy.runtime.Utils::RuntimeNew(class [mscorlib]System.Type)') generator.cast_to(op.result.concretetype) -class _OOString(MicroInstruction): - def render(self, generator, op): - ARGTYPE = op.args[0].concretetype - if isinstance(ARGTYPE, ootype.Instance): - argtype = 'object' - else: - argtype = generator.cts.lltype_to_cts(ARGTYPE) - generator.load(op.args[0]) - generator.load(op.args[1]) - generator.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) - -class _OOUnicode(MicroInstruction): - def render(self, generator, op): - from pypy.objspace.flow.model import Constant - ARGTYPE = op.args[0].concretetype - argtype = generator.cts.lltype_to_cts(ARGTYPE) - v_base = op.args[1] - assert v_base.value == -1, "The second argument of oounicode must be -1" - - generator.load(op.args[0]) - generator.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) - class _NewCustomDict(MicroInstruction): def render(self, generator, op): DICT = op.args[0].value @@ -236,8 +214,6 @@ CallMethod = _CallMethod() IndirectCall = _IndirectCall() RuntimeNew = _RuntimeNew() -OOString = _OOString() -OOUnicode = _OOUnicode() NewCustomDict = _NewCustomDict() #CastWeakAdrToPtr = _CastWeakAdrToPtr() Box = _Box() Modified: pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py Sat Nov 10 11:47:06 2007 @@ -1,9 +1,9 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ - IndirectCall, GetField, SetField, OOString, DownCast, NewCustomDict,\ + IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, OOUnicode + TypeOf, CastPrimitive from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ - New, RuntimeNew, CastTo, PushPrimitive + New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode from pypy.translator.cli.cts import WEAKREF from pypy.rpython.ootypesystem import ootype Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/builtin.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/builtin.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/builtin.py Sat Nov 10 11:47:06 2007 @@ -149,3 +149,18 @@ jvmgen.Method.v(jArrayList, "get", (jInt,), jObject), } + +# ootype.String[Builder] and ootype.Unicode[Builder] are mapped to the +# same JVM type, so we reuse the same builtin methods also for them +def add_unicode_methods(): + mapping = { + ootype.String.__class__: ootype.Unicode.__class__, + ootype.StringBuilder.__class__: ootype.UnicodeBuilder.__class__ + } + + for (TYPE, name), value in built_in_methods.items(): + if TYPE in mapping: + TYPE = mapping[TYPE] + built_in_methods[TYPE, name] = value +add_unicode_methods() +del add_unicode_methods Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/conftest.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/conftest.py Sat Nov 10 11:47:06 2007 @@ -15,8 +15,8 @@ help="don't assemble jasmin files"), Option('--package', action='store', dest='package', default='pypy', help='Package to output generated classes into'), - Option('--trace', action='store_true', dest='trace', default=False, - help='Trace execution of generated code'), +## Option('--trace', action='store_true', dest='trace', default=False, +## help='Trace execution of generated code'), Option('--byte-arrays', action='store_true', dest='byte-arrays', default=False, help='Use byte arrays rather than native strings'), ) Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/database.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/database.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/database.py Sat Nov 10 11:47:06 2007 @@ -412,7 +412,9 @@ ootype.Bool:jvmgen.PYPYSERIALIZEBOOLEAN, ootype.Void:jvmgen.PYPYSERIALIZEVOID, ootype.Char:jvmgen.PYPYESCAPEDCHAR, + ootype.UniChar:jvmgen.PYPYESCAPEDUNICHAR, ootype.String:jvmgen.PYPYESCAPEDSTRING, + ootype.Unicode:jvmgen.PYPYESCAPEDUNICODE, } def toString_method_for_ootype(self, OOTYPE): @@ -466,7 +468,9 @@ # will return a JvmBuiltInType based on the value ootype_to_builtin = { ootype.String: jvmtype.jString, + ootype.Unicode: jvmtype.jString, ootype.StringBuilder: jvmtype.jStringBuilder, + ootype.UnicodeBuilder: jvmtype.jStringBuilder, ootype.List: jvmtype.jArrayList, ootype.Dict: jvmtype.jHashMap, ootype.DictItemsIterator:jvmtype.jPyPyDictItemsIterator, Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/generator.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/generator.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/generator.py Sat Nov 10 11:47:06 2007 @@ -389,7 +389,9 @@ PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,), jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) +PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) +PYPYESCAPEDUNICODE = Method.s(jPyPy, 'escaped_unicode', (jString,), jString) PYPYSERIALIZEOBJECT = Method.s(jPyPy, 'serializeObject', (jObject,), jString) PYPYRUNTIMENEW = Method.s(jPyPy, 'RuntimeNew', (jClass,), jObject) PYPYSTRING2BYTES = Method.s(jPyPy, 'string2bytes', (jString,), jByteArray) @@ -1018,6 +1020,13 @@ self.emit(mthd) if self.db.using_byte_array: self.emit(PYPYSTRING2BYTES) + + def call_oounicode(self, OOTYPE): + cts_type = self.db.lltype_to_cts(OOTYPE) + mthd = Method.s(jPyPy, 'oounicode', [cts_type], jString) + self.emit(mthd) + if self.db.using_byte_array: + self.emit(PYPYSTRING2BYTES) def new(self, TYPE): jtype = self.db.lltype_to_cts(TYPE) @@ -1087,11 +1096,13 @@ self._push_long_constant(value) elif TYPE is ootype.Float: self._push_double_constant(float(value)) - elif TYPE is ootype.String: - if value == ootype.null(ootype.String): + elif TYPE in (ootype.String, ootype.Unicode): + if value == ootype.null(TYPE): self.emit(ACONST_NULL) else: self.load_string(str(value._str)) + else: + assert False, 'Unknown constant type: %s' % TYPE def _push_long_constant(self, value): if value == 0: @@ -1336,7 +1347,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - #self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/opcodes.py Sat Nov 10 11:47:06 2007 @@ -7,7 +7,8 @@ from pypy.translator.oosupport.metavm import \ PushArg, PushAllArgs, StoreResult, InstructionList, New, DoNothing, Call,\ - SetField, GetField, DownCast, RuntimeNew, OOString, CastTo, PushPrimitive + SetField, GetField, DownCast, RuntimeNew, OOString, OOUnicode, \ + CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ CastPrimitive @@ -56,6 +57,7 @@ 'ooidentityhash': [PushAllArgs, jvmgen.OBJHASHCODE, StoreResult], 'oohash': [PushAllArgs, jvmgen.OBJHASHCODE, StoreResult], 'oostring': [OOString, StoreResult], + 'oounicode': [OOUnicode, StoreResult], #'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': jvmgen.PYPYOOPARSEFLOAT, 'oonewcustomdict': [NewCustomDict, StoreResult], Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/prebuiltnodes.py Sat Nov 10 11:47:06 2007 @@ -14,6 +14,9 @@ def throwValueError(): raise ValueError +def throwUnicodeDecodeError(): + raise UnicodeDecodeError + # ___________________________________________________________________________ def create_interlink_node(db): Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/Interlink.java Sat Nov 10 11:47:06 2007 @@ -14,5 +14,6 @@ public void throwIndexError(); public void throwOverflowError(); public void throwValueError(); + public void throwUnicodeDecodeError(); public void throwOSError(int errCode); } Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/PyPy.java Sat Nov 10 11:47:06 2007 @@ -319,6 +319,10 @@ return sb.toString(); } + public static String escaped_unichar(char c) { + return "u" + escaped_char(c); + } + public static String escaped_string(String b) { if (b == null) return "None"; @@ -332,6 +336,10 @@ return sb.toString(); } + public static String escaped_unicode(String b) { + return "u" + escaped_string(b); + } + // used in running unit tests // not really part of the dump_XXX set of objects, hence the lack // of an indent parameter @@ -806,6 +814,24 @@ } // ---------------------------------------------------------------------- + // OOUnicode support + + public static String oounicode(char ch) + { + return new Character(ch).toString(); + } + + public static String oounicode(String s) + { + for(int i=0; i 127) + throwUnicodeDecodeError(); + } + return s; + } + + // ---------------------------------------------------------------------- // Primitive built-in functions public static double ll_time_clock() { @@ -957,6 +983,10 @@ public static void throwValueError() { interlink.throwValueError(); } + + public static void throwUnicodeDecodeError() { + interlink.throwUnicodeDecodeError(); + } // ---------------------------------------------------------------------- // Self Test Modified: pypy/branch/unicode-objspace/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/jvm/test/runtest.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/test/runtest.py Sat Nov 10 11:47:06 2007 @@ -140,6 +140,9 @@ def ll_to_string(self, s): return s + def ll_to_unicode(self, s): + return s + def ll_to_list(self, l): return l Added: pypy/branch/unicode-objspace/pypy/translator/jvm/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/branch/unicode-objspace/pypy/translator/jvm/test/test_unicode.py Sat Nov 10 11:47:06 2007 @@ -0,0 +1,20 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_runicode import BaseTestRUnicode + +# ====> ../../../rpython/test/test_runicode.py + +class TestJvmUnicode(JvmTest, BaseTestRUnicode): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("JVM doesn't support unicode for command line arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_getitem_exc(self): + py.test.skip('fixme!') Modified: pypy/branch/unicode-objspace/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/translator/oosupport/metavm.py (original) +++ pypy/branch/unicode-objspace/pypy/translator/oosupport/metavm.py Sat Nov 10 11:47:06 2007 @@ -452,6 +452,15 @@ generator.load(op.args[1]) generator.call_oostring(ARGTYPE) +class _OOUnicode(MicroInstruction): + def render(self, generator, op): + v_base = op.args[1] + assert v_base.value == -1, "The second argument of oounicode must be -1" + + ARGTYPE = op.args[0].concretetype + generator.load(op.args[0]) + generator.call_oounicode(ARGTYPE) + class _CastTo(MicroInstruction): def render(self, generator, op): generator.load(op.args[0]) @@ -471,5 +480,6 @@ CallMethod = _CallMethod() RuntimeNew = _RuntimeNew() OOString = _OOString() +OOUnicode = _OOUnicode() CastTo = _CastTo() From fijal at codespeak.net Sat Nov 10 12:30:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 12:30:30 +0100 (CET) Subject: [pypy-svn] r48496 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110113030.F043F8178@code0.codespeak.net> Author: fijal Date: Sat Nov 10 12:30:29 2007 New Revision: 48496 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: Hum. A typo? Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 12:30:29 2007 @@ -149,10 +149,10 @@ w_decode = space.getattr(w_codecs, space.wrap("decode")) if errors is None: w_retval, w_length = space.unpacktuple( - space.call(w_decode, obj, space.wrap(encoding)), 2) + space.call(w_decode, w_obj, space.wrap(encoding)), 2) else: w_retval, w_length = space.unpacktuple( - space.call(w_decode, obj, space.wrap(encoding), + space.call(w_decode, w_obj, space.wrap(encoding), space.wrap(errors)), 2) if not space.is_true(space.isinstance(w_retval, space.w_unicode)): raise OperationError( From fijal at codespeak.net Sat Nov 10 12:36:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 12:36:16 +0100 (CET) Subject: [pypy-svn] r48497 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110113616.07F14819E@code0.codespeak.net> Author: fijal Date: Sat Nov 10 12:36:15 2007 New Revision: 48497 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: Another typo. Test anyone? Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 12:36:15 2007 @@ -160,7 +160,7 @@ space.wrap( "decoder did not return an unicode object (type=%s)" % space.type(retval).getname('?'))) - return retval + return w_retval def unicode_from_object(space, w_obj): From fijal at codespeak.net Sat Nov 10 12:43:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 12:43:38 +0100 (CET) Subject: [pypy-svn] r48498 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110114338.9BE0581C0@code0.codespeak.net> Author: fijal Date: Sat Nov 10 12:43:38 2007 New Revision: 48498 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: Another type :-/ Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 12:43:38 2007 @@ -159,7 +159,7 @@ space.w_TypeError, space.wrap( "decoder did not return an unicode object (type=%s)" % - space.type(retval).getname('?'))) + space.type(w_retval).getname('?'))) return w_retval From fijal at codespeak.net Sat Nov 10 12:52:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 12:52:38 +0100 (CET) Subject: [pypy-svn] r48499 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110115238.B448C8153@code0.codespeak.net> Author: fijal Date: Sat Nov 10 12:52:38 2007 New Revision: 48499 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: A real annotation problem this time Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 12:52:38 2007 @@ -831,7 +831,7 @@ raise OperationError( space.w_TypeError, space.wrap("character mapping must return integer, None or unicode")) - return W_UnicodeObject(''.join(result)) + return W_UnicodeObject(u''.join(result)) # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): From fijal at codespeak.net Sat Nov 10 13:05:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 13:05:49 +0100 (CET) Subject: [pypy-svn] r48500 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110120549.60C1981E6@code0.codespeak.net> Author: fijal Date: Sat Nov 10 13:05:49 2007 New Revision: 48500 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py Log: annotator seems not to like this Modified: pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py Sat Nov 10 13:05:49 2007 @@ -42,11 +42,10 @@ except UnicodeDecodeError: for i in range(len(s)): if ord(s[i]) > 127: - break - raise OperationError( - space.w_UnicodeDecodeError, - space.wrap(("'ascii' codec can't decode byte %s in position %s:" - " ordinal not in range(128)") % (hex(ord(i)), i))) + raise OperationError( + space.w_UnicodeDecodeError, + space.wrap(("'ascii' codec can't decode byte %s in position %s:" + " ordinal not in range(128)") % (hex(ord(i)), i))) def unicode_w__String(space, w_self): # XXX should this use the default encoding? From fijal at codespeak.net Sat Nov 10 13:07:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 13:07:03 +0100 (CET) Subject: [pypy-svn] r48501 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110120703.8994B81E6@code0.codespeak.net> Author: fijal Date: Sat Nov 10 13:07:03 2007 New Revision: 48501 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: This is wrong optimisation. It would not work for ie tproxy of unicode or different unicode implementation. Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 13:07:03 2007 @@ -124,7 +124,8 @@ values_list = [] for i in range(len(l)): item = l[i] - if space.is_true(space.isinstance(item, space.w_unicode)): + if isinstance(item, W_UnicodeObject): + # shortcut for performane item = item._value elif space.is_true(space.isinstance(item, space.w_str)): item = space.unicode_w(item) From fijal at codespeak.net Sat Nov 10 13:38:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 13:38:57 +0100 (CET) Subject: [pypy-svn] r48502 - in pypy/branch/unicode-objspace/pypy/objspace/std: . test Message-ID: <20071110123857.28FC481F9@code0.codespeak.net> Author: fijal Date: Sat Nov 10 13:38:56 2007 New Revision: 48502 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: I don't get it, but at least test is passing Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 13:38:56 2007 @@ -365,3 +365,8 @@ raises(TypeError, u'hello'.translate) raises(TypeError, u'abababc'.translate, {ord('a'):''}) + + def test_unicode_form_encoded_object(self): + assert unicode('x', 'utf-8') == u'x' + assert unicode('x', 'utf-8', 'strict') == u'x' + Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 13:38:56 2007 @@ -148,12 +148,12 @@ encoding = space.defaultencoding w_decode = space.getattr(w_codecs, space.wrap("decode")) if errors is None: - w_retval, w_length = space.unpacktuple( - space.call(w_decode, w_obj, space.wrap(encoding)), 2) + w_retval = space.call(w_decode, space.newlist([w_obj, + space.wrap(encoding)])) else: - w_retval, w_length = space.unpacktuple( - space.call(w_decode, w_obj, space.wrap(encoding), - space.wrap(errors)), 2) + w_retval = space.call(w_decode, space.newlist([w_obj, + space.wrap(encoding), + space.wrap(errors)])) if not space.is_true(space.isinstance(w_retval, space.w_unicode)): raise OperationError( space.w_TypeError, @@ -180,7 +180,7 @@ w_res = space.call(w_unicode_method) if space.is_true(space.isinstance(w_res, space.w_unicode)): return w_res - return unicode_from_encoded_object(w_res, None, "strict") + return unicode_from_encoded_object(space, w_res, None, "strict") def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack From fijal at codespeak.net Sat Nov 10 13:57:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 13:57:00 +0100 (CET) Subject: [pypy-svn] r48503 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110125700.29B398178@code0.codespeak.net> Author: fijal Date: Sat Nov 10 13:56:59 2007 New Revision: 48503 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: getname eats space as well Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 13:56:59 2007 @@ -159,7 +159,7 @@ space.w_TypeError, space.wrap( "decoder did not return an unicode object (type=%s)" % - space.type(w_retval).getname('?'))) + space.type(w_retval).getname(space, '?'))) return w_retval From fijal at codespeak.net Sat Nov 10 14:13:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 14:13:42 +0100 (CET) Subject: [pypy-svn] r48505 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110131342.63985814B@code0.codespeak.net> Author: fijal Date: Sat Nov 10 14:13:39 2007 New Revision: 48505 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: Another towards-annotation-complete fixes. Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Sat Nov 10 14:13:39 2007 @@ -442,7 +442,7 @@ fmt = space.unicode_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() - return space.wrap(''.join(result)) + return space.wrap(u''.join(result)) def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 14:13:39 2007 @@ -177,7 +177,7 @@ else: raise else: - w_res = space.call(w_unicode_method) + w_res = space.call(w_unicode_method, space.newlist([])) if space.is_true(space.isinstance(w_res, space.w_unicode)): return w_res return unicode_from_encoded_object(space, w_res, None, "strict") From cfbolz at codespeak.net Sat Nov 10 15:47:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 15:47:42 +0100 (CET) Subject: [pypy-svn] r48506 - in pypy/branch/unicode-objspace/pypy: interpreter module/sys objspace/std Message-ID: <20071110144742.7C4F68186@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 15:47:40 2007 New Revision: 48506 Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: sticking the encoding to the space is a bad idea Modified: pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py Sat Nov 10 15:47:40 2007 @@ -188,7 +188,6 @@ self.interned_strings = {} self.pending_actions = [] self.setoptions(**kw) - self.defaultencoding = "ascii" # if self.config.objspace.logbytecodes: # self.bytecodecounts = {} Modified: pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py Sat Nov 10 15:47:40 2007 @@ -1,8 +1,13 @@ +class EncodingState(object): + def __init__(self, space): + self.space = space + self.defaultencoding = "ascii" + def getdefaultencoding(space): """Return the current default string encoding used by the Unicode implementation.""" - return space.wrap(space.defaultencoding) + return space.wrap(space.fromcache(EncodingState).defaultencoding) def setdefaultencoding(space, w_encoding): """Set the current default string encoding used by the Unicode @@ -12,4 +17,4 @@ w_lookup = space.getattr(mod, space.wrap("lookup")) # check whether the encoding is there space.call_function(w_lookup, w_encoding) - space.defaultencoding = encoding + space.fromcache(EncodingState).defaultencoding = encoding Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 15:47:40 2007 @@ -142,18 +142,22 @@ # ____________________________________________________________ +def getdefaultencoding(space): + w_sys = space.getbuiltinmodule("sys") + return space.str_w( + space.call_function( + space.getattr(w_sys, space.wrap("getdefaultencoding")))) + def unicode_from_encoded_object(space, w_obj, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: - encoding = space.defaultencoding + encoding = getdefaultencoding(space) w_decode = space.getattr(w_codecs, space.wrap("decode")) if errors is None: - w_retval = space.call(w_decode, space.newlist([w_obj, - space.wrap(encoding)])) + w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding)) else: - w_retval = space.call(w_decode, space.newlist([w_obj, - space.wrap(encoding), - space.wrap(errors)])) + w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding), + space.wrap(errors)) if not space.is_true(space.isinstance(w_retval, space.w_unicode)): raise OperationError( space.w_TypeError, @@ -177,7 +181,7 @@ else: raise else: - w_res = space.call(w_unicode_method, space.newlist([])) + w_res = space.call_function(w_unicode_method) if space.is_true(space.isinstance(w_res, space.w_unicode)): return w_res return unicode_from_encoded_object(space, w_res, None, "strict") @@ -185,7 +189,7 @@ def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack from pypy.objspace.std.unicodeobject import W_UnicodeObject - encoding = space.defaultencoding + encoding = getdefaultencoding(space) if encoding != 'ascii': return unicode_from_object(space, w_str) s = space.str_w(w_str) From cfbolz at codespeak.net Sat Nov 10 15:54:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 15:54:17 +0100 (CET) Subject: [pypy-svn] r48507 - in pypy/branch/unicode-objspace/pypy: module/sys objspace/std Message-ID: <20071110145417.098CC81FA@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 15:54:15 2007 New Revision: 48507 Modified: pypy/branch/unicode-objspace/pypy/module/sys/__init__.py pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: yet another approach: do the same thing as with the recursionlimit and stick the encoding directly onto the module. probably a good idea, it's easily accessible and doesn't require a function call. Thanks Alexander. Modified: pypy/branch/unicode-objspace/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/__init__.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/__init__.py Sat Nov 10 15:54:15 2007 @@ -8,6 +8,7 @@ super(Module, self).__init__(space, w_name) self.checkinterval = 100 self.recursionlimit = 100 + self.defaultencoding = "ascii" interpleveldefs = { '__name__' : '(space.wrap("sys"))', Modified: pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py (original) +++ pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py Sat Nov 10 15:54:15 2007 @@ -1,13 +1,7 @@ - -class EncodingState(object): - def __init__(self, space): - self.space = space - self.defaultencoding = "ascii" - def getdefaultencoding(space): """Return the current default string encoding used by the Unicode implementation.""" - return space.wrap(space.fromcache(EncodingState).defaultencoding) + return space.wrap(space.sys.defaultencoding) def setdefaultencoding(space, w_encoding): """Set the current default string encoding used by the Unicode @@ -17,4 +11,4 @@ w_lookup = space.getattr(mod, space.wrap("lookup")) # check whether the encoding is there space.call_function(w_lookup, w_encoding) - space.fromcache(EncodingState).defaultencoding = encoding + space.sys.defaultencoding = encoding Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 15:54:15 2007 @@ -143,10 +143,7 @@ # ____________________________________________________________ def getdefaultencoding(space): - w_sys = space.getbuiltinmodule("sys") - return space.str_w( - space.call_function( - space.getattr(w_sys, space.wrap("getdefaultencoding")))) + return space.sys.defaultencoding def unicode_from_encoded_object(space, w_obj, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") From cfbolz at codespeak.net Sat Nov 10 16:14:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 16:14:08 +0100 (CET) Subject: [pypy-svn] r48508 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110151408.90530820C@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 16:14:06 2007 New Revision: 48508 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Log: helper function for unwrapping encoding and error. will be needed in various places Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Sat Nov 10 16:14:06 2007 @@ -197,22 +197,32 @@ return unicode_from_object(space, w_str) +def _get_encoding_and_errors(space, w_encoding, w_errors): + if space.is_w(w_encoding, space.w_None): + encoding = None + else: + encoding = space.str_w(w_encoding) + if space.is_w(w_errors, space.w_None): + errors = None + else: + errors = space.str_w(w_errors) + return encoding, errors + def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work from pypy.objspace.std.unicodeobject import W_UnicodeObject w_obj_type = space.type(w_obj) + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if space.is_w(w_obj_type, space.w_unicode): - if (not space.is_w(w_encoding, space.w_None) or - not space.is_w(w_errors, space.w_None)): + if encoding is not None or errors is not None: raise OperationError(space.w_TypeError, space.wrap('decoding Unicode is not supported')) if space.is_w(w_unicodetype, space.w_unicode): return w_obj w_value = w_obj - elif (space.is_w(w_encoding, space.w_None) and - space.is_w(w_errors, space.w_None)): + elif encoding is None and errors is None: if space.is_true(space.isinstance(w_obj, space.w_str)): w_value = unicode_from_string(space, w_obj) elif space.is_true(space.isinstance(w_obj, space.w_unicode)): @@ -220,15 +230,6 @@ else: w_value = unicode_from_object(space, w_obj) else: - if space.is_w(w_encoding, space.w_None): - encoding = None - else: - encoding = space.str_w(w_encoding) - if space.is_w(w_errors, space.w_None): - errors = None - else: - errors = space.str_w(w_errors) - errors w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) # help the annotator! also the ._value depends on W_UnicodeObject layout assert isinstance(w_value, W_UnicodeObject) From fijal at codespeak.net Sat Nov 10 16:48:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 16:48:04 +0100 (CET) Subject: [pypy-svn] r48510 - in pypy/branch/unicode-objspace/pypy/objspace/std: . test Message-ID: <20071110154804.B67D981FC@code0.codespeak.net> Author: fijal Date: Sat Nov 10 16:48:04 2007 New Revision: 48510 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: * Slightly factor out unicode_startswith and unicode_endswith * Fix formatting Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Sat Nov 10 16:48:04 2007 @@ -439,7 +439,7 @@ else: return space.wrap(''.join(result)) else: - fmt = space.unicode_w(w_fmt) + fmt = space.unichars_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() return space.wrap(u''.join(result)) Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 16:48:04 2007 @@ -370,3 +370,6 @@ assert unicode('x', 'utf-8') == u'x' assert unicode('x', 'utf-8', 'strict') == u'x' + def test_unicode_startswith_tuple(self): + assert u'xxx'.startswith(('x', 'y', 'z'), 0) + assert u'xxx'.endswith(('x', 'y', 'z'), 0) Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 16:48:04 2007 @@ -407,40 +407,48 @@ return (self, start, end) -def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self, start, end = _convert_idx_params(space, w_self, w_start, w_end) - substr = w_substr._value +def _check_startswith_substring(str, substr, start, end): substr_len = len(substr) if end - start < substr_len: - return space.w_False # substring is too long + return False # substring is too long + + for i in range(substr_len): + if str[start + i] != substr[i]: + return False + return True + +def _check_endswith_substring(str, substr, start, end): + substr_len = len(substr) + + if end - start < substr_len: + return False # substring is too long start = end - substr_len for i in range(substr_len): - if self[start + i] != substr[i]: - return space.w_False - return space.w_True + if str[start + i] != substr[i]: + return False + return True + +def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + substr = w_substr._value + return space.wrap(_check_endswith_substring(self, substr, start, end)) def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + # XXX this stuff can be waaay better for ootypebased backends if + # we re-use more of our rpython machinery (ie implement startswith + # with additional parameters as rpython) substr = w_substr._value - substr_len = len(substr) - - if end - start < substr_len: - return space.w_False # substring is too long - - for i in range(substr_len): - if self[start + i] != substr[i]: - return space.w_False - return space.w_True - + return space.wrap(_check_startswith_substring(self, substr, start, end)) def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): prefix = space.unicode_w(w_prefix) - if unistr.startswith(prefix, start, end): + if _check_startswith_substring(unistr, prefix, start, end): return space.w_True return space.w_False @@ -449,7 +457,7 @@ unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): suffix = space.unicode_w(w_suffix) - if unistr.endswith(suffix, start, end): + if _check_endswith_substring(unistr, suffix, start, end): return space.w_True return space.w_False From cfbolz at codespeak.net Sat Nov 10 16:53:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 10 Nov 2007 16:53:34 +0100 (CET) Subject: [pypy-svn] r48511 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110155334.6F1EE81FC@code0.codespeak.net> Author: cfbolz Date: Sat Nov 10 16:53:34 2007 New Revision: 48511 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: translate is no longer at applevel Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 16:53:34 2007 @@ -756,7 +756,6 @@ -unicode_translate__Unicode_ANY = app.interphook('unicode_translate__Unicode_ANY') unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): From fijal at codespeak.net Sat Nov 10 16:58:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 16:58:11 +0100 (CET) Subject: [pypy-svn] r48512 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110155811.407258200@code0.codespeak.net> Author: fijal Date: Sat Nov 10 16:58:10 2007 New Revision: 48512 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py Log: Typo Modified: pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py Sat Nov 10 16:58:10 2007 @@ -45,7 +45,7 @@ raise OperationError( space.w_UnicodeDecodeError, space.wrap(("'ascii' codec can't decode byte %s in position %s:" - " ordinal not in range(128)") % (hex(ord(i)), i))) + " ordinal not in range(128)") % (hex(ord(s[i])), i))) def unicode_w__String(space, w_self): # XXX should this use the default encoding? From fijal at codespeak.net Sat Nov 10 17:01:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 17:01:19 +0100 (CET) Subject: [pypy-svn] r48513 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110160119.8FC038200@code0.codespeak.net> Author: fijal Date: Sat Nov 10 17:01:19 2007 New Revision: 48513 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: Another typo Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 17:01:19 2007 @@ -180,7 +180,7 @@ length = len(uni) start, stop, step, sl = w_slice.indices4(space, length) if sl == 0: - r = [] + r = u"" elif step == 1: assert start >= 0 and stop >= 0 r = uni[start:stop] From fijal at codespeak.net Sat Nov 10 17:05:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 17:05:19 +0100 (CET) Subject: [pypy-svn] r48514 - pypy/branch/unicode-objspace/pypy/objspace/std/test Message-ID: <20071110160519.49CF281FA@code0.codespeak.net> Author: fijal Date: Sat Nov 10 17:05:18 2007 New Revision: 48514 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Log: Add test for missing case Modified: pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py Sat Nov 10 17:05:18 2007 @@ -373,3 +373,7 @@ def test_unicode_startswith_tuple(self): assert u'xxx'.startswith(('x', 'y', 'z'), 0) assert u'xxx'.endswith(('x', 'y', 'z'), 0) + + def test_missing_cases(self): + # some random cases, which are discovered to not be tested during annotation + assert u'xxx'[1:1] == u'' From fijal at codespeak.net Sat Nov 10 17:18:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 17:18:36 +0100 (CET) Subject: [pypy-svn] r48515 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110161836.3351B8210@code0.codespeak.net> Author: fijal Date: Sat Nov 10 17:18:35 2007 New Revision: 48515 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Log: Actually *run* the tests before checking in the fix Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Sat Nov 10 17:18:35 2007 @@ -435,11 +435,11 @@ result = formatter.format() except NeedUnicodeFormattingError: # fall through to the unicode case - fmt = [c for c in fmt] # string => list of unichars + fmt = unicode(fmt) else: return space.wrap(''.join(result)) else: - fmt = space.unichars_w(w_fmt) + fmt = space.unicode_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() return space.wrap(u''.join(result)) From fijal at codespeak.net Sat Nov 10 18:19:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 18:19:30 +0100 (CET) Subject: [pypy-svn] r48518 - pypy/extradoc/talk/pycon2008 Message-ID: <20071110171930.3ECE281FC@code0.codespeak.net> Author: fijal Date: Sat Nov 10 18:19:29 2007 New Revision: 48518 Added: pypy/extradoc/talk/pycon2008/ pypy/extradoc/talk/pycon2008/proxy-abstract.txt (contents, props changed) Log: First draft of pycon talk. Added: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Sat Nov 10 18:19:29 2007 @@ -0,0 +1,26 @@ +XXX Title (inside pypy seems to be too "inside") +================================================ + +In this talk, we would like to present few features exclusive to +the PyPy python interpret and novel-implementation approaches +which made it easy to implement those features. The talk would +consist on: + +* Transparent proxy cocept, which allows programmer to create + transparent object of any type in python (including built-in + python types like frames), with various usage showcases icluding + our completely transparent remote access scheme. + +* Sandboxing concept, which allows to run non-stripped version of + python interpreter in controlled environment with custom + safety policy. + +* Taint objspace, which allows programmer to make sure that sensitive + information does not cross the I/O barrier. + +* XXX anything else? + +* Why our approach is "working by design", rather then "fix as long as + anyone complains", why it's only permitted by our design decisions and + abstraction levels. How our all of the mentioned above examples could + be reduce to simple "proxy operation" concept. From jacob at codespeak.net Sat Nov 10 18:26:45 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Sat, 10 Nov 2007 18:26:45 +0100 (CET) Subject: [pypy-svn] r48521 - pypy/extradoc/talk/roadshow-google Message-ID: <20071110172645.73AA981D9@code0.codespeak.net> Author: jacob Date: Sat Nov 10 18:26:44 2007 New Revision: 48521 Added: pypy/extradoc/talk/roadshow-google/Overview.pdf (contents, props changed) Log: Single slide overview of PyPy architecture. Added: pypy/extradoc/talk/roadshow-google/Overview.pdf ============================================================================== Binary file. No diff available. From fijal at codespeak.net Sat Nov 10 18:30:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 18:30:06 +0100 (CET) Subject: [pypy-svn] r48522 - in pypy/branch/unicode-objspace/pypy/rlib: . test Message-ID: <20071110173006.F2DBF8202@code0.codespeak.net> Author: fijal Date: Sat Nov 10 18:30:05 2007 New Revision: 48522 Modified: pypy/branch/unicode-objspace/pypy/rlib/rsocket.py pypy/branch/unicode-objspace/pypy/rlib/test/test_rsocket.py Log: * Fix rpython warnings * Fix test (might be 'www' instead of 'http') Modified: pypy/branch/unicode-objspace/pypy/rlib/rsocket.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rlib/rsocket.py (original) +++ pypy/branch/unicode-objspace/pypy/rlib/rsocket.py Sat Nov 10 18:30:05 2007 @@ -128,6 +128,11 @@ raise RSocketError("unknown address family") from_object = staticmethod(from_object) + def fill_from_object(self, space, w_address): + """ Purely abstract + """ + raise NotImplementedError + # ____________________________________________________________ def makeipaddr(name, result=None): @@ -189,6 +194,11 @@ host, serv = getnameinfo(self, NI_NUMERICHOST | NI_NUMERICSERV) return host + def lock_in_addr(self): + """ Purely abstract + """ + raise NotImplementedError + # ____________________________________________________________ class INETAddress(IPAddress): Modified: pypy/branch/unicode-objspace/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/rlib/test/test_rsocket.py (original) +++ pypy/branch/unicode-objspace/pypy/rlib/test/test_rsocket.py Sat Nov 10 18:30:05 2007 @@ -1,6 +1,7 @@ import py, errno, sys from pypy.rlib import rsocket from pypy.rlib.rsocket import * +import socket as cpy_socket # cannot test error codes in Win32 because ll2ctypes doesn't save # the errors that WSAGetLastError() should return, making it likely @@ -80,8 +81,8 @@ assert getservbyname('http', 'tcp') == 80 def test_getservbyport(): - assert getservbyport(80) == 'http' - assert getservbyport(80, 'tcp') == 'http' + assert getservbyport(80) == cpy_socket.getservbyport(80) + assert getservbyport(80, 'tcp') == cpy_socket.getservbyport(80) def test_getprotobyname(): assert getprotobyname('tcp') == IPPROTO_TCP From fijal at codespeak.net Sat Nov 10 19:45:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 19:45:34 +0100 (CET) Subject: [pypy-svn] r48525 - in pypy/branch/unicode-objspace/pypy/annotation: . test Message-ID: <20071110184534.4774C81FA@code0.codespeak.net> Author: fijal Date: Sat Nov 10 19:45:32 2007 New Revision: 48525 Modified: pypy/branch/unicode-objspace/pypy/annotation/binaryop.py pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py Log: Support for reporting negative slices early, as it sounds unlikely that support will happen soon. Modified: pypy/branch/unicode-objspace/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/annotation/binaryop.py (original) +++ pypy/branch/unicode-objspace/pypy/annotation/binaryop.py Sat Nov 10 19:45:32 2007 @@ -599,19 +599,28 @@ lst1.listdef.resize() delitem.can_only_throw = [IndexError] +def check_negative_slice(s_slice): + if isinstance(s_slice.start, SomeInteger) and not s_slice.start.nonneg: + raise TypeError("%s not proven to have negative start" % s_slice) + if isinstance(s_slice.stop, SomeInteger) and not s_slice.stop.nonneg: + raise TypeError("%s not proven to have negative stop" % s_slice) + class __extend__(pairtype(SomeList, SomeSlice)): def getitem((lst, slic)): + check_negative_slice(slic) return lst.listdef.offspring() getitem.can_only_throw = [] def setitem((lst, slic), s_iterable): + check_negative_slice(slic) # we need the same unifying effect as the extend() method for # the case lst1[x:y] = lst2. lst.method_extend(s_iterable) setitem.can_only_throw = [] def delitem((lst1, slic)): + check_negative_slice(slic) lst1.listdef.resize() delitem.can_only_throw = [] @@ -619,6 +628,7 @@ pairtype(SomeUnicodeString, SomeSlice)): def getitem((str1, slic)): + check_negative_slice(slic) return str1.basestringclass() getitem.can_only_throw = [] Modified: pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py Sat Nov 10 19:45:32 2007 @@ -2962,6 +2962,15 @@ s = a.build_types(f, [unicode, str]) assert isinstance(s, annmodel.SomeUnicodeCodePoint) + def test_negative_slice(self): + def f(s, e): + return [1, 2, 3][s:e] + + a = self.RPythonAnnotator() + py.test.raises(TypeError, "a.build_types(f, [int, int])") + a.build_types(f, [annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True)]) + def g(n): return [0,1,2,n] From fijal at codespeak.net Sat Nov 10 19:57:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 19:57:15 +0100 (CET) Subject: [pypy-svn] r48526 - in pypy/branch/unicode-objspace/pypy/annotation: . test Message-ID: <20071110185715.75A6C8207@code0.codespeak.net> Author: fijal Date: Sat Nov 10 19:57:15 2007 New Revision: 48526 Modified: pypy/branch/unicode-objspace/pypy/annotation/binaryop.py pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py Log: The case str[:-1] is also fine. Modified: pypy/branch/unicode-objspace/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/annotation/binaryop.py (original) +++ pypy/branch/unicode-objspace/pypy/annotation/binaryop.py Sat Nov 10 19:57:15 2007 @@ -602,7 +602,8 @@ def check_negative_slice(s_slice): if isinstance(s_slice.start, SomeInteger) and not s_slice.start.nonneg: raise TypeError("%s not proven to have negative start" % s_slice) - if isinstance(s_slice.stop, SomeInteger) and not s_slice.stop.nonneg: + if isinstance(s_slice.stop, SomeInteger) and not s_slice.stop.nonneg and\ + getattr(s_slice.stop, 'const', 0) != -1: raise TypeError("%s not proven to have negative stop" % s_slice) class __extend__(pairtype(SomeList, SomeSlice)): Modified: pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/unicode-objspace/pypy/annotation/test/test_annrpython.py Sat Nov 10 19:57:15 2007 @@ -2970,6 +2970,10 @@ py.test.raises(TypeError, "a.build_types(f, [int, int])") a.build_types(f, [annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True)]) + def f(x): + return x[:-1] + + a.build_types(f, [str]) def g(n): return [0,1,2,n] From fijal at codespeak.net Sat Nov 10 20:32:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 10 Nov 2007 20:32:36 +0100 (CET) Subject: [pypy-svn] r48527 - pypy/branch/unicode-objspace/pypy/objspace/std Message-ID: <20071110193236.07255820B@code0.codespeak.net> Author: fijal Date: Sat Nov 10 20:32:35 2007 New Revision: 48527 Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Log: A bunch of accumulated changes to make this rpython. Modified: pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py Sat Nov 10 20:32:35 2007 @@ -132,11 +132,16 @@ # to build two subclasses of the BaseStringFormatter class, # each one getting its own subtle differences and RPython types. + if do_unicode: + const = unicode + else: + const = str + class StringFormatter(BaseStringFormatter): def __init__(self, space, fmt, values_w, w_valuedict): BaseStringFormatter.__init__(self, space, values_w, w_valuedict) - self.fmt = fmt # either a string or a list of unichars + self.fmt = fmt # either a string or a unicode def peekchr(self): # return the 'current' character @@ -262,9 +267,9 @@ break i += 1 else: - result += fmt[i0:] + result.append(const(fmt[i0:])) break # end of 'fmt' string - result += fmt[i0:i] + result.append(const(fmt[i0:i])) self.fmtpos = i + 1 # interpret the next formatter @@ -317,10 +322,12 @@ result = self.result padding = self.width - length if not self.f_ljust: - result += ' ' * padding # add any padding at the left of 'r' + result.append(const(' ' * padding)) + # add any padding at the left of 'r' padding = 0 - result += r[:length] # add 'r' itself - result += ' ' * padding # add any remaining padding at the right + result.append(const(r[:length])) # add 'r' itself + result.append(const(' ' * padding)) + # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' def std_wp_number(self, r, prefix=''): @@ -347,15 +354,15 @@ padnumber = '>' if padnumber == '>': - result += ' ' * padding # pad with spaces on the left + result.append(const(' ' * padding)) # pad with spaces on the left if sign: - result.append(r[0]) # the sign - result += prefix # the prefix + result.append(const(r[0])) # the sign + result.append(const(prefix)) # the prefix if padnumber == '0': - result += '0' * padding # pad with zeroes - result += r[int(sign):] # the rest of the number + result.append(const('0' * padding)) # pad with zeroes + result.append(const(r[int(sign):])) # the rest of the number if padnumber == '<': # spaces on the right - result += ' ' * padding + result.append(const(' ' * padding)) def fmt_s(self, w_value): space = self.space @@ -386,11 +393,11 @@ elif space.is_true(space.isinstance(w_value, space.w_unicode)): if not do_unicode: raise NeedUnicodeFormattingError - lst = space.unicode_w(w_value) - if len(lst) != 1: + ustr = space.unicode_w(w_value) + if len(ustr) != 1: raise OperationError(space.w_TypeError, space.wrap("%c requires int or unichar")) - self.std_wp(lst) + self.std_wp(ustr) else: n = space.int_w(w_value) if do_unicode: @@ -399,7 +406,7 @@ except ValueError: raise OperationError(space.w_OverflowError, space.wrap("unicode character code out of range")) - self.std_wp([c]) + self.std_wp(c) else: try: s = chr(n) Modified: pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py Sat Nov 10 20:32:35 2007 @@ -201,7 +201,7 @@ return W_UnicodeObject.EMPTY try: result_size = ovfcheck(length * times) - result = uni * times + result = u''.join([uni] * times) except (OverflowError, MemoryError): raise OperationError(space.w_OverflowError, space.wrap('repeated string is too long')) return W_UnicodeObject(result) @@ -769,6 +769,7 @@ return space.newtuple([w_unistr, W_UnicodeObject.EMPTY, W_UnicodeObject.EMPTY]) else: + assert pos > 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, space.wrap(unistr[pos+len(unisub):])]) @@ -783,6 +784,7 @@ return space.newtuple([W_UnicodeObject.EMPTY, W_UnicodeObject.EMPTY, w_unistr]) else: + assert pos > 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, space.wrap(unistr[pos+len(unisub):])]) From rxe at codespeak.net Sat Nov 10 22:05:01 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:05:01 +0100 (CET) Subject: [pypy-svn] r48528 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071110210501.D62E2819B@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:05:00 2007 New Revision: 48528 Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py Log: failing test on linux only Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_extfunc.py Sat Nov 10 22:05:00 2007 @@ -15,6 +15,8 @@ assert abs(f()-fn()) < 10.0 def test_external_function_ll_time_clock(): + if sys.platform == 'linux2': + py.test.skip('fix me') import time def fn(): return time.clock() From rxe at codespeak.net Sat Nov 10 22:16:50 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:16:50 +0100 (CET) Subject: [pypy-svn] r48529 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071110211650.417FC819B@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:16:48 2007 New Revision: 48529 Modified: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Log: skip these until ready Modified: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Sat Nov 10 22:16:48 2007 @@ -1,6 +1,7 @@ import py from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin from pypy.translator.llvm.test.runtest import * +py.test.skip("skip these until we have resolved most external functions issues") class TestLLVMBuiltin(LLVMTest, BaseTestRbuiltin): pass From rxe at codespeak.net Sat Nov 10 22:22:08 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:22:08 +0100 (CET) Subject: [pypy-svn] r48530 - pypy/dist/pypy/translator/llvm Message-ID: <20071110212208.6317D821D@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:22:08 2007 New Revision: 48530 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: why did i think rtuples were homoegeneous? Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Sat Nov 10 22:22:08 2007 @@ -90,10 +90,13 @@ else: return None -def struct_to_tuple(res, size, C_TYPE, action): +def struct_to_tuple(res, C_TYPE_actions): if res: - t = ctypes.cast(res, ctypes.POINTER(C_TYPE * size)).contents - return {'type':'tuple', 'value':tuple([action(ii) for ii in t])} + class S(ctypes.Structure): + _fields_ = [("item%%s" %% ii, C_TYPE) for ii, (C_TYPE, _) in enumerate(C_TYPE_actions)] + s = ctypes.cast(res, ctypes.POINTER(S)).contents + items = [action(getattr(s, 'item%%s' %% ii)) for ii, (_, action) in enumerate(C_TYPE_actions)] + return {'type':'tuple', 'value':tuple(items)} else: return None @@ -234,10 +237,9 @@ if fields: F0, name = fields[0] if name.startswith("item"): - _c_type, _action = self.build_lltype_to_ctypes_to_res(F0) - action = self.create_simple_closure('res', 'struct_to_tuple(res, %s, %s, %s)' % (len(fields), - _c_type, - _action)) + ctype_actions = "%s" % [self.build_lltype_to_ctypes_to_res(f[0]) for f in fields] + ctype_actions = ctype_actions.replace("'", "") + action = self.create_simple_closure('res', 'struct_to_tuple(res, %s)' % (ctype_actions)) elif name == "length" and fields[1][1] == "items": _c_type, _action = self.build_lltype_to_ctypes_to_res(fields[1][0]) action = self.create_simple_closure('res', 'list_to_array(res, %s)' % (_action)) From rxe at codespeak.net Sat Nov 10 22:22:49 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:22:49 +0100 (CET) Subject: [pypy-svn] r48531 - pypy/dist/pypy/translator/llvm Message-ID: <20071110212249.2E6CD821F@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:22:48 2007 New Revision: 48531 Modified: pypy/dist/pypy/translator/llvm/database.py Log: work around llvm2.1 bug, seems it doesnt like constants for floats Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat Nov 10 22:22:48 2007 @@ -473,7 +473,9 @@ assert len(packed) == 4 repr = "0x" + "".join([("%02x" % ord(ii)) for ii in packed]) else: - repr = "%f" % f + #repr = "%f" % f + # XXX work around llvm2.1 bug, seems it doesnt like constants for floats + repr = "fptrunc(double %f to float)" % f # llvm requires a . when using e notation if "e" in repr and "." not in repr: From rxe at codespeak.net Sat Nov 10 22:26:17 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:26:17 +0100 (CET) Subject: [pypy-svn] r48532 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071110212617.F0E268225@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:26:17 2007 New Revision: 48532 Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py Log: unskip a test Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rstr.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rstr.py Sat Nov 10 22:26:17 2007 @@ -3,13 +3,12 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMStr(LLVMTest, BaseTestRstr): + EMPTY_STRING_HASH = -1 + def test_int(self): py.test.skip('XXX special case me') def test_float(self): py.test.skip('XXX special case me') - def test_hash(self): - py.test.skip('XXX special case me') - From rxe at codespeak.net Sat Nov 10 22:38:38 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 10 Nov 2007 22:38:38 +0100 (CET) Subject: [pypy-svn] r48533 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071110213838.E06078229@code0.codespeak.net> Author: rxe Date: Sat Nov 10 22:38:37 2007 New Revision: 48533 Modified: pypy/dist/pypy/translator/llvm/test/test_rpbc.py Log: one failing test - lets skip it for now Modified: pypy/dist/pypy/translator/llvm/test/test_rpbc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rpbc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rpbc.py Sat Nov 10 22:38:37 2007 @@ -3,4 +3,6 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMPBC(LLVMTest, BaseTestRPBC): - pass + def test_disjoint_pbcs_2(self): + py.test.skip('XXX no idea why this failing FIXME') + From fijal at codespeak.net Sun Nov 11 13:38:14 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Nov 2007 13:38:14 +0100 (CET) Subject: [pypy-svn] r48558 - in pypy/dist/pypy: annotation annotation/test interpreter interpreter/test module/__builtin__ module/_sre module/sys module/sys/test module/unicodedata objspace objspace/cpy objspace/cpy/test objspace/fake objspace/std objspace/std/test rlib rlib/test translator translator/cli translator/jvm translator/jvm/src/pypy translator/jvm/test translator/oosupport Message-ID: <20071111123814.6BC768240@code0.codespeak.net> Author: fijal Date: Sun Nov 11 13:38:13 2007 New Revision: 48558 Added: pypy/dist/pypy/module/sys/interp_encoding.py (contents, props changed) pypy/dist/pypy/translator/jvm/test/test_unicode.py (contents, props changed) Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/test/test_objspace.py pypy/dist/pypy/interpreter/test/test_typedef.py pypy/dist/pypy/module/__builtin__/operation.py pypy/dist/pypy/module/_sre/interp_sre.py pypy/dist/pypy/module/sys/__init__.py pypy/dist/pypy/module/sys/app.py pypy/dist/pypy/module/sys/test/test_sysmodule.py pypy/dist/pypy/module/unicodedata/interp_ucd.py pypy/dist/pypy/objspace/cpy/objspace.py pypy/dist/pypy/objspace/cpy/test/test_objspace.py pypy/dist/pypy/objspace/dump.py pypy/dist/pypy/objspace/fake/objspace.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/std/default.py pypy/dist/pypy/objspace/std/formatting.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/test/test_index.py pypy/dist/pypy/objspace/std/test/test_unicodeobject.py pypy/dist/pypy/objspace/std/unicodeobject.py pypy/dist/pypy/objspace/std/unicodetype.py pypy/dist/pypy/objspace/thunk.py pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/geninterplevel.py pypy/dist/pypy/translator/jvm/builtin.py pypy/dist/pypy/translator/jvm/conftest.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/prebuiltnodes.py pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/runtest.py pypy/dist/pypy/translator/oosupport/metavm.py Log: (antocuni, cfbolz, fijal) - Merge the unicode-objspace branch, history below ------------------------------------------------------------------------ r48508 | cfbolz | 2007-11-10 16:14:06 +0100 (Sat, 10 Nov 2007) | 3 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py helper function for unwrapping encoding and error. will be needed in various places ------------------------------------------------------------------------ r48507 | cfbolz | 2007-11-10 15:54:15 +0100 (Sat, 10 Nov 2007) | 4 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/module/sys/__init__.py M /pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py yet another approach: do the same thing as with the recursionlimit and stick the encoding directly onto the module. probably a good idea, it's easily accessible and doesn't require a function call. Thanks Alexander. ------------------------------------------------------------------------ r48506 | cfbolz | 2007-11-10 15:47:40 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py M /pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py sticking the encoding to the space is a bad idea ------------------------------------------------------------------------ r48505 | fijal | 2007-11-10 14:13:39 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Another towards-annotation-complete fixes. ------------------------------------------------------------------------ r48503 | fijal | 2007-11-10 13:56:59 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py getname eats space as well ------------------------------------------------------------------------ r48502 | fijal | 2007-11-10 13:38:56 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py I don't get it, but at least test is passing ------------------------------------------------------------------------ r48501 | fijal | 2007-11-10 13:07:03 +0100 (Sat, 10 Nov 2007) | 3 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py This is wrong optimisation. It would not work for ie tproxy of unicode or different unicode implementation. ------------------------------------------------------------------------ r48500 | fijal | 2007-11-10 13:05:49 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py annotator seems not to like this ------------------------------------------------------------------------ r48499 | fijal | 2007-11-10 12:52:38 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py A real annotation problem this time ------------------------------------------------------------------------ r48498 | fijal | 2007-11-10 12:43:38 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Another type :-/ ------------------------------------------------------------------------ r48497 | fijal | 2007-11-10 12:36:15 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Another typo. Test anyone? ------------------------------------------------------------------------ r48496 | fijal | 2007-11-10 12:30:29 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py Hum. A typo? ------------------------------------------------------------------------ r48495 | antocuni | 2007-11-10 11:47:06 +0100 (Sat, 10 Nov 2007) | 3 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/translator/cli/function.py M /pypy/branch/unicode-objspace/pypy/translator/cli/metavm.py M /pypy/branch/unicode-objspace/pypy/translator/cli/opcodes.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/builtin.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/conftest.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/database.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/generator.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/opcodes.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/prebuiltnodes.py M /pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/Interlink.java M /pypy/branch/unicode-objspace/pypy/translator/jvm/src/pypy/PyPy.java M /pypy/branch/unicode-objspace/pypy/translator/jvm/test/runtest.py A /pypy/branch/unicode-objspace/pypy/translator/jvm/test/test_unicode.py M /pypy/branch/unicode-objspace/pypy/translator/oosupport/metavm.py add unicode support to genjvm ------------------------------------------------------------------------ r48494 | cfbolz | 2007-11-10 03:13:55 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py part 2: no more applevel code in the __new__ ------------------------------------------------------------------------ r48493 | cfbolz | 2007-11-10 02:56:44 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py more applevel moving: this time in unicode.__new__ (!). Part 1 ------------------------------------------------------------------------ r48492 | cfbolz | 2007-11-10 02:42:28 +0100 (Sat, 10 Nov 2007) | 3 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py move translate from app to interplevel. add tests. Why is the test coverage of unicode so terribly spotty? ------------------------------------------------------------------------ r48491 | cfbolz | 2007-11-10 02:26:22 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py move expandtabs from app to interplevel ------------------------------------------------------------------------ r48490 | cfbolz | 2007-11-10 02:13:44 +0100 (Sat, 10 Nov 2007) | 4 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py M /pypy/branch/unicode-objspace/pypy/module/sys/__init__.py M /pypy/branch/unicode-objspace/pypy/module/sys/app.py A /pypy/branch/unicode-objspace/pypy/module/sys/interp_encoding.py M /pypy/branch/unicode-objspace/pypy/module/sys/test/test_sysmodule.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py attach the default-encoding to the space for easier accessibility. not sure this is a very nice solution, but I think it is way better than a global in an applevel implementation module in the sys mixed-module. ------------------------------------------------------------------------ r48486 | cfbolz | 2007-11-10 01:28:38 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py whoops, that was nonsense ------------------------------------------------------------------------ r48485 | cfbolz | 2007-11-10 01:25:59 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/test/test_index.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py move more stuff to interplevel. also fix indexing behaviour + tests ------------------------------------------------------------------------ r48484 | cfbolz | 2007-11-10 00:50:26 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py partition, rpartition go from applevel to interplevel ------------------------------------------------------------------------ r48483 | cfbolz | 2007-11-10 00:40:56 +0100 (Sat, 10 Nov 2007) | 2 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py M /pypy/branch/unicode-objspace/pypy/interpreter/test/test_objspace.py M /pypy/branch/unicode-objspace/pypy/interpreter/test/test_typedef.py M /pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py M /pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/dump.py M /pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/logic.py M /pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/thunk.py M /pypy/branch/unicode-objspace/pypy/translator/geninterplevel.py get rid of newstring as well ------------------------------------------------------------------------ r48482 | cfbolz | 2007-11-10 00:24:58 +0100 (Sat, 10 Nov 2007) | 3 lines Changed paths: M /pypy/branch/unicode-objspace/pypy/interpreter/baseobjspace.py M /pypy/branch/unicode-objspace/pypy/module/__builtin__/operation.py M /pypy/branch/unicode-objspace/pypy/module/_sre/interp_sre.py M /pypy/branch/unicode-objspace/pypy/module/unicodedata/interp_ucd.py M /pypy/branch/unicode-objspace/pypy/objspace/cpy/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/cpy/test/test_objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/dump.py M /pypy/branch/unicode-objspace/pypy/objspace/fake/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/logic.py M /pypy/branch/unicode-objspace/pypy/objspace/std/default.py M /pypy/branch/unicode-objspace/pypy/objspace/std/formatting.py M /pypy/branch/unicode-objspace/pypy/objspace/std/objspace.py M /pypy/branch/unicode-objspace/pypy/objspace/std/stringobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodeobject.py M /pypy/branch/unicode-objspace/pypy/objspace/std/unicodetype.py M /pypy/branch/unicode-objspace/pypy/objspace/thunk.py plotch. refactor the std object space to use rpython unicode objects to represent unicode objects. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Sun Nov 11 13:38:13 2007 @@ -599,19 +599,29 @@ lst1.listdef.resize() delitem.can_only_throw = [IndexError] +def check_negative_slice(s_slice): + if isinstance(s_slice.start, SomeInteger) and not s_slice.start.nonneg: + raise TypeError("%s not proven to have negative start" % s_slice) + if isinstance(s_slice.stop, SomeInteger) and not s_slice.stop.nonneg and\ + getattr(s_slice.stop, 'const', 0) != -1: + raise TypeError("%s not proven to have negative stop" % s_slice) + class __extend__(pairtype(SomeList, SomeSlice)): def getitem((lst, slic)): + check_negative_slice(slic) return lst.listdef.offspring() getitem.can_only_throw = [] def setitem((lst, slic), s_iterable): + check_negative_slice(slic) # we need the same unifying effect as the extend() method for # the case lst1[x:y] = lst2. lst.method_extend(s_iterable) setitem.can_only_throw = [] def delitem((lst1, slic)): + check_negative_slice(slic) lst1.listdef.resize() delitem.can_only_throw = [] @@ -619,6 +629,7 @@ pairtype(SomeUnicodeString, SomeSlice)): def getitem((str1, slic)): + check_negative_slice(slic) return str1.basestringclass() getitem.can_only_throw = [] Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Sun Nov 11 13:38:13 2007 @@ -2962,6 +2962,19 @@ s = a.build_types(f, [unicode, str]) assert isinstance(s, annmodel.SomeUnicodeCodePoint) + def test_negative_slice(self): + def f(s, e): + return [1, 2, 3][s:e] + + a = self.RPythonAnnotator() + py.test.raises(TypeError, "a.build_types(f, [int, int])") + a.build_types(f, [annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True)]) + def f(x): + return x[:-1] + + a.build_types(f, [str]) + def g(n): return [0,1,2,n] Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Sun Nov 11 13:38:13 2007 @@ -1036,8 +1036,6 @@ # is_true(w_x) -> True or False # newtuple([w_1, w_2,...]) -> w_tuple # newlist([w_1, w_2,...]) -> w_list -# newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) -# newunicode([i1, i2,...]) -> w_unicode from integers # newdict() -> empty w_dict # newslice(w_start,w_stop,w_step) -> w_slice # call_args(w_obj,Arguments()) -> w_result @@ -1049,15 +1047,13 @@ 'float_w', 'uint_w', 'bigint_w', - 'unichars_w', + 'unicode_w', 'interpclass_w', 'unwrap', 'is_true', 'is_w', 'newtuple', 'newlist', - 'newstring', - 'newunicode', 'newdict', 'newslice', 'call_args', Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Sun Nov 11 13:38:13 2007 @@ -9,23 +9,6 @@ # -- it's more to test that it's *there* class TestObjSpace: - def test_newstring(self): - w = self.space.wrap - s = 'abc' - chars_w = [w(ord(c)) for c in s] - assert self.space.eq_w(w(s), self.space.newstring(chars_w)) - - def test_newstring_fail(self): - w = self.space.wrap - s = 'abc' - not_chars_w = [w(c) for c in s] - self.space.raises_w(self.space.w_TypeError, - self.space.newstring, - not_chars_w) - self.space.raises_w(self.space.w_ValueError, - self.space.newstring, - [w(-1)]) - def test_newlist(self): w = self.space.wrap l = range(10) Modified: pypy/dist/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_typedef.py (original) +++ pypy/dist/pypy/interpreter/test/test_typedef.py Sun Nov 11 13:38:13 2007 @@ -5,6 +5,7 @@ class AppTestTraceBackAttributes: def test_newstring(self): + # XXX why is this called newstring? import sys def f(): raise TypeError, "hello" Modified: pypy/dist/pypy/module/__builtin__/operation.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/operation.py (original) +++ pypy/dist/pypy/module/__builtin__/operation.py Sun Nov 11 13:38:13 2007 @@ -14,8 +14,12 @@ def chr(space, w_ascii): "Return a string of one character with the given ascii code." - w_character = space.newstring([w_ascii]) - return w_character + try: + char = __builtin__.chr(space.int_w(w_ascii)) + except ValueError: # chr(out-of-range) + raise OperationError(space.w_ValueError, + space.wrap("character code not in range(256)")) + return space.wrap(char) def unichr(space, code): "Return a Unicode string of one character with the given ordinal." @@ -25,7 +29,7 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("unichr() arg out of range")) - return space.newunicode([c]) + return space.wrap(c) unichr.unwrap_spec = [ObjSpace, int] def len(space, w_obj): Modified: pypy/dist/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/dist/pypy/module/_sre/interp_sre.py (original) +++ pypy/dist/pypy/module/_sre/interp_sre.py Sun Nov 11 13:38:13 2007 @@ -150,11 +150,11 @@ rsre.insert_sre_methods(locals(), 'unicode') def unwrap_object(self): - self.unichars = self.space.unichars_w(self.w_string) - return len(self.unichars) + self.unicode = self.space.unicode_w(self.w_string) + return len(self.unicode) def get_char_ord(self, p): - return ord(self.unichars[p]) + return ord(self.unicode[p]) class W_GenericState(W_State): Modified: pypy/dist/pypy/module/sys/__init__.py ============================================================================== --- pypy/dist/pypy/module/sys/__init__.py (original) +++ pypy/dist/pypy/module/sys/__init__.py Sun Nov 11 13:38:13 2007 @@ -8,6 +8,7 @@ super(Module, self).__init__(space, w_name) self.checkinterval = 100 self.recursionlimit = 100 + self.defaultencoding = "ascii" interpleveldefs = { '__name__' : '(space.wrap("sys"))', @@ -64,6 +65,9 @@ 'path_hooks' : 'space.wrap([])', 'path_importer_cache' : 'space.wrap({})', #'subversion' : added in Python 2.5 + + 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', + 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', } appleveldefs = { #'displayhook' : 'app.displayhook', @@ -75,8 +79,6 @@ 'pypy__exithandlers__' : 'app.pypy__exithandlers__', # internal 'getfilesystemencoding' : 'app.getfilesystemencoding', 'callstats' : 'app.callstats', - 'getdefaultencoding' : 'app.getdefaultencoding', - 'setdefaultencoding' : 'app.setdefaultencoding', } def setbuiltinmodule(self, w_module, name): Modified: pypy/dist/pypy/module/sys/app.py ============================================================================== --- pypy/dist/pypy/module/sys/app.py (original) +++ pypy/dist/pypy/module/sys/app.py Sun Nov 11 13:38:13 2007 @@ -46,17 +46,3 @@ """Not implemented.""" return None -defaultencoding = 'ascii' - -def getdefaultencoding(): - """Return the current default string encoding used by the Unicode -implementation.""" - return defaultencoding - -def setdefaultencoding(encoding): - """Set the current default string encoding used by the Unicode -implementation.""" - global defaultencoding - import codecs - codecs.lookup(encoding) - defaultencoding = encoding Added: pypy/dist/pypy/module/sys/interp_encoding.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/sys/interp_encoding.py Sun Nov 11 13:38:13 2007 @@ -0,0 +1,14 @@ +def getdefaultencoding(space): + """Return the current default string encoding used by the Unicode +implementation.""" + return space.wrap(space.sys.defaultencoding) + +def setdefaultencoding(space, w_encoding): + """Set the current default string encoding used by the Unicode +implementation.""" + encoding = space.str_w(w_encoding) + mod = space.getbuiltinmodule("_codecs") + w_lookup = space.getattr(mod, space.wrap("lookup")) + # check whether the encoding is there + space.call_function(w_lookup, w_encoding) + space.sys.defaultencoding = encoding Modified: pypy/dist/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/dist/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/dist/pypy/module/sys/test/test_sysmodule.py Sun Nov 11 13:38:13 2007 @@ -246,6 +246,16 @@ raises(TypeError, sys.getdefaultencoding, 42) # can't check more than the type, as the user might have changed it assert isinstance(sys.getdefaultencoding(), str) + + def test_getdefaultencoding(self): + encoding = sys.getdefaultencoding() + sys.setdefaultencoding("ascii") + try: + assert sys.getdefaultencoding() == 'ascii' + raises(UnicodeDecodeError, unicode, '\x80') + finally: + sys.setdefaultencoding(encoding) + # testing sys.settrace() is done in test_trace.py # testing sys.setprofile() is done in test_profile.py Modified: pypy/dist/pypy/module/unicodedata/interp_ucd.py ============================================================================== --- pypy/dist/pypy/module/unicodedata/interp_ucd.py (original) +++ pypy/dist/pypy/module/unicodedata/interp_ucd.py Sun Nov 11 13:38:13 2007 @@ -215,10 +215,10 @@ result[0] = ch if not composed: # If decomposed normalization we are done - return space.newunicode([unichr(i) for i in result[:j]]) + return space.wrap(u''.join([unichr(i) for i in result[:j]])) if j <= 1: - return space.newunicode([unichr(i) for i in result[:j]]) + return space.wrap(u''.join([unichr(i) for i in result[:j]])) current = result[0] starter_pos = 0 @@ -268,7 +268,7 @@ result[starter_pos] = current - return space.newunicode([unichr(i) for i in result[:next_insert]]) + return space.wrap(u''.join([unichr(i) for i in result[:next_insert]])) normalize.unwrap_spec = ['self', ObjSpace, str, W_Root] Modified: pypy/dist/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/objspace.py Sun Nov 11 13:38:13 2007 @@ -71,6 +71,9 @@ return PyInt_FromLong(x) if isinstance(x, str): return PyString_FromStringAndSize(x, len(x)) + if isinstance(x, str): + # XXX fix me + raise NotImplementedError if isinstance(x, float): return PyFloat_FromDouble(x) if isinstance(x, r_uint): @@ -206,7 +209,7 @@ buf[i] = p[i] return buf.raw - def unichars_w(self, w_obj): + def unicode_w(self, w_obj): not_implemented_sorry def call_function(self, w_callable, *args_w): @@ -227,20 +230,6 @@ PyString_InternInPlace(byref(w_s)) return w_s - def newstring(self, bytes_w): - length = len(bytes_w) - buf = ctypes.create_string_buffer(length) - for i in range(length): - buf[i] = chr(self.int_w(bytes_w[i])) - return PyString_FromStringAndSize(buf, length) - - def newunicode(self, codes): - # XXX inefficient - lst = [PyUnicode_FromOrdinal(ord(code)) for code in codes] - w_lst = self.newlist(lst) - w_emptyunicode = PyUnicode_FromUnicode(None, 0) - return self.call_method(w_emptyunicode, 'join', w_lst) - def newint(self, intval): return PyInt_FromLong(intval) Modified: pypy/dist/pypy/objspace/cpy/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_objspace.py Sun Nov 11 13:38:13 2007 @@ -49,28 +49,30 @@ w2 = space.wrap(11) raises_w(space, space.w_TypeError, space.sub, w1, w2) -def test_newstring(): +def test_wrapstring(): space = CPyObjSpace() - w = space.newstring([space.wrap(65), space.wrap(66)]) + w = space.wrap('AB') assert space.str_w(w) == 'AB' -def test_newunicode(): +def test_wrapunicode(): + py.test.skip("fix me") space = CPyObjSpace() - w = space.newunicode([unichr(65), unichr(66)]) + w = space.wrap(unichr(65) + unichr(66)) assert space.is_w(space.type(w), space.w_unicode) for i in range(2): code = space.int_w(space.ord(space.getitem(w, space.wrap(i)))) assert code == 65+i def test_ord(): + py.test.skip("fix me") space = CPyObjSpace() w = space.wrap('A') assert space.int_w(space.ord(w)) == 65 w = space.wrap('\x00') assert space.int_w(space.ord(w)) == 0 - w = space.newunicode([unichr(65)]) + w = space.wrap(unichr(65)) assert space.int_w(space.ord(w)) == 65 - w = space.newunicode([unichr(0)]) + w = space.wrap(unichr(0)) assert space.int_w(space.ord(w)) == 0 def test_id(): Modified: pypy/dist/pypy/objspace/dump.py ============================================================================== --- pypy/dist/pypy/objspace/dump.py (original) +++ pypy/dist/pypy/objspace/dump.py Sun Nov 11 13:38:13 2007 @@ -145,7 +145,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -153,8 +153,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, @@ -165,8 +163,6 @@ 'wrap': True, 'newtuple': True, 'newlist': True, - 'newstring': True, - 'newunicode': True, 'newdict': True, 'newslice': True, 'call_args': True, Modified: pypy/dist/pypy/objspace/fake/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/fake/objspace.py (original) +++ pypy/dist/pypy/objspace/fake/objspace.py Sun Nov 11 13:38:13 2007 @@ -23,7 +23,7 @@ uint_dummy = make_dummy(r_uint(42), r_uint(43)) str_dummy = make_dummy('foo', 'bar') bool_dummy = make_dummy(True, False) -unichars_dummy = make_dummy([u'a', u'b'], [u'c', u'd']) +unicode_dummy = make_dummy(u'abc', u'cde') bigint_dummy = make_dummy(rbigint([0]), rbigint([1])) class FakeObjSpace(ObjSpace): @@ -75,7 +75,7 @@ int_w = int_dummy uint_w = uint_dummy float_w = float_dummy - unichars_w = unichars_dummy + unicode_w = unicode_dummy bigint_w = bigint_dummy iter = make_dummy() type = make_dummy() @@ -88,8 +88,6 @@ str_w = str_dummy call_args = make_dummy() new_interned_str = make_dummy() - newstring = make_dummy() - newunicode = make_dummy() newint = make_dummy() newlong = make_dummy() newfloat = make_dummy() Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Sun Nov 11 13:38:13 2007 @@ -49,7 +49,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -57,8 +57,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, Modified: pypy/dist/pypy/objspace/std/default.py ============================================================================== --- pypy/dist/pypy/objspace/std/default.py (original) +++ pypy/dist/pypy/objspace/std/default.py Sun Nov 11 13:38:13 2007 @@ -35,9 +35,9 @@ raise OperationError(space.w_TypeError, typed_unwrap_error_msg(space, "integer", w_obj)) -def unichars_w__ANY(space,w_obj): +def unicode_w__ANY(space,w_obj): raise OperationError(space.w_TypeError, - typed_unwrap_error_msg(space, "string", w_obj)) + typed_unwrap_error_msg(space, "unicode", w_obj)) def bigint_w__ANY(space,w_obj): raise OperationError(space.w_TypeError, Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Sun Nov 11 13:38:13 2007 @@ -132,11 +132,16 @@ # to build two subclasses of the BaseStringFormatter class, # each one getting its own subtle differences and RPython types. + if do_unicode: + const = unicode + else: + const = str + class StringFormatter(BaseStringFormatter): def __init__(self, space, fmt, values_w, w_valuedict): BaseStringFormatter.__init__(self, space, values_w, w_valuedict) - self.fmt = fmt # either a string or a list of unichars + self.fmt = fmt # either a string or a unicode def peekchr(self): # return the 'current' character @@ -176,10 +181,7 @@ if self.w_valuedict is None: raise OperationError(space.w_TypeError, space.wrap("format requires a mapping")) - if do_unicode: - w_key = space.newunicode(key) - else: - w_key = space.wrap(key) + w_key = space.wrap(key) return space.getitem(self.w_valuedict, w_key) def parse_fmt(self): @@ -265,9 +267,9 @@ break i += 1 else: - result += fmt[i0:] + result.append(const(fmt[i0:])) break # end of 'fmt' string - result += fmt[i0:i] + result.append(const(fmt[i0:i])) self.fmtpos = i + 1 # interpret the next formatter @@ -301,7 +303,7 @@ if do_unicode: w_defaultencoding = space.call_function( space.sys.get('getdefaultencoding')) - w_s = space.call_method(space.newunicode([c]), + w_s = space.call_method(space.wrap(c), "encode", w_defaultencoding, space.wrap('replace')) @@ -320,10 +322,12 @@ result = self.result padding = self.width - length if not self.f_ljust: - result += ' ' * padding # add any padding at the left of 'r' + result.append(const(' ' * padding)) + # add any padding at the left of 'r' padding = 0 - result += r[:length] # add 'r' itself - result += ' ' * padding # add any remaining padding at the right + result.append(const(r[:length])) # add 'r' itself + result.append(const(' ' * padding)) + # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' def std_wp_number(self, r, prefix=''): @@ -350,15 +354,15 @@ padnumber = '>' if padnumber == '>': - result += ' ' * padding # pad with spaces on the left + result.append(const(' ' * padding)) # pad with spaces on the left if sign: - result.append(r[0]) # the sign - result += prefix # the prefix + result.append(const(r[0])) # the sign + result.append(const(prefix)) # the prefix if padnumber == '0': - result += '0' * padding # pad with zeroes - result += r[int(sign):] # the rest of the number + result.append(const('0' * padding)) # pad with zeroes + result.append(const(r[int(sign):])) # the rest of the number if padnumber == '<': # spaces on the right - result += ' ' * padding + result.append(const(' ' * padding)) def fmt_s(self, w_value): space = self.space @@ -371,7 +375,7 @@ else: if not got_unicode: w_value = space.call_function(space.w_unicode, w_value) - s = space.unichars_w(w_value) + s = space.unicode_w(w_value) self.std_wp(s) def fmt_r(self, w_value): @@ -389,11 +393,11 @@ elif space.is_true(space.isinstance(w_value, space.w_unicode)): if not do_unicode: raise NeedUnicodeFormattingError - lst = space.unichars_w(w_value) - if len(lst) != 1: + ustr = space.unicode_w(w_value) + if len(ustr) != 1: raise OperationError(space.w_TypeError, space.wrap("%c requires int or unichar")) - self.std_wp(lst) + self.std_wp(ustr) else: n = space.int_w(w_value) if do_unicode: @@ -402,7 +406,7 @@ except ValueError: raise OperationError(space.w_OverflowError, space.wrap("unicode character code out of range")) - self.std_wp([c]) + self.std_wp(c) else: try: s = chr(n) @@ -438,14 +442,14 @@ result = formatter.format() except NeedUnicodeFormattingError: # fall through to the unicode case - fmt = [c for c in fmt] # string => list of unichars + fmt = unicode(fmt) else: return space.wrap(''.join(result)) else: - fmt = space.unichars_w(w_fmt) + fmt = space.unicode_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() - return space.newunicode(result) + return space.wrap(u''.join(result)) def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Sun Nov 11 13:38:13 2007 @@ -400,7 +400,7 @@ from pypy.objspace.std.stringtype import wrapstr return wrapstr(self, x) if isinstance(x, unicode): - return W_UnicodeObject([unichr(ord(u)) for u in x]) # xxx + return W_UnicodeObject(x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, Wrappable): @@ -525,17 +525,6 @@ def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) - def newstring(self, chars_w): - try: - chars = [chr(self.int_w(w_c)) for w_c in chars_w] - except ValueError: # chr(out-of-range) - raise OperationError(self.w_ValueError, - self.wrap("character code not in range(256)")) - return self.wrap(''.join(chars)) - - def newunicode(self, chars): - return W_UnicodeObject(chars) - def newseqiter(self, w_obj): return W_SeqIterObject(w_obj) @@ -653,7 +642,7 @@ str_w = StdObjSpaceMultiMethod('str_w', 1, []) # returns an unwrapped string float_w = StdObjSpaceMultiMethod('float_w', 1, []) # returns an unwrapped float uint_w = StdObjSpaceMultiMethod('uint_w', 1, []) # returns an unwrapped unsigned int (r_uint) - unichars_w = StdObjSpaceMultiMethod('unichars_w', 1, []) # returns an unwrapped list of unicode characters + unicode_w = StdObjSpaceMultiMethod('unicode_w', 1, []) # returns an unwrapped list of unicode characters bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint # NOTE: when adding more sometype_w() methods, you need to write a # stub in default.py to raise a space.w_TypeError Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Sun Nov 11 13:38:13 2007 @@ -36,6 +36,21 @@ W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)] del i +def _decode_ascii(space, s): + try: + return s.decode("ascii") + except UnicodeDecodeError: + for i in range(len(s)): + if ord(s[i]) > 127: + raise OperationError( + space.w_UnicodeDecodeError, + space.wrap(("'ascii' codec can't decode byte %s in position %s:" + " ordinal not in range(128)") % (hex(ord(s[i])), i))) + +def unicode_w__String(space, w_self): + # XXX should this use the default encoding? + return _decode_ascii(space, w_self._value) + def _is_generic(space, w_self, fun): v = w_self._value Modified: pypy/dist/pypy/objspace/std/test/test_index.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_index.py (original) +++ pypy/dist/pypy/objspace/std/test/test_index.py Sun Nov 11 13:38:13 2007 @@ -223,15 +223,50 @@ SeqTestCase.setup_method(self, method) self.w_seq = self.space.newtuple([self.space.wrap(x) for x in (0,10,20,30,40,50)]) -class AppTest_StringTestCase(SeqTestCase): +class StringTestCase(object): + def test_startswith(self): + self.o.ind = 1 + assert self.const('abc').startswith(self.const('b'), self.o) + self.o.ind = 2 + assert not self.const('abc').startswith(self.const('abc'), 0, self.o) + + def test_endswith(self): + self.o.ind = 1 + assert self.const('abc').endswith(self.const('a'), 0, self.o) + self.o.ind = 2 + assert not self.const('abc').endswith(self.const('abc'), 0, self.o) + + def test_index(self): + self.o.ind = 3 + assert self.const('abcabc').index(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').index(self.const('abc'), self.o) == 3 + assert self.const('abcabc').rindex(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').rindex(self.const('abc'), self.o) == 3 + + def test_find(self): + self.o.ind = 3 + assert self.const('abcabc').find(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').find(self.const('abc'), self.o) == 3 + assert self.const('abcabc').rfind(self.const('abc'), 0, self.o) == 0 + assert self.const('abcabc').rfind(self.const('abc'), self.o) == 3 + + def test_count(self): + self.o.ind = 3 + assert self.const('abcabc').count(self.const('abc'), 0, self.o) == 1 + assert self.const('abcabc').count(self.const('abc'), self.o) == 1 + + +class AppTest_StringTestCase(SeqTestCase, StringTestCase): def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap("this is a test") + self.w_const = self.space.w_str -class AppTest_UnicodeTestCase(SeqTestCase): +class AppTest_UnicodeTestCase(SeqTestCase, StringTestCase): def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap(u"this is a test") + self.w_const = self.space.w_unicode class AppTest_XRangeTestCase: Modified: pypy/dist/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_unicodeobject.py Sun Nov 11 13:38:13 2007 @@ -21,9 +21,6 @@ check(u'a' + 'b', u'ab') check('a' + u'b', u'ab') - def test_hash(self): - assert hash(u'') == 0 - def test_join(self): def check(a, b): assert a == b @@ -278,3 +275,105 @@ else: raise Exception("DID NOT RAISE") + def test_startswith(self): + assert u'ab'.startswith(u'ab') is True + assert u'ab'.startswith(u'a') is True + assert u'ab'.startswith(u'') is True + assert u'x'.startswith(u'a') is False + assert u'x'.startswith(u'x') is True + assert u''.startswith(u'') is True + assert u''.startswith(u'a') is False + assert u'x'.startswith(u'xx') is False + assert u'y'.startswith(u'xx') is False + + def test_startswith_more(self): + assert u'ab'.startswith(u'a', 0) is True + assert u'ab'.startswith(u'a', 1) is False + assert u'ab'.startswith(u'b', 1) is True + assert u'abc'.startswith(u'bc', 1, 2) is False + assert u'abc'.startswith(u'c', -1, 4) is True + + def test_startswith_tuples(self): + assert u'hello'.startswith((u'he', u'ha')) + assert not u'hello'.startswith((u'lo', u'llo')) + assert u'hello'.startswith((u'hellox', u'hello')) + assert not u'hello'.startswith(()) + assert u'helloworld'.startswith((u'hellowo', u'rld', u'lowo'), 3) + assert not u'helloworld'.startswith((u'hellowo', u'ello', u'rld'), 3) + assert u'hello'.startswith((u'lo', u'he'), 0, -1) + assert not u'hello'.startswith((u'he', u'hel'), 0, 1) + assert u'hello'.startswith((u'he', u'hel'), 0, 2) + raises(TypeError, u'hello'.startswith, (42,)) + + def test_endswith(self): + assert u'ab'.endswith(u'ab') is True + assert u'ab'.endswith(u'b') is True + assert u'ab'.endswith(u'') is True + assert u'x'.endswith(u'a') is False + assert u'x'.endswith(u'x') is True + assert u''.endswith(u'') is True + assert u''.endswith(u'a') is False + assert u'x'.endswith(u'xx') is False + assert u'y'.endswith(u'xx') is False + + def test_endswith_more(self): + assert u'abc'.endswith(u'ab', 0, 2) is True + assert u'abc'.endswith(u'bc', 1) is True + assert u'abc'.endswith(u'bc', 2) is False + assert u'abc'.endswith(u'b', -3, -1) is True + + def test_endswith_tuple(self): + assert not u'hello'.endswith((u'he', u'ha')) + assert u'hello'.endswith((u'lo', u'llo')) + assert u'hello'.endswith((u'hellox', u'hello')) + assert not u'hello'.endswith(()) + assert u'helloworld'.endswith((u'hellowo', u'rld', u'lowo'), 3) + assert not u'helloworld'.endswith((u'hellowo', u'ello', u'rld'), 3, -1) + assert u'hello'.endswith((u'hell', u'ell'), 0, -1) + assert not u'hello'.endswith((u'he', u'hel'), 0, 1) + assert u'hello'.endswith((u'he', u'hell'), 0, 4) + raises(TypeError, u'hello'.endswith, (42,)) + + def test_expandtabs(self): + assert u'abc\rab\tdef\ng\thi'.expandtabs() == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(8) == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(4) == u'abc\rab def\ng hi' + assert u'abc\r\nab\tdef\ng\thi'.expandtabs(4) == u'abc\r\nab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs() == u'abc\rab def\ng hi' + assert u'abc\rab\tdef\ng\thi'.expandtabs(8) == u'abc\rab def\ng hi' + assert u'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == u'abc\r\nab\r\ndef\ng\r\nhi' + + s = u'xy\t' + assert s.expandtabs() =='xy ' + + s = u'\txy\t' + assert s.expandtabs() ==' xy ' + assert s.expandtabs(1) ==' xy ' + assert s.expandtabs(2) ==' xy ' + assert s.expandtabs(3) ==' xy ' + + assert u'xy'.expandtabs() =='xy' + assert u''.expandtabs() =='' + + def test_translate(self): + assert u'bbbc' == u'abababc'.translate({ord('a'):None}) + assert u'iiic' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i')}) + assert u'iiix' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):u'x'}) + assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''}) + assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''}) + assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'}) + + raises(TypeError, u'hello'.translate) + raises(TypeError, u'abababc'.translate, {ord('a'):''}) + + def test_unicode_form_encoded_object(self): + assert unicode('x', 'utf-8') == u'x' + assert unicode('x', 'utf-8', 'strict') == u'x' + + def test_unicode_startswith_tuple(self): + assert u'xxx'.startswith(('x', 'y', 'z'), 0) + assert u'xxx'.endswith(('x', 'y', 'z'), 0) + + def test_missing_cases(self): + # some random cases, which are discovered to not be tested during annotation + assert u'xxx'[1:1] == u'' Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Sun Nov 11 13:38:13 2007 @@ -4,26 +4,29 @@ from pypy.objspace.std.ropeobject import W_RopeObject from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.sliceobject import W_SliceObject +from pypy.objspace.std import slicetype from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.module.unicodedata import unicodedb_3_2_0 as unicodedb +from pypy.tool.sourcetools import func_with_new_name from pypy.objspace.std.formatting import mod_format class W_UnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef - def __init__(w_self, unicodechars): - w_self._value = unicodechars - w_self.w_hash = None + def __init__(w_self, unistr): + assert isinstance(unistr, unicode) + w_self._value = unistr + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%r)" % (w_self.__class__.__name__, w_self._value) def unwrap(w_self, space): - # For faked functions taking unicodearguments. - # Remove when we no longer need faking. - return u''.join(w_self._value) + # for testing + return w_self._value +W_UnicodeObject.EMPTY = W_UnicodeObject(u'') registerimplementation(W_UnicodeObject) @@ -63,7 +66,7 @@ def str_w__Unicode(space, w_uni): return space.str_w(space.str(w_uni)) -def unichars_w__Unicode(space, w_uni): +def unicode_w__Unicode(space, w_uni): return w_uni._value def str__Unicode(space, w_uni): @@ -75,11 +78,7 @@ def lt__Unicode_Unicode(space, w_left, w_right): left = w_left._value right = w_right._value - for i in range(min(len(left), len(right))): - if left[i] != right[i]: - return space.newbool(ord(left[i]) < ord(right[i])) - # NB. 'unichar < unichar' is not RPython at the moment - return space.newbool(len(left) < len(right)) + return space.newbool(left < right) def ord__Unicode(space, w_uni): if len(w_uni._value) != 1: @@ -107,108 +106,59 @@ contains__Rope_Unicode = contains__String_Unicode -def _find(self, sub, start, end): - if len(sub) == 0: - return start - if start >= end: - return -1 - for i in range(start, end - len(sub) + 1): - for j in range(len(sub)): - if self[i + j] != sub[j]: - break - else: - return i - return -1 - -def _rfind(self, sub, start, end): - if len(sub) == 0: - return end - if end - start < len(sub): - return -1 - for i in range(end - len(sub), start - 1, -1): - for j in range(len(sub)): - if self[i + j] != sub[j]: - break - else: - return i - return -1 - def contains__Unicode_Unicode(space, w_container, w_item): item = w_item._value container = w_container._value - return space.newbool(_find(container, item, 0, len(container)) >= 0) + return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - list = space.unpackiterable(w_list) + l = space.unpackiterable(w_list) delim = w_self._value totlen = 0 - if len(list) == 0: - return W_UnicodeObject([]) - if (len(list) == 1 and - space.is_w(space.type(list[0]), space.w_unicode)): - return list[0] + if len(l) == 0: + return W_UnicodeObject.EMPTY + if (len(l) == 1 and + space.is_w(space.type(l[0]), space.w_unicode)): + return l[0] - values_list = [None] * len(list) - values_list[0] = [u'\0'] - for i in range(len(list)): - item = list[i] - if space.is_true(space.isinstance(item, space.w_unicode)): - pass + values_list = [] + for i in range(len(l)): + item = l[i] + if isinstance(item, W_UnicodeObject): + # shortcut for performane + item = item._value elif space.is_true(space.isinstance(item, space.w_str)): - item = space.call_function(space.w_unicode, item) + item = space.unicode_w(item) else: w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - assert isinstance(item, W_UnicodeObject) - item = item._value - totlen += len(item) - values_list[i] = item - totlen += len(delim) * (len(values_list) - 1) - if len(values_list) == 1: - return W_UnicodeObject(values_list[0]) - # Allocate result - result = [u'\0'] * totlen - first = values_list[0] - for i in range(len(first)): - result[i] = first[i] - offset = len(first) - for i in range(1, len(values_list)): - item = values_list[i] - # Add delimiter - for j in range(len(delim)): - result[offset + j] = delim[j] - offset += len(delim) - # Add item from values_list - for j in range(len(item)): - result[offset + j] = item[j] - offset += len(item) - return W_UnicodeObject(result) - + values_list.append(item) + return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): - if w_uni.w_hash is None: - # hrmpf - chars = w_uni._value - if len(chars) == 0: + s = w_uni._value + if space.config.objspace.std.withrope: + # be compatible with the special ropes hash + # XXX no caching + if len(s) == 0: return space.wrap(0) - if space.config.objspace.std.withrope: - x = 0 - for c in chars: - x = intmask((1000003 * x) + ord(c)) - x <<= 1 - x ^= len(chars) - x ^= ord(chars[0]) - h = intmask(x) - else: - x = ord(chars[0]) << 7 - for c in chars: - x = intmask((1000003 * x) ^ ord(c)) - h = intmask(x ^ len(chars)) - if h == -1: - h = -2 - w_uni.w_hash = space.wrap(h) - return w_uni.w_hash + x = 0 + for c in s: + x = intmask((1000003 * x) + ord(c)) + x <<= 1 + x ^= len(s) + x ^= ord(s[0]) + h = intmask(x) + return space.wrap(h) + if we_are_translated(): + x = hash(s) # to use the hash cache in rpython strings + else: + from pypy.rlib.rarithmetic import _hash_string + x = _hash_string(s) # to make sure we get the same hash as rpython + # (otherwise translation will freeze W_DictObjects where we can't find + # the keys any more!) + return space.wrap(x) def len__Unicode(space, w_uni): return space.wrap(len(w_uni._value)) @@ -223,19 +173,19 @@ exc = space.call_function(space.w_IndexError, space.wrap("unicode index out of range")) raise OperationError(space.w_IndexError, exc) - return W_UnicodeObject([uni[ival]]) + return W_UnicodeObject(uni[ival]) def getitem__Unicode_Slice(space, w_uni, w_slice): uni = w_uni._value length = len(uni) start, stop, step, sl = w_slice.indices4(space, length) if sl == 0: - r = [] + r = u"" elif step == 1: assert start >= 0 and stop >= 0 r = uni[start:stop] else: - r = [uni[start + i*step] for i in range(sl)] + r = u"".join([uni[start + i*step] for i in range(sl)]) return W_UnicodeObject(r) def mul__Unicode_ANY(space, w_uni, w_times): @@ -245,18 +195,13 @@ if e.match(space, space.w_TypeError): raise FailedToImplement raise - chars = w_uni._value - charlen = len(chars) - if times <= 0 or charlen == 0: - return W_UnicodeObject([]) - if times == 1: - return space.call_function(space.w_unicode, w_uni) - if charlen == 1: - return W_UnicodeObject([w_uni._value[0]] * times) - + uni = w_uni._value + length = len(uni) + if times <= 0 or length == 0: + return W_UnicodeObject.EMPTY try: - result_size = ovfcheck(charlen * times) - result = chars * times + result_size = ovfcheck(length * times) + result = u''.join([uni] * times) except (OverflowError, MemoryError): raise OperationError(space.w_OverflowError, space.wrap('repeated string is too long')) return W_UnicodeObject(result) @@ -267,53 +212,23 @@ def _isspace(uchar): return unicodedb.isspace(ord(uchar)) -def unicode_isspace__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isspace(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isalpha__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isalpha(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isalnum__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isalnum(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isdecimal__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isdecimal(ord(uchar)): +def make_generic(funcname): + def func(space, w_self): + v = w_self._value + if len(v) == 0: return space.w_False - return space.w_True + for idx in range(len(v)): + if not getattr(unicodedb, funcname)(ord(v[idx])): + return space.w_False + return space.w_True + return func_with_new_name(func, "unicode_%s__Unicode" % (funcname, )) -def unicode_isdigit__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isdigit(ord(uchar)): - return space.w_False - return space.w_True - -def unicode_isnumeric__Unicode(space, w_unicode): - if len(w_unicode._value) == 0: - return space.w_False - for uchar in w_unicode._value: - if not unicodedb.isnumeric(ord(uchar)): - return space.w_False - return space.w_True +unicode_isspace__Unicode = make_generic("isspace") +unicode_isalpha__Unicode = make_generic("isalpha") +unicode_isalnum__Unicode = make_generic("isalnum") +unicode_isdecimal__Unicode = make_generic("isdecimal") +unicode_isdigit__Unicode = make_generic("isdigit") +unicode_isnumeric__Unicode = make_generic("isnumeric") def unicode_islower__Unicode(space, w_unicode): cased = False @@ -423,12 +338,12 @@ def unicode_capitalize__Unicode(space, w_self): input = w_self._value if len(input) == 0: - return W_UnicodeObject([]) + return W_UnicodeObject.EMPTY result = [u'\0'] * len(input) result[0] = unichr(unicodedb.toupper(ord(input[0]))) for i in range(1, len(input)): result[i] = unichr(unicodedb.tolower(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_title__Unicode(space, w_self): input = w_self._value @@ -436,7 +351,7 @@ return w_self result = [u'\0'] * len(input) - previous_is_cased = 0 + previous_is_cased = False for i in range(len(input)): unichar = ord(input[i]) if previous_is_cased: @@ -444,21 +359,21 @@ else: result[i] = unichr(unicodedb.totitle(unichar)) previous_is_cased = unicodedb.iscased(unichar) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_lower__Unicode(space, w_self): input = w_self._value result = [u'\0'] * len(input) for i in range(len(input)): result[i] = unichr(unicodedb.tolower(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_upper__Unicode(space, w_self): input = w_self._value result = [u'\0'] * len(input) for i in range(len(input)): result[i] = unichr(unicodedb.toupper(ord(input[i]))) - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_swapcase__Unicode(space, w_self): input = w_self._value @@ -471,7 +386,7 @@ result[i] = unichr(unicodedb.tolower(unichar)) else: result[i] = input[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def _normalize_index(length, index): if index < 0: @@ -482,41 +397,73 @@ index = length return index -def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): +def _convert_idx_params(space, w_self, w_start, w_end): self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + start = slicetype.adapt_bound(space, len(self), w_start) + end = slicetype.adapt_bound(space, len(self), w_end) - substr = w_substr._value + assert start >= 0 + assert end >= 0 + + return (self, start, end) + +def _check_startswith_substring(str, substr, start, end): substr_len = len(substr) if end - start < substr_len: - return space.w_False # substring is too long + return False # substring is too long + + for i in range(substr_len): + if str[start + i] != substr[i]: + return False + return True + +def _check_endswith_substring(str, substr, start, end): + substr_len = len(substr) + + if end - start < substr_len: + return False # substring is too long start = end - substr_len for i in range(substr_len): - if self[start + i] != substr[i]: - return space.w_False - return space.w_True + if str[start + i] != substr[i]: + return False + return True + +def unicode_endswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + substr = w_substr._value + return space.wrap(_check_endswith_substring(self, substr, start, end)) def unicode_startswith__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + # XXX this stuff can be waaay better for ootypebased backends if + # we re-use more of our rpython machinery (ie implement startswith + # with additional parameters as rpython) substr = w_substr._value - substr_len = len(substr) - - if end - start < substr_len: - return space.w_False # substring is too long - - for i in range(substr_len): - if self[start + i] != substr[i]: - return space.w_False - return space.w_True + return space.wrap(_check_startswith_substring(self, substr, start, end)) + +def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for w_prefix in space.unpacktuple(w_prefixes): + prefix = space.unicode_w(w_prefix) + if _check_startswith_substring(unistr, prefix, start, end): + return space.w_True + return space.w_False + +def unicode_endswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for w_suffix in space.unpacktuple(w_suffixes): + suffix = space.unicode_w(w_suffix) + if _check_endswith_substring(unistr, suffix, start, end): + return space.w_True + return space.w_False def _to_unichar_w(space, w_char): try: - w_unichar = unicodetype.unicode_from_object(space, w_char) + unistr = space.unicode_w(w_char) except OperationError, e: if e.match(space, space.w_TypeError): msg = 'The fill character cannot be converted to Unicode' @@ -524,10 +471,9 @@ else: raise - if space.int_w(space.len(w_unichar)) != 1: + if len(unistr) != 1: raise OperationError(space.w_TypeError, space.wrap('The fill character must be exactly one character long')) - unichar = unichr(space.int_w(space.ord(w_unichar))) - return unichar + return unistr[0] def unicode_center__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -540,7 +486,7 @@ result = [fillchar] * width for i in range(len(self)): result[leftpad + i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_ljust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -552,7 +498,7 @@ result = [fillchar] * width for i in range(len(self)): result[i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_rjust__Unicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._value @@ -564,13 +510,13 @@ result = [fillchar] * width for i in range(len(self)): result[padding + i] = self[i] - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_zfill__Unicode_ANY(space, w_self, w_width): self = w_self._value width = space.int_w(w_width) if len(self) == 0: - return W_UnicodeObject([u'0'] * width) + return W_UnicodeObject(u'0' * width) padding = width - len(self) if padding <= 0: return space.call_function(space.w_unicode, w_self) @@ -581,7 +527,7 @@ if self[0] in (u'+', u'-'): result[0] = self[0] result[padding] = u'0' - return W_UnicodeObject(result) + return W_UnicodeObject(u''.join(result)) def unicode_splitlines__Unicode_ANY(space, w_self, w_keepends): self = w_self._value @@ -613,55 +559,37 @@ return space.newlist(lines) def unicode_find__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value - return space.wrap(_find(self, substr, start, end)) + return space.wrap(self.find(substr, start, end)) def unicode_rfind__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value - return space.wrap(_rfind(self, substr, start, end)) + return space.wrap(self.rfind(substr, start, end)) def unicode_index__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value - index = _find(self, substr, start, end) + index = self.find(substr, start, end) if index < 0: raise OperationError(space.w_ValueError, space.wrap('substring not found')) return space.wrap(index) def unicode_rindex__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value - index = _rfind(self, substr, start, end) + index = self.rfind(substr, start, end) if index < 0: raise OperationError(space.w_ValueError, space.wrap('substring not found')) return space.wrap(index) def unicode_count__Unicode_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - self = w_self._value - start = _normalize_index(len(self), space.int_w(w_start)) - end = _normalize_index(len(self), space.int_w(w_end)) + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) substr = w_substr._value - count = 0 - while start <= end: - index = _find(self, substr, start, end) - if index < 0: - break - start = index + 1 - count += 1 - return space.wrap(count) - + return space.wrap(self.count(substr, start, end)) def unicode_split__Unicode_None_ANY(space, w_self, w_none, w_maxsplit): self = w_self._value @@ -703,18 +631,8 @@ if delim_len == 0: raise OperationError(space.w_ValueError, space.wrap('empty separator')) - parts = [] - start = 0 - end = len(self) - while maxsplit != 0: - index = _find(self, delim, start, end) - if index < 0: - break - parts.append(W_UnicodeObject(self[start:index])) - start = index + delim_len - maxsplit -= 1 - parts.append(W_UnicodeObject(self[start:])) - return space.newlist(parts) + parts = _split_with(self, delim, maxsplit) + return space.newlist([W_UnicodeObject(part) for part in parts]) def unicode_rsplit__Unicode_None_ANY(space, w_self, w_none, w_maxsplit): @@ -764,7 +682,7 @@ start = 0 end = len(self) while maxsplit != 0: - index = _rfind(self, delim, 0, end) + index = self.rfind(delim, 0, end) if index < 0: break parts.append(W_UnicodeObject(self[index+delim_len:end])) @@ -774,76 +692,52 @@ parts.reverse() return space.newlist(parts) -def _split(space, self, maxsplit): +def _split_into_chars(self, maxsplit): if maxsplit == 0: - return [W_UnicodeObject(self)] + return [self] index = 0 end = len(self) - parts = [W_UnicodeObject([])] + parts = [u''] maxsplit -= 1 while maxsplit != 0: if index >= end: break - parts.append(W_UnicodeObject([self[index]])) + parts.append(self[index]) index += 1 maxsplit -= 1 - parts.append(W_UnicodeObject(self[index:])) + parts.append(self[index:]) + return parts + +def _split_with(self, with_, maxsplit=-1): + parts = [] + start = 0 + end = len(self) + length = len(with_) + while maxsplit != 0: + index = self.find(with_, start, end) + if index < 0: + break + parts.append(self[start:index]) + start = index + length + maxsplit -= 1 + parts.append(self[start:]) return parts def unicode_replace__Unicode_Unicode_Unicode_ANY(space, w_self, w_old, w_new, w_maxsplit): if len(w_old._value): - w_parts = space.call_method(w_self, 'split', w_old, w_maxsplit) + parts = _split_with(w_self._value, w_old._value, + space.int_w(w_maxsplit)) else: self = w_self._value maxsplit = space.int_w(w_maxsplit) - w_parts = space.newlist(_split(space, self, maxsplit)) - return space.call_method(w_new, 'join', w_parts) + parts = _split_into_chars(self, maxsplit) + return W_UnicodeObject(w_new._value.join(parts)) app = gateway.applevel(r''' import sys -def unicode_expandtabs__Unicode_ANY(self, tabsize): - parts = self.split(u'\t') - result = [ parts[0] ] - prevsize = 0 - for ch in parts[0]: - prevsize += 1 - if ch in (u"\n", u"\r"): - prevsize = 0 - for i in range(1, len(parts)): - pad = tabsize - prevsize % tabsize - result.append(u' ' * pad) - nextpart = parts[i] - result.append(nextpart) - prevsize = 0 - for ch in nextpart: - prevsize += 1 - if ch in (u"\n", u"\r"): - prevsize = 0 - return u''.join(result) - -def unicode_translate__Unicode_ANY(self, table): - result = [] - for unichar in self: - try: - newval = table[ord(unichar)] - except KeyError: - result.append(unichar) - else: - if newval is None: - continue - elif isinstance(newval, int): - if newval < 0 or newval > sys.maxunicode: - raise TypeError("character mapping must be in range(0x%x)"%(sys.maxunicode + 1,)) - result.append(unichr(newval)) - elif isinstance(newval, unicode): - result.append(newval) - else: - raise TypeError("character mapping must return integer, None or unicode") - return ''.join(result) - def unicode_encode__Unicode_ANY_ANY(unistr, encoding=None, errors=None): import codecs, sys if encoding is None: @@ -858,44 +752,96 @@ raise TypeError("encoder did not return a string object (type=%s)" % type(retval).__name__) return retval +''') + -# XXX: These should probably be written on interplevel -def unicode_partition__Unicode_Unicode(unistr, unisub): +unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') + +def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): + unistr = w_unistr._value + unisub = w_unisub._value + if not unisub: + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) pos = unistr.find(unisub) if pos == -1: - return (unistr, u'', u'') + return space.newtuple([w_unistr, W_UnicodeObject.EMPTY, + W_UnicodeObject.EMPTY]) else: - return (unistr[:pos], unisub, unistr[pos+len(unisub):]) + assert pos > 0 + return space.newtuple([space.wrap(unistr[:pos]), w_unisub, + space.wrap(unistr[pos+len(unisub):])]) -def unicode_rpartition__Unicode_Unicode(unistr, unisub): +def unicode_rpartition__Unicode_Unicode(space, w_unistr, w_unisub): + unistr = w_unistr._value + unisub = w_unisub._value + if not unisub: + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) pos = unistr.rfind(unisub) if pos == -1: - return (u'', u'', unistr) + return space.newtuple([W_UnicodeObject.EMPTY, + W_UnicodeObject.EMPTY, w_unistr]) else: - return (unistr[:pos], unisub, unistr[pos+len(unisub):]) + assert pos > 0 + return space.newtuple([space.wrap(unistr[:pos]), w_unisub, + space.wrap(unistr[pos+len(unisub):])]) -def unicode_startswith__Unicode_Tuple_ANY_ANY(unistr, prefixes, start, end): - for prefix in prefixes: - if unistr.startswith(prefix): - return True - return False - -def unicode_endswith__Unicode_Tuple_ANY_ANY(unistr, suffixes, start, end): - for suffix in suffixes: - if unistr.endswith(suffix): - return True - return False -''') +def unicode_expandtabs__Unicode_ANY(space, w_self, w_tabsize): + self = w_self._value + tabsize = space.int_w(w_tabsize) + parts = _split_with(self, u'\t') + result = [parts[0]] + prevsize = 0 + for ch in parts[0]: + prevsize += 1 + if ch == u"\n" or ch == u"\r": + prevsize = 0 + for i in range(1, len(parts)): + pad = tabsize - prevsize % tabsize + result.append(u' ' * pad) + nextpart = parts[i] + result.append(nextpart) + prevsize = 0 + for ch in nextpart: + prevsize += 1 + if ch in (u"\n", u"\r"): + prevsize = 0 + return space.wrap(u''.join(result)) -unicode_expandtabs__Unicode_ANY = app.interphook('unicode_expandtabs__Unicode_ANY') -unicode_translate__Unicode_ANY = app.interphook('unicode_translate__Unicode_ANY') -unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') -unicode_partition__Unicode_Unicode = app.interphook('unicode_partition__Unicode_Unicode') -unicode_rpartition__Unicode_Unicode = app.interphook('unicode_rpartition__Unicode_Unicode') -unicode_startswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_startswith__Unicode_Tuple_ANY_ANY') -unicode_endswith__Unicode_Tuple_ANY_ANY = app.interphook('unicode_endswith__Unicode_Tuple_ANY_ANY') + +def unicode_translate__Unicode_ANY(space, w_self, w_table): + self = w_self._value + w_sys = space.getbuiltinmodule('sys') + maxunicode = space.int_w(space.getattr(w_sys, space.wrap("maxunicode"))) + result = [] + for unichar in self: + try: + w_newval = space.getitem(w_table, space.wrap(ord(unichar))) + except OperationError, e: + if e.match(space, space.w_KeyError): + result.append(unichar) + else: + raise + else: + if space.is_w(w_newval, space.w_None): + continue + elif space.is_true(space.isinstance(w_newval, space.w_int)): + newval = space.int_w(w_newval) + if newval < 0 or newval > maxunicode: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must be in range(0x%x)" % (maxunicode + 1,))) + result.append(unichr(newval)) + elif space.is_true(space.isinstance(w_newval, space.w_unicode)): + result.append(space.unicode_w(w_newval)) + else: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must return integer, None or unicode")) + return W_UnicodeObject(u''.join(result)) # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__Unicode(space, w_unicode): @@ -1016,7 +962,6 @@ i += 1 return space.wrap(''.join(result[:i])) -#repr__Unicode = app.interphook('repr__Unicode') # uncomment when repr code is moved to _codecs def mod__Unicode_ANY(space, w_format, w_values): return mod_format(space, w_format, w_values, do_unicode=True) @@ -1028,6 +973,10 @@ # str.strip(unicode) needs to convert self to unicode and call unicode.strip we # use the following magic to register strip_string_unicode as a String # multimethod. + +# XXX couldn't string and unicode _share_ the multimethods that make up their +# methods? + class str_methods: import stringtype W_UnicodeObject = W_UnicodeObject Modified: pypy/dist/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodetype.py (original) +++ pypy/dist/pypy/objspace/std/unicodetype.py Sun Nov 11 13:38:13 2007 @@ -142,73 +142,87 @@ # ____________________________________________________________ -app = gateway.applevel(''' -def unicode_from_encoded_object(obj, encoding, errors): - import codecs, sys +def getdefaultencoding(space): + return space.sys.defaultencoding + +def unicode_from_encoded_object(space, w_obj, encoding, errors): + w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: - encoding = sys.getdefaultencoding() - decoder = codecs.getdecoder(encoding) + encoding = getdefaultencoding(space) + w_decode = space.getattr(w_codecs, space.wrap("decode")) if errors is None: - retval, length = decoder(obj) + w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding)) else: - retval, length = decoder(obj, errors) - if not isinstance(retval, unicode): - raise TypeError("decoder did not return an unicode object (type=%s)" % - type(retval).__name__) - return retval - -def unicode_from_object(obj): - if isinstance(obj, str): - res = obj + w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding), + space.wrap(errors)) + if not space.is_true(space.isinstance(w_retval, space.w_unicode)): + raise OperationError( + space.w_TypeError, + space.wrap( + "decoder did not return an unicode object (type=%s)" % + space.type(w_retval).getname(space, '?'))) + return w_retval + + +def unicode_from_object(space, w_obj): + if space.is_true(space.isinstance(w_obj, space.w_str)): + w_res = w_obj else: try: - unicode_method = obj.__unicode__ - except AttributeError: - res = str(obj) + # XXX should we have a space.unicode so we can go through + # descroperation? + w_unicode_method = space.getattr(w_obj, space.wrap("__unicode__")) + except OperationError, e: + if e.match(space, space.w_AttributeError): + w_res = space.str(w_obj) + else: + raise else: - res = unicode_method() - if isinstance(res, unicode): - return res - return unicode_from_encoded_object(res, None, "strict") - -''') -unicode_from_object = app.interphook('unicode_from_object') -unicode_from_encoded_object = app.interphook('unicode_from_encoded_object') + w_res = space.call_function(w_unicode_method) + if space.is_true(space.isinstance(w_res, space.w_unicode)): + return w_res + return unicode_from_encoded_object(space, w_res, None, "strict") def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack from pypy.objspace.std.unicodeobject import W_UnicodeObject - w_encoding = space.call_function(space.sys.get('getdefaultencoding')) - if not space.eq_w(w_encoding, space.wrap('ascii')): + encoding = getdefaultencoding(space) + if encoding != 'ascii': return unicode_from_object(space, w_str) s = space.str_w(w_str) - codelist = [] - for i in range(len(s)): - code = ord(s[i]) - if code >= 128: - # raising UnicodeDecodeError is messy, so "please crash for me" - return unicode_from_object(space, w_str) - codelist.append(unichr(code)) - return W_UnicodeObject(codelist) + try: + return W_UnicodeObject(s.decode("ascii")) + except UnicodeDecodeError: + # raising UnicodeDecodeError is messy, "please crash for me" + return unicode_from_object(space, w_str) + +def _get_encoding_and_errors(space, w_encoding, w_errors): + if space.is_w(w_encoding, space.w_None): + encoding = None + else: + encoding = space.str_w(w_encoding) + if space.is_w(w_errors, space.w_None): + errors = None + else: + errors = space.str_w(w_errors) + return encoding, errors -def descr__new__(space, w_unicodetype, w_string='', w_encoding=None, w_errors=None): - # NB. the default value of w_string is really a *wrapped* empty string: +def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): + # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work from pypy.objspace.std.unicodeobject import W_UnicodeObject - w_obj = w_string w_obj_type = space.type(w_obj) + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if space.is_w(w_obj_type, space.w_unicode): - if (not space.is_w(w_encoding, space.w_None) or - not space.is_w(w_errors, space.w_None)): + if encoding is not None or errors is not None: raise OperationError(space.w_TypeError, space.wrap('decoding Unicode is not supported')) if space.is_w(w_unicodetype, space.w_unicode): return w_obj w_value = w_obj - elif (space.is_w(w_encoding, space.w_None) and - space.is_w(w_errors, space.w_None)): + elif encoding is None and errors is None: if space.is_true(space.isinstance(w_obj, space.w_str)): w_value = unicode_from_string(space, w_obj) elif space.is_true(space.isinstance(w_obj, space.w_unicode)): @@ -216,7 +230,7 @@ else: w_value = unicode_from_object(space, w_obj) else: - w_value = unicode_from_encoded_object(space, w_obj, w_encoding, w_errors) + w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) # help the annotator! also the ._value depends on W_UnicodeObject layout assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) Modified: pypy/dist/pypy/objspace/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/thunk.py (original) +++ pypy/dist/pypy/objspace/thunk.py Sun Nov 11 13:38:13 2007 @@ -150,7 +150,7 @@ 'int_w': 1, 'float_w': 1, 'uint_w': 1, - 'unichars_w': 1, + 'unicode_w': 1, 'bigint_w': 1, 'interpclass_w': 1, 'unwrap': 1, @@ -158,8 +158,6 @@ 'is_w': 2, 'newtuple': 0, 'newlist': 0, - 'newstring': 0, - 'newunicode': 0, 'newdict': 0, 'newslice': 0, 'call_args': 1, Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sun Nov 11 13:38:13 2007 @@ -128,6 +128,11 @@ raise RSocketError("unknown address family") from_object = staticmethod(from_object) + def fill_from_object(self, space, w_address): + """ Purely abstract + """ + raise NotImplementedError + # ____________________________________________________________ def makeipaddr(name, result=None): @@ -189,6 +194,11 @@ host, serv = getnameinfo(self, NI_NUMERICHOST | NI_NUMERICSERV) return host + def lock_in_addr(self): + """ Purely abstract + """ + raise NotImplementedError + # ____________________________________________________________ class INETAddress(IPAddress): Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sun Nov 11 13:38:13 2007 @@ -1,6 +1,7 @@ import py, errno, sys from pypy.rlib import rsocket from pypy.rlib.rsocket import * +import socket as cpy_socket # cannot test error codes in Win32 because ll2ctypes doesn't save # the errors that WSAGetLastError() should return, making it likely @@ -80,8 +81,8 @@ assert getservbyname('http', 'tcp') == 80 def test_getservbyport(): - assert getservbyport(80) == 'http' - assert getservbyport(80, 'tcp') == 'http' + assert getservbyport(80) == cpy_socket.getservbyport(80) + assert getservbyport(80, 'tcp') == cpy_socket.getservbyport(80) def test_getprotobyname(): assert getprotobyname('tcp') == IPPROTO_TCP Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Sun Nov 11 13:38:13 2007 @@ -162,6 +162,17 @@ for link, lbl in cases.itervalues(): self.render_switch_case(link, lbl) + def call_oostring(self, ARGTYPE): + if isinstance(ARGTYPE, ootype.Instance): + argtype = self.cts.types.object + else: + argtype = self.cts.lltype_to_cts(ARGTYPE) + self.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) + + def call_oounicode(self, ARGTYPE): + argtype = self.cts.lltype_to_cts(ARGTYPE) + self.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + # Those parts of the generator interface that are function # specific Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Sun Nov 11 13:38:13 2007 @@ -95,28 +95,6 @@ generator.call_signature('object [pypylib]pypy.runtime.Utils::RuntimeNew(class [mscorlib]System.Type)') generator.cast_to(op.result.concretetype) -class _OOString(MicroInstruction): - def render(self, generator, op): - ARGTYPE = op.args[0].concretetype - if isinstance(ARGTYPE, ootype.Instance): - argtype = 'object' - else: - argtype = generator.cts.lltype_to_cts(ARGTYPE) - generator.load(op.args[0]) - generator.load(op.args[1]) - generator.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) - -class _OOUnicode(MicroInstruction): - def render(self, generator, op): - from pypy.objspace.flow.model import Constant - ARGTYPE = op.args[0].concretetype - argtype = generator.cts.lltype_to_cts(ARGTYPE) - v_base = op.args[1] - assert v_base.value == -1, "The second argument of oounicode must be -1" - - generator.load(op.args[0]) - generator.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) - class _NewCustomDict(MicroInstruction): def render(self, generator, op): DICT = op.args[0].value @@ -236,8 +214,6 @@ CallMethod = _CallMethod() IndirectCall = _IndirectCall() RuntimeNew = _RuntimeNew() -OOString = _OOString() -OOUnicode = _OOUnicode() NewCustomDict = _NewCustomDict() #CastWeakAdrToPtr = _CastWeakAdrToPtr() Box = _Box() Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Sun Nov 11 13:38:13 2007 @@ -1,9 +1,9 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ - IndirectCall, GetField, SetField, OOString, DownCast, NewCustomDict,\ + IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, OOUnicode + TypeOf, CastPrimitive from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ - New, RuntimeNew, CastTo, PushPrimitive + New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode from pypy.translator.cli.cts import WEAKREF from pypy.rpython.ootypesystem import ootype Modified: pypy/dist/pypy/translator/geninterplevel.py ============================================================================== --- pypy/dist/pypy/translator/geninterplevel.py (original) +++ pypy/dist/pypy/translator/geninterplevel.py Sun Nov 11 13:38:13 2007 @@ -177,7 +177,7 @@ # special constructors: self.has_listarg = {} - for name in "newtuple newlist newstring".split(): + for name in "newtuple newlist".split(): self.has_listarg[name] = name # catching all builtins in advance, to avoid problems Modified: pypy/dist/pypy/translator/jvm/builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/builtin.py (original) +++ pypy/dist/pypy/translator/jvm/builtin.py Sun Nov 11 13:38:13 2007 @@ -149,3 +149,18 @@ jvmgen.Method.v(jArrayList, "get", (jInt,), jObject), } + +# ootype.String[Builder] and ootype.Unicode[Builder] are mapped to the +# same JVM type, so we reuse the same builtin methods also for them +def add_unicode_methods(): + mapping = { + ootype.String.__class__: ootype.Unicode.__class__, + ootype.StringBuilder.__class__: ootype.UnicodeBuilder.__class__ + } + + for (TYPE, name), value in built_in_methods.items(): + if TYPE in mapping: + TYPE = mapping[TYPE] + built_in_methods[TYPE, name] = value +add_unicode_methods() +del add_unicode_methods Modified: pypy/dist/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/conftest.py (original) +++ pypy/dist/pypy/translator/jvm/conftest.py Sun Nov 11 13:38:13 2007 @@ -15,8 +15,8 @@ help="don't assemble jasmin files"), Option('--package', action='store', dest='package', default='pypy', help='Package to output generated classes into'), - Option('--trace', action='store_true', dest='trace', default=False, - help='Trace execution of generated code'), +## Option('--trace', action='store_true', dest='trace', default=False, +## help='Trace execution of generated code'), Option('--byte-arrays', action='store_true', dest='byte-arrays', default=False, help='Use byte arrays rather than native strings'), ) Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Sun Nov 11 13:38:13 2007 @@ -412,7 +412,9 @@ ootype.Bool:jvmgen.PYPYSERIALIZEBOOLEAN, ootype.Void:jvmgen.PYPYSERIALIZEVOID, ootype.Char:jvmgen.PYPYESCAPEDCHAR, + ootype.UniChar:jvmgen.PYPYESCAPEDUNICHAR, ootype.String:jvmgen.PYPYESCAPEDSTRING, + ootype.Unicode:jvmgen.PYPYESCAPEDUNICODE, } def toString_method_for_ootype(self, OOTYPE): @@ -466,7 +468,9 @@ # will return a JvmBuiltInType based on the value ootype_to_builtin = { ootype.String: jvmtype.jString, + ootype.Unicode: jvmtype.jString, ootype.StringBuilder: jvmtype.jStringBuilder, + ootype.UnicodeBuilder: jvmtype.jStringBuilder, ootype.List: jvmtype.jArrayList, ootype.Dict: jvmtype.jHashMap, ootype.DictItemsIterator:jvmtype.jPyPyDictItemsIterator, Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sun Nov 11 13:38:13 2007 @@ -389,7 +389,9 @@ PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,), jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) +PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) +PYPYESCAPEDUNICODE = Method.s(jPyPy, 'escaped_unicode', (jString,), jString) PYPYSERIALIZEOBJECT = Method.s(jPyPy, 'serializeObject', (jObject,), jString) PYPYRUNTIMENEW = Method.s(jPyPy, 'RuntimeNew', (jClass,), jObject) PYPYSTRING2BYTES = Method.s(jPyPy, 'string2bytes', (jString,), jByteArray) @@ -1018,6 +1020,13 @@ self.emit(mthd) if self.db.using_byte_array: self.emit(PYPYSTRING2BYTES) + + def call_oounicode(self, OOTYPE): + cts_type = self.db.lltype_to_cts(OOTYPE) + mthd = Method.s(jPyPy, 'oounicode', [cts_type], jString) + self.emit(mthd) + if self.db.using_byte_array: + self.emit(PYPYSTRING2BYTES) def new(self, TYPE): jtype = self.db.lltype_to_cts(TYPE) @@ -1087,11 +1096,13 @@ self._push_long_constant(value) elif TYPE is ootype.Float: self._push_double_constant(float(value)) - elif TYPE is ootype.String: - if value == ootype.null(ootype.String): + elif TYPE in (ootype.String, ootype.Unicode): + if value == ootype.null(TYPE): self.emit(ACONST_NULL) else: self.load_string(str(value._str)) + else: + assert False, 'Unknown constant type: %s' % TYPE def _push_long_constant(self, value): if value == 0: @@ -1336,7 +1347,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - #self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Sun Nov 11 13:38:13 2007 @@ -7,7 +7,8 @@ from pypy.translator.oosupport.metavm import \ PushArg, PushAllArgs, StoreResult, InstructionList, New, DoNothing, Call,\ - SetField, GetField, DownCast, RuntimeNew, OOString, CastTo, PushPrimitive + SetField, GetField, DownCast, RuntimeNew, OOString, OOUnicode, \ + CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ CastPrimitive @@ -56,6 +57,7 @@ 'ooidentityhash': [PushAllArgs, jvmgen.OBJHASHCODE, StoreResult], 'oohash': [PushAllArgs, jvmgen.OBJHASHCODE, StoreResult], 'oostring': [OOString, StoreResult], + 'oounicode': [OOUnicode, StoreResult], #'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': jvmgen.PYPYOOPARSEFLOAT, 'oonewcustomdict': [NewCustomDict, StoreResult], Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/jvm/prebuiltnodes.py Sun Nov 11 13:38:13 2007 @@ -14,6 +14,9 @@ def throwValueError(): raise ValueError +def throwUnicodeDecodeError(): + raise UnicodeDecodeError + # ___________________________________________________________________________ def create_interlink_node(db): Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Sun Nov 11 13:38:13 2007 @@ -14,5 +14,6 @@ public void throwIndexError(); public void throwOverflowError(); public void throwValueError(); + public void throwUnicodeDecodeError(); public void throwOSError(int errCode); } Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sun Nov 11 13:38:13 2007 @@ -319,6 +319,10 @@ return sb.toString(); } + public static String escaped_unichar(char c) { + return "u" + escaped_char(c); + } + public static String escaped_string(String b) { if (b == null) return "None"; @@ -332,6 +336,10 @@ return sb.toString(); } + public static String escaped_unicode(String b) { + return "u" + escaped_string(b); + } + // used in running unit tests // not really part of the dump_XXX set of objects, hence the lack // of an indent parameter @@ -806,6 +814,24 @@ } // ---------------------------------------------------------------------- + // OOUnicode support + + public static String oounicode(char ch) + { + return new Character(ch).toString(); + } + + public static String oounicode(String s) + { + for(int i=0; i 127) + throwUnicodeDecodeError(); + } + return s; + } + + // ---------------------------------------------------------------------- // Primitive built-in functions public static double ll_time_clock() { @@ -957,6 +983,10 @@ public static void throwValueError() { interlink.throwValueError(); } + + public static void throwUnicodeDecodeError() { + interlink.throwUnicodeDecodeError(); + } // ---------------------------------------------------------------------- // Self Test Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Sun Nov 11 13:38:13 2007 @@ -140,6 +140,9 @@ def ll_to_string(self, s): return s + def ll_to_unicode(self, s): + return s + def ll_to_list(self, l): return l Added: pypy/dist/pypy/translator/jvm/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_unicode.py Sun Nov 11 13:38:13 2007 @@ -0,0 +1,20 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_runicode import BaseTestRUnicode + +# ====> ../../../rpython/test/test_runicode.py + +class TestJvmUnicode(JvmTest, BaseTestRUnicode): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("JVM doesn't support unicode for command line arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_getitem_exc(self): + py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Sun Nov 11 13:38:13 2007 @@ -452,6 +452,15 @@ generator.load(op.args[1]) generator.call_oostring(ARGTYPE) +class _OOUnicode(MicroInstruction): + def render(self, generator, op): + v_base = op.args[1] + assert v_base.value == -1, "The second argument of oounicode must be -1" + + ARGTYPE = op.args[0].concretetype + generator.load(op.args[0]) + generator.call_oounicode(ARGTYPE) + class _CastTo(MicroInstruction): def render(self, generator, op): generator.load(op.args[0]) @@ -471,5 +480,6 @@ CallMethod = _CallMethod() RuntimeNew = _RuntimeNew() OOString = _OOString() +OOUnicode = _OOUnicode() CastTo = _CastTo() From antocuni at codespeak.net Sun Nov 11 13:45:37 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 11 Nov 2007 13:45:37 +0100 (CET) Subject: [pypy-svn] r48559 - pypy/extradoc/talk/pycon2008 Message-ID: <20071111124537.7DA238241@code0.codespeak.net> Author: antocuni Date: Sun Nov 11 13:45:37 2007 New Revision: 48559 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: some typos, one XXX Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Sun Nov 11 13:45:37 2007 @@ -6,17 +6,18 @@ which made it easy to implement those features. The talk would consist on: -* Transparent proxy cocept, which allows programmer to create +* Transparent proxy concept, which allows programmers to create transparent object of any type in python (including built-in - python types like frames), with various usage showcases icluding - our completely transparent remote access scheme. + python types like frames), with various usage showcases including + our completely transparent remote access scheme. (XXX: I know what + you mean, but "create transparent object" is a bit vague) * Sandboxing concept, which allows to run non-stripped version of - python interpreter in controlled environment with custom + python interpreter in a controlled environment with custom safety policy. -* Taint objspace, which allows programmer to make sure that sensitive - information does not cross the I/O barrier. +* Taint object space, which allows programmers to make sure that sensitive + informations do not cross the I/O barrier. * XXX anything else? From fijal at codespeak.net Sun Nov 11 13:46:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Nov 2007 13:46:22 +0100 (CET) Subject: [pypy-svn] r48560 - pypy/dist/pypy/doc/config Message-ID: <20071111124622.B61958241@code0.codespeak.net> Author: fijal Date: Sun Nov 11 13:46:22 2007 New Revision: 48560 Modified: pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt Log: Fix rest (?) Modified: pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt ============================================================================== --- pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt (original) +++ pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt Sun Nov 11 13:46:22 2007 @@ -1,4 +1,4 @@ turn allocations into coallocations when appropriate. Requires the use of a generational GC. See the paper `Finding your Cronies`_. -.. _`Finding your Cronies`:: http://www.cs.utexas.edu/~sammy/cronies-oopsla-2004.pdf +.. _`Finding your Cronies`: http://www.cs.utexas.edu/~sammy/cronies-oopsla-2004.pdf From fijal at codespeak.net Sun Nov 11 14:38:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Nov 2007 14:38:56 +0100 (CET) Subject: [pypy-svn] r48564 - in pypy/dist/pypy/module/zipimport: . test Message-ID: <20071111133856.A4F1D824B@code0.codespeak.net> Author: fijal Date: Sun Nov 11 14:38:56 2007 New Revision: 48564 Modified: pypy/dist/pypy/module/zipimport/__init__.py pypy/dist/pypy/module/zipimport/app_zipimport.py pypy/dist/pypy/module/zipimport/interp_zipimport.py pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: Expose _zip_directory_cache, test it and actually use it. Modified: pypy/dist/pypy/module/zipimport/__init__.py ============================================================================== --- pypy/dist/pypy/module/zipimport/__init__.py (original) +++ pypy/dist/pypy/module/zipimport/__init__.py Sun Nov 11 14:38:56 2007 @@ -10,6 +10,7 @@ interpleveldefs = {'zipimporter':'interp_zipimport.W_ZipImporter'} appleveldefs = { - 'ZipImportError' : 'app_zipimport.ZipImportError', + 'ZipImportError' : 'app_zipimport.ZipImportError', + '_zip_directory_cache': 'app_zipimport._zip_directory_cache', } Modified: pypy/dist/pypy/module/zipimport/app_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/app_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/app_zipimport.py Sun Nov 11 14:38:56 2007 @@ -1,3 +1,5 @@ class ZipImportError(Exception): pass + +_zip_directory_cache = {} Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Sun Nov 11 14:38:56 2007 @@ -16,8 +16,6 @@ # note that zipfiles always use slash, but for OSes with other # separators, we need to pretend that we had the os.sep. -zip_importer_cache = {} - ENUMERATE_EXTS = unrolling_iterable( [(True, True, os.path.sep + '__init__.pyc'), (True, True, os.path.sep + '__init__.pyo'), @@ -167,14 +165,19 @@ return space.wrap(self.name) def descr_new_zipimporter(space, w_type, name): + w_zip_cache = space.getattr(space.getbuiltinmodule('zipimport'), + space.wrap('_zip_directory_cache')) try: - result = zip_importer_cache[name] - if result is None: + w_result = space.getitem(w_zip_cache, space.wrap(name)) + if space.is_w(w_result, space.w_None): raise OperationError(space.w_ImportError, space.wrap( "Cannot import %s from zipfile, recursion detected or" "already tried and failed" % (name,))) - except KeyError: - zip_importer_cache[name] = None + return w_result + except OperationError, o: + if not o.match(space, space.w_KeyError): + raise + space.setitem(w_zip_cache, space.wrap(name), space.w_None) ok = False parts = name.split(os.path.sep) filename = "" # make annotator happy @@ -205,9 +208,9 @@ except OperationError, e: # we catch everything as this function raise OperationError(space.w_ImportError, space.wrap( "%s seems not to be a zipfile" % (filename,))) - result = space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) - zip_importer_cache[name] = result - return result + w_result = space.wrap(W_ZipImporter(space, name, w_dir, w_zipfile)) + space.setitem(w_zip_cache, space.wrap(name), w_result) + return w_result descr_new_zipimporter.unwrap_spec = [ObjSpace, W_Root, str] Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Sun Nov 11 14:38:56 2007 @@ -42,7 +42,7 @@ """).compile() space = gettestobjspace(usemodules=['zipimport', 'zlib', 'rctime']) cls.space = space - tmpdir = udir.ensure('zipimport', dir=1) + tmpdir = udir.ensure('zipimport_%s' % cls.__name__, dir=1) now = time.time() cls.w_now = space.wrap(now) test_pyc = cls.make_pyc(space, co, now) @@ -90,6 +90,15 @@ sys.path.pop(0) """) + def test_cache(self): + self.writefile(self, 'x.py', 'y') + from zipimport import _zip_directory_cache, zipimporter + new_importer = zipimporter(self.zipfile) + try: + assert zipimporter(self.zipfile) is new_importer + finally: + del _zip_directory_cache[self.zipfile] + def test_good_bad_arguments(self): from zipimport import zipimporter import os From cfbolz at codespeak.net Sun Nov 11 15:03:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Nov 2007 15:03:29 +0100 (CET) Subject: [pypy-svn] r48567 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071111140329.25FB08240@code0.codespeak.net> Author: cfbolz Date: Sun Nov 11 15:03:27 2007 New Revision: 48567 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Log: these tasks are done Modified: pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/announce.txt Sun Nov 11 15:03:27 2007 @@ -24,13 +24,8 @@ - move away from implicit keepalives - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) - - kill simulator, run the semi space tests on llinterp (may happen - before) - - have a moving gc correct version of the gc framework transformation - delegate finding type stuff like vtables etc to GC, cleaner interface for rtti, simplify translator/c/gc.py - - think about approaches to id, especially concerning boehm, where the - id will keep the object alive and concerning a moving GC - clean up the tangle of including headers in the C backend From rxe at codespeak.net Sun Nov 11 16:43:12 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 11 Nov 2007 16:43:12 +0100 (CET) Subject: [pypy-svn] r48574 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071111154312.367B9825C@code0.codespeak.net> Author: rxe Date: Sun Nov 11 16:43:11 2007 New Revision: 48574 Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py Log: skip remaining failing test - no idea why GCHeaderOffset is appearing at all? Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Sun Nov 11 16:43:11 2007 @@ -92,7 +92,7 @@ assert res == 51 def test_computed_int_symbolic(): - llvm_test() + py.test.skip("'GCHeaderOffset' object has no attribute 'TYPE'") too_early = True def compute_fn(): assert not too_early From rxe at codespeak.net Sun Nov 11 16:45:42 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 11 Nov 2007 16:45:42 +0100 (CET) Subject: [pypy-svn] r48575 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071111154542.7DF488260@code0.codespeak.net> Author: rxe Date: Sun Nov 11 16:45:42 2007 New Revision: 48575 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: make isolate work again - re-enable llvm testssvn diff test/runtest.py modwrapper.py | less Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Sun Nov 11 16:45:42 2007 @@ -10,7 +10,6 @@ prolog = """ import ctypes -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from os.path import join, dirname, realpath _c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) @@ -49,32 +48,29 @@ def identity(res): return res -def from_unichar(arg): - return ord(arg) - -def from_float(arg): - return ctypes.c_double(arg) - -def to_unichar(res): - return unichr(res) - def from_str(arg): - # XXX wont work over isolate : arg should be converted into a string first - n = len(arg.chars) class Chars(ctypes.Structure): _fields_ = [("size", ctypes.c_int), - ("data", ctypes.c_byte * n)] - + ("data", ctypes.c_byte * len(arg))] class STR(ctypes.Structure): _fields_ = [("hash", ctypes.c_int), ("chars", Chars)] s = STR() s.hash = 0 - s.chars.size = len(arg.chars) - for ii in range(s.chars.size): - s.chars.data[ii] = ord(arg.chars[ii]) + s.chars.size = len(arg) + for ii in range(len(arg)): + s.chars.data[ii] = ord(arg[ii]) return ctypes.addressof(s) +def to_r_uint(res): + return {'type':'r_uint', 'value':long(res)} + +def to_r_longlong(res): + return {'type':'r_longlong', 'value':long(res)} + +def to_r_ulonglong(res): + return {'type':'r_ulonglong', 'value':long(res)} + def to_str(res): class Chars(ctypes.Structure): _fields_ = [("size", ctypes.c_int), @@ -197,10 +193,10 @@ ctype_s.append(self.to_ctype(A)) if A is lltype.UniChar: - action = 'from_unichar' + action = 'ord' elif A is lltype.Float: - action = 'from_float' + action = 'ctypes.c_double' elif isinstance(A, lltype.Ptr) and A.TO is STR: action = 'from_str' @@ -220,13 +216,13 @@ action = 'unichr' elif T is lltype.Unsigned: - action = 'r_uint' + action = 'to_r_uint' elif T is lltype.SignedLongLong: - action = 'r_longlong' + action = 'to_r_longlong' elif T is lltype.UnsignedLongLong: - action = 'r_ulonglong' + action = 'to_r_ulonglong' elif isinstance(T, lltype.Ptr) and T.TO is STR: action = 'to_str' Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Sun Nov 11 16:45:42 2007 @@ -1,7 +1,7 @@ import py -py.test.skip("llvm is a state of flux") from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM from pypy.annotation.model import lltype_to_annotation @@ -9,32 +9,36 @@ optimize_tests = False native_llvm_backend = True -MINIMUM_LLVM_VERSION = 2.0 +MINIMUM_LLVM_VERSION = 2.1 FLOAT_PRECISION = 8 -ext_modules = [] - # prevents resource leaking -use_isolate = False +use_isolate = True # if test can't be run using isolate, skip the test (useful for buildbots) -run_isolated_only = False +run_isolated_only = True from pypy import conftest +_ext_modules = [] + def _cleanup(leave=0): from pypy.tool import isolate if leave: - mods = ext_modules[:-leave] + mods = _ext_modules[:-leave] else: - mods = ext_modules + mods = _ext_modules for mod in mods: if isinstance(mod, isolate.Isolate): - isolate.close_isolate(mod) + try: + isolate.close_isolate(mod) + except EOFError: + pass + if leave: - del ext_modules[:-leave] + del _ext_modules[:-leave] else: - del ext_modules[:] + del _ext_modules[:] def teardown_module(mod): _cleanup() @@ -49,6 +53,43 @@ #______________________________________________________________________________ +class ExceptionWrapper: + def __init__(self, class_name): + self.class_name = class_name + + def __repr__(self): + return 'ExceptionWrapper(%s)' % repr(self.class_name) + +class StructTuple(tuple): + def __getattr__(self, name): + if name.startswith('item'): + i = int(name[len('item'):]) + return self[i] + else: + raise AttributeError, name + +def wrapfn(fn): + def wrapped(*args): + callargs = [] + for a in args: + if hasattr(a, 'chars'): + callargs.append(''.join(a.chars)) + else: + callargs.append(a) + res = fn(*callargs) + if isinstance(res, dict): + # these mappings are a simple protocol to work over isolate + mapping = { + "exceptiontypename": ExceptionWrapper, + "tuple": StructTuple, + "r_uint": r_uint, + "r_longlong": r_longlong, + "r_ulonglong": r_ulonglong, + } + res = mapping[res["type"]](res["value"]) + return res + return wrapped + def genllvm_compile(function, annotation, @@ -107,25 +148,21 @@ mod, fn = genllvm_compile(function, annotation, optimize=optimize, isolate=isolate, **kwds) if isolate: - ext_modules.append(mod) - return mod, fn + _ext_modules.append(mod) + return mod, wrapfn(fn) def compile_function(function, annotation, isolate_hint=True, **kwds): " returns compiled function " return compile_test(function, annotation, isolate_hint=isolate_hint, **kwds)[1] +#______________________________________________________________________________ + # XXX Work in progress, this was mostly copied from cli -class InstanceWrapper: - def __init__(self, class_name): - self.class_name = class_name -class ExceptionWrapper: +class InstanceWrapper: def __init__(self, class_name): self.class_name = class_name - def __repr__(self): - return 'ExceptionWrapper(%s)' % repr(self.class_name) - class LLVMTest(BaseRtypingTest, LLRtypeMixin): def __init__(self): self._func = None @@ -155,17 +192,12 @@ pass def interpret(self, fn, args, annotation=None): - f = self._compile(fn, args, annotation) - res = f(*args) - - # a start to making this work over Isolate - if isinstance(res, dict): - if res["type"] == "exceptiontypename": - raise ExceptionWrapper(res["value"]) - elif res["type"] == "tuple": - res = StructTuple(res["value"]) + fn = self._compile(fn, args, annotation) + res = fn(*args) + if isinstance(res, ExceptionWrapper): + raise res return res - + def interpret_raises(self, exception, fn, args): import exceptions # needed by eval try: @@ -200,10 +232,4 @@ def read_attr(self, obj, name): py.test.skip('read_attr not supported on llvm tests') -class StructTuple(tuple): - def __getattr__(self, name): - if name.startswith('item'): - i = int(name[len('item'):]) - return self[i] - else: - raise AttributeError, name + From lac at codespeak.net Sun Nov 11 16:53:22 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 11 Nov 2007 16:53:22 +0100 (CET) Subject: [pypy-svn] r48578 - pypy/extradoc/talk/pycon2008 Message-ID: <20071111155322.704278260@code0.codespeak.net> Author: lac Date: Sun Nov 11 16:53:21 2007 New Revision: 48578 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: My grammar fixes, and some thoughts before its conference time Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Sun Nov 11 16:53:21 2007 @@ -1,27 +1,45 @@ -XXX Title (inside pypy seems to be too "inside") -================================================ +XXX Understanding PyPy, and How It Let's You Do Things You Have Only Dreamed About. (*) +================================================================================ +(* for particularly nerdy dreams :-) ) -In this talk, we would like to present few features exclusive to -the PyPy python interpret and novel-implementation approaches -which made it easy to implement those features. The talk would -consist on: +In this talk, we would like to present our paradigm-shifting architecture +for dynamic langauges and some features which, due to our appraoch +are exclusively available with PyPy. -* Transparent proxy concept, which allows programmers to create +Topics would include: + +* The transparent proxy, which allows programmers to create transparent object of any type in python (including built-in python types like frames), with various usage showcases including our completely transparent remote access scheme. (XXX: I know what you mean, but "create transparent object" is a bit vague) -* Sandboxing concept, which allows to run non-stripped version of +* the PyPy Sandbox, which allows to run non-stripped version of python interpreter in a controlled environment with custom safety policy. -* Taint object space, which allows programmers to make sure that sensitive - informations do not cross the I/O barrier. +* The Taint Object Space, which allows programmers to make sure that sensitive + information does not cross the I/O barrier. -* XXX anything else? +* XXX anything else? lazy evaluation? * Why our approach is "working by design", rather then "fix as long as anyone complains", why it's only permitted by our design decisions and abstraction levels. How our all of the mentioned above examples could be reduce to simple "proxy operation" concept. + + 'working by design' means something else. it means 'we designed it + to work, which, of course, is true of everything that didn't happen + by accident (and maybe that too, if some people's ideas of God are + are correct). + +* _why_ design matters. _when_ design matters. Some thoughts on when + incremental improvements are not the answer, and why successive + improvements in painting things red will not turn your children's + wagon into a firetruck, despite being the perfect approach to the + problem of 'making what I have red'. + +I am getting all these weird ideas of how a caterpillar becomes a +larger caterpillar is not the same way a caterpillar becomes a +butterfly. Either this is a useful analogy, or I am nuts. + From fijal at codespeak.net Sun Nov 11 18:09:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Nov 2007 18:09:37 +0100 (CET) Subject: [pypy-svn] r48579 - pypy/dist/pypy/objspace/cpy/test Message-ID: <20071111170937.A7B41822D@code0.codespeak.net> Author: fijal Date: Sun Nov 11 18:09:36 2007 New Revision: 48579 Modified: pypy/dist/pypy/objspace/cpy/test/test_objspace.py Log: Import py if we want to skip Modified: pypy/dist/pypy/objspace/cpy/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_objspace.py Sun Nov 11 18:09:36 2007 @@ -2,6 +2,7 @@ from pypy.tool.pytest.appsupport import raises_w from pypy.rlib.rarithmetic import r_longlong, r_ulonglong from pypy.rlib.rbigint import rbigint +import py def test_simple(): space = CPyObjSpace() From fijal at codespeak.net Sun Nov 11 18:17:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Nov 2007 18:17:57 +0100 (CET) Subject: [pypy-svn] r48580 - pypy/dist/pypy/module/zipimport/test Message-ID: <20071111171757.F366F824C@code0.codespeak.net> Author: fijal Date: Sun Nov 11 18:17:56 2007 New Revision: 48580 Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py Log: Fix finally this breaking test. Modified: pypy/dist/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/test/test_zipimport.py Sun Nov 11 18:17:56 2007 @@ -81,6 +81,9 @@ space.appexec([space.wrap(self)], """(self): self.write_files = [] """) + space.setattr(space.getbuiltinmodule('zipimport'), + space.wrap('_zip_directory_cache'), + space.newdict({})) def teardown_method(self, meth): space = self.space @@ -183,12 +186,13 @@ def test_package(self): import os, sys - self.writefile(self, "xx"+os.sep+"__init__.py", "") - self.writefile(self, "xx"+os.sep+"yy.py", "def f(x): return x") - mod = __import__("xx", globals(), locals(), ['yy']) + self.writefile(self, "xxuuu"+os.sep+"__init__.py", "") + self.writefile(self, "xxuuu"+os.sep+"yy.py", "def f(x): return x") + mod = __import__("xxuuu", globals(), locals(), ['yy']) assert mod.__path__ assert mod.yy.f(3) == 3 - del sys.modules['xx'] + del sys.modules['xxuuu'] + del sys.modules['xxuuu.yy'] def test_functions(self): import os From cfbolz at codespeak.net Sun Nov 11 22:45:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Nov 2007 22:45:53 +0100 (CET) Subject: [pypy-svn] r48581 - pypy/branch/more-unicode-improvements Message-ID: <20071111214553.9A0B8810E@code0.codespeak.net> Author: cfbolz Date: Sun Nov 11 22:45:52 2007 New Revision: 48581 Added: pypy/branch/more-unicode-improvements/ - copied from r48580, pypy/dist/ Log: create a new branch for more unicode improvements From rxe at codespeak.net Sun Nov 11 22:58:54 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 11 Nov 2007 22:58:54 +0100 (CET) Subject: [pypy-svn] r48582 - pypy/dist/pypy/translator/llvm Message-ID: <20071111215854.D40F58227@code0.codespeak.net> Author: rxe Date: Sun Nov 11 22:58:54 2007 New Revision: 48582 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/opwriter.py Log: cosmetics Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Sun Nov 11 22:58:54 2007 @@ -63,7 +63,9 @@ def header_comment(self, s): self.newline() - self.comment(s) + self.comment('=' * 78, indent=False) + self.comment(s, indent=False) + self.comment('=' * 78, indent=False) self.newline() def newline(self): Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sun Nov 11 22:58:54 2007 @@ -150,7 +150,10 @@ if not meth: raise Exception, "operation %s not found" % op.opname meth(opr) - + + if self.db.genllvm.config.translation.llvm.debug: + self.codewriter.newline() + def _skipped(self, opr): self.codewriter.comment('***Skipping operation %s()' % opr.op.opname) keepalive = _skipped From rxe at codespeak.net Sun Nov 11 23:06:23 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 11 Nov 2007 23:06:23 +0100 (CET) Subject: [pypy-svn] r48583 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071111220623.879BA822F@code0.codespeak.net> Author: rxe Date: Sun Nov 11 23:06:23 2007 New Revision: 48583 Removed: pypy/dist/pypy/translator/llvm/module/protos.h Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/genexterns.c pypy/dist/pypy/translator/llvm/module/support.py Log: kill a whole bunch of code which was mostly hacks to work with externs - most the tests still pass (ll_strtod_parts_to_float and ll_strtod_formatd will soon be implemented via rffi) we also dont rely on translator/c/extfunc.py anymore Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sun Nov 11 23:06:23 2007 @@ -6,7 +6,7 @@ from pypy.translator.llvm.typedefnode import create_typedef_node from pypy.translator.llvm.funcnode import FuncImplNode -from pypy.translator.llvm.extfuncnode import ExternalFuncNode, SimplerExternalFuncNode +from pypy.translator.llvm.extfuncnode import ExternalFuncNode from pypy.translator.llvm.opaquenode import OpaqueNode, ExtOpaqueNode from pypy.translator.llvm.structnode import StructNode, StructVarsizeNode, \ getindexhelper, FixedSizeArrayNode @@ -65,12 +65,11 @@ node = None if isinstance(type_, lltype.FuncType): if getattr(value._callable, "suggested_primitive", False): - node = ExternalFuncNode(self, value) + node = ExternalFuncNode(self, value, value._callable) elif hasattr(value, '_external_name'): node = ExternalFuncNode(self, value, value._external_name) - elif getattr(value, 'external', None) == 'C': - node = SimplerExternalFuncNode(self, value) + node = ExternalFuncNode(self, value) else: node = FuncImplNode(self, value) Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sun Nov 11 23:06:23 2007 @@ -9,22 +9,66 @@ from pypy.translator.llvm.buildllvm import llvm_gcc_version from pypy.tool.udir import udir +from pypy.rpython.module import ll_stack +from pypy.rpython.lltypesystem.module import ll_strtod + +ll_stack_too_big = """ + +define internal ccc i1 @LL_stack_too_big_() { + %result = call ccc i32 @LL_stack_too_big() + %tmp = trunc i32 %result to i1 + ret i1 %tmp +} + +""" + +# table of functions hand-written in src/ll_*.h +# Note about *.im_func: The annotator and the rtyper expect direct +# references to functions, so we cannot insert classmethods here. + +EXTERNALS = { + ll_stack.ll_stack_unwind: 'LL_stack_unwind', + ll_stack.ll_stack_too_big: 'LL_stack_too_big_', + } + + +math_functions = [ + 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', + 'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', + 'pow', 'atan2', 'fmod', 'ldexp', 'hypot' + ] +import math +for name in math_functions: + EXTERNALS['ll_math.ll_math_%s' % name] = 'LL_math_%s' % name + +def predeclare_stuff(c_db): + modules = {} + def module_name(c_name): + frags = c_name[3:].split('_') + if frags[0] == '': + return '_' + frags[1] + else: + return frags[0] + + for func, funcobj in c_db.externalfuncs.items(): + c_name = EXTERNALS[func] + # construct a define LL_NEED_ to make it possible to isolate in-development externals and headers + modname = module_name(c_name) + if modname not in modules: + modules[modname] = True + yield 'LL_NEED_%s' % modname.upper(), 1 + funcptr = funcobj._as_ptr() + yield c_name, funcptr + + exctransformer = c_db.exctransformer + yield ('_rpyexc_occured_ptr', exctransformer._rpyexc_occured_ptr.value) + yield ('rpyexc_fetch_type_ptr', exctransformer.rpyexc_fetch_type_ptr.value) + yield ('rpyexc_clear_ptr', exctransformer.rpyexc_clear_ptr.value) support_functions = [ - "@raisePyExc_IOError", - "@raisePyExc_ValueError", - "@raisePyExc_OverflowError", - "@raisePyExc_ZeroDivisionError", - "@raisePyExc_RuntimeError", - "@raisePyExc_thread_error", - "@RPyString_FromString", - "@RPyString_AsString", - "@RPyString_Size", - "@RPyExceptionOccurred", "@LLVM_RPython_StartupCode", ] - skip_lines = [ "%RPyString = type opaque", "__main", @@ -120,23 +164,21 @@ ll_lines2.append(line) ll_lines2.append("declare ccc void @abort()") + return'\n'.join(ll_lines2) - llcode = '\n'.join(ll_lines2) - decl, impl = '', llcode - #try: - # decl, impl = llcode.split('implementation') - #except: - # raise Exception("Can't compile external function code (llcode.c)") - return decl, impl - +def find_list_of_str(rtyper): + from pypy.rpython.lltypesystem import rlist + for r in rtyper.reprs.itervalues(): + if isinstance(r, rlist.ListRepr) and r.item_repr is rstr.string_repr: + return r.lowleveltype.TO + return None def setup_externs(c_db, db): - rtyper = db.translator.rtyper - from pypy.translator.c.extfunc import predeclare_all - # hacks to make predeclare_all work - decls = list(predeclare_all(c_db, rtyper)) + rtyper = db.translator.rtyper + decls = list(predeclare_stuff(c_db)) + for c_name, obj in decls: if isinstance(obj, lltype.LowLevelType): db.prepare_type(obj) @@ -197,30 +239,29 @@ predeclarefn("__ENTRY_POINT__", entrynode.get_ref()) ccode.append('#define ENTRY_POINT_DEFINED 1\n\n') + LL_stack_too_big_ = False for c_name, obj in extern_decls: if isinstance(obj, lltype.LowLevelType): s = "#define %s struct %s\n%s;\n" % (c_name, c_name, c_name) ccode.append(s) + elif isinstance(obj, FunctionGraph): funcptr = db.translator.rtyper.getcallable(obj) c = inputconst(lltype.typeOf(funcptr), funcptr) predeclarefn(c_name, db.repr_arg(c)) + elif isinstance(lltype.typeOf(obj), lltype.Ptr): - if c_name.startswith("RPyExc_"): - c_name = c_name[1:] - ccode.append("void raise%s(char *);\n" % c_name) - #else: - # # XXX we really shouldnt do this - # predeclarefn(c_name, db.obj2node[obj._obj].ref) + pass + elif type(c_name) is str and type(obj) is int: + if c_name == 'LL_NEED_STACK': + LL_stack_too_big_ = True ccode.append("#define\t%s\t%d\n" % (c_name, obj)) + else: assert False, "unhandled extern_decls %s %s %s" % (c_name, type(obj), obj) - # append protos - ccode.append(open(get_module_file('protos.h')).read()) - # include this early to get constants and macros for any further includes ccode.append('#include \n') @@ -238,4 +279,10 @@ # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) - return get_ll("".join(ccode), function_names, default_cconv) + llcode = get_ll("".join(ccode), function_names, default_cconv) + + # XXX enormous temporary hack + if LL_stack_too_big_: + llcode += ll_stack_too_big + + return llcode Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Sun Nov 11 23:06:23 2007 @@ -1,33 +1,16 @@ from pypy.translator.llvm.node import FuncNode -from pypy.translator.llvm.log import log -from pypy.translator.c.extfunc import EXTERNALS +from pypy.translator.llvm.externs2ll import EXTERNALS from pypy.rpython.lltypesystem import lltype -log = log.extfuncnode - -from sys import maxint - -class ExtFuncSig(object): - def __init__(self, rettype, args): - self.rettype = rettype - self.args = args - -# signature of external functions differ from C's implementation -ext_func_sigs = { - "@LL_stack_too_big_" : ExtFuncSig("i32", None), - } - -# XXX 64 bit is really broken since 2.x -#if maxint != 2**31-1: -# ext_func_sigs["@LL_math_ldexp"] = ExtFuncSig(None, [None, "int"]) - - -class SimplerExternalFuncNode(FuncNode): - - def __init__(self, db, value): +class ExternalFuncNode(FuncNode): + def __init__(self, db, value, mapname=None): + if mapname is None: + name = value._name + else: + name = EXTERNALS[mapname] self.db = db self.value = value - self.name = "@" + value._name + self.name = "@" + name def external_c_source(self): # return a list of unique includes and sources in C @@ -58,132 +41,3 @@ def writedecl(self, codewriter): codewriter.declare(self.getdecl()) -class ExternalFuncNode(FuncNode): - prefix = '@' - - def __init__(self, db, value, extname=None): - self.db = db - self.value = value - name = value._callable.__name__ - - self.callable = value._callable - if extname is not None: - mapped_name = EXTERNALS[extname] - else: - mapped_name = EXTERNALS[self.callable] - - self.make_name(mapped_name) - if ext_func_sigs.get(self.ref + '_', None): - self.origref = self.ref - self.name += "_" - - def setup(self): - self.db.prepare_type(self.value._TYPE.RESULT) - self.db.prepare_type_multi(self.value._TYPE._trueargs()) - - def _get_wrapper(self): - wrapper = ext_func_sigs.get(self.ref, None) - -# if wrapper is None and maxint != 2**31-1: -# #log("ref=%s" % self.ref) -# rettype, args = self.getdecl_parts() -# conversions = False -# if rettype == "long": -# rettype = "int" -# conversions = True -# elif rettype == "ulong": -# rettype = "uint" -# conversions = True -# else: -# rettype = None -# for i, a in enumerate(args): -# if a == "long": -# args[i] = "int" -# conversions = True -# elif a == "ulong": -# args[i] = "uint" -# conversions = True -# else: -# args[i] = None -# if conversions: -# wrapper = ExtFuncSig(rettype, args) -# #log(" rettype=%s" % str(rettype)) -# #log(" args =%s" % str(args)) - return wrapper - - def getdecl_parts(self): - T = self.value._TYPE - rettype = self.db.repr_type(T.RESULT) - argtypes = [self.db.repr_type(a) for a in T.ARGS if a is not lltype.Void] - return rettype, argtypes - - def getdecl(self): - rettype, argtypes = self.getdecl_parts() - return "%s %s(%s)" % (rettype, self.ref, ", ".join(argtypes)) - - def writedecl(self, codewriter): - codewriter.declare(self.getdecl()) - - def writeimpl(self, codewriter): - wrapper = self._get_wrapper() - if wrapper is None: - return - - rettype, argtypes = self.getdecl_parts() - argrefs = [self.db.repr_tmpvar() for ii in argtypes] - arg_desription = ", ".join([ - "%s %s" % (typ_, name) - for typ_, name in zip(argtypes, argrefs)]) - open_decl = "%s %s(%s)" % (rettype, self.ref, arg_desription) - codewriter.openfunc(open_decl) - - returnval = self.db.repr_tmpvar() - - # call function with this - expected_argrefs = [] - expected_argtypes = [] - - # find out what the args/types should be - if wrapper.args is not None: - assert len(wrapper.args) == len(argtypes) - - for expected_typ, typ, ref in zip(wrapper.args, - argtypes, - argrefs): - if expected_typ is not None: - - # cast to desired arg type - expected_ref = self.db.repr_tmpvar() - codewriter.cast(expected_ref, typ, ref, expected_typ) - - else: - expected_ref = ref - expected_typ = typ - - expected_argrefs.append(expected_ref) - expected_argtypes.append(expected_typ) - else: - expected_argrefs = argrefs - expected_argtypes = argtypes - - # find out what the return type should be - expected_rettype = wrapper.rettype or rettype - - # call - codewriter.call(returnval, expected_rettype, self.origref, - expected_argtypes, expected_argrefs) - - if wrapper.rettype: - # cast to desired return type - tmpval = returnval - returnval = self.db.repr_tmpvar() - # XXX note we only have one case above - more will need - # to specialize the cast - codewriter.cast(returnval, wrapper.rettype, - tmpval, rettype, 'trunc') - - codewriter.ret(rettype, returnval) - codewriter.closefunc() - - def writeglobalconstants(self, codewriter): - pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sun Nov 11 23:06:23 2007 @@ -10,8 +10,7 @@ from pypy.tool.udir import udir from pypy.translator.llvm.codewriter import CodeWriter from pypy.translator.llvm import extfuncnode -from pypy.translator.llvm.module.support import \ - extdeclarations, extfunctions, extfunctions_standalone, write_raise_exc +from pypy.translator.llvm.module.support import extfunctions from pypy.translator.llvm.node import Node from pypy.translator.llvm.externs2ll import setup_externs, generate_llfile from pypy.translator.llvm.gc import GcPolicy @@ -32,19 +31,41 @@ self.config = translator.config def gen_source(self, func): - self._checkpoint() + self._checkpoint("before gen source") codewriter = self.setup(func) - # write top part of llvm file - self.write_headers(codewriter) + codewriter.header_comment("Extern code") + codewriter.write_lines(self.llcode) - codewriter.startimpl() + codewriter.header_comment("Type declarations") + for typ_decl in self.db.gettypedefnodes(): + typ_decl.writetypedef(codewriter) - # write bottom part of llvm file - self.write_implementations(codewriter) + codewriter.header_comment("Function prototypes") + for node in self.db.getnodes(): + if hasattr(node, 'writedecl'): + node.writedecl(codewriter) + + codewriter.header_comment("Prebuilt constants") + for node in self.db.getnodes(): + # XXX tmp + if hasattr(node, "writeglobalconstants"): + node.writeglobalconstants(codewriter) + + self._checkpoint("before definitions") + + codewriter.header_comment('Suppport definitions') + codewriter.write_lines(extfunctions, patch=True) + + codewriter.header_comment('Startup definition') + self.write_startup_impl(codewriter) + + codewriter.header_comment("Function definitions") + for node in self.db.getnodes(): + if hasattr(node, 'writeimpl'): + node.writeimpl(codewriter) - # write entry point if there is one codewriter.comment("End of file") codewriter.close() self._checkpoint('done') @@ -98,65 +119,7 @@ self._checkpoint('setup_externs') return codewriter - - def write_headers(self, codewriter): - # write external function headers - codewriter.header_comment('External Function Headers') - codewriter.write_lines(self.llexterns_header) - - codewriter.header_comment("Type Declarations") - - # write extern type declarations - self.write_extern_decls(codewriter) - self._checkpoint('write externs type declarations') - - # write node type declarations - for typ_decl in self.db.gettypedefnodes(): - typ_decl.writetypedef(codewriter) - self._checkpoint('write data type declarations') - - codewriter.header_comment("Global Data") - - # write pbcs - for node in self.db.getnodes(): - # XXX tmp - if hasattr(node, "writeglobalconstants"): - node.writeglobalconstants(codewriter) - self._checkpoint('write global constants') - - codewriter.header_comment("Function Prototypes") - - # write external protos - codewriter.write_lines(extdeclarations, patch=True) - - # write node protos - for node in self.db.getnodes(): - if hasattr(node, 'writedecl'): - node.writedecl(codewriter) - - self._checkpoint('write function prototypes') - - def write_implementations(self, codewriter): - codewriter.header_comment("Function Implementation") - - # write external function implementations - codewriter.header_comment('External Function Implementation') - codewriter.write_lines(self.llexterns_functions) - codewriter.write_lines(extfunctions, patch=True) - if self.standalone: - codewriter.write_lines(extfunctions_standalone, patch=True) - self.write_extern_impls(codewriter) - self.write_setup_impl(codewriter) - - self._checkpoint('write support implentations') - - # write all node implementations - for node in self.db.getnodes(): - if hasattr(node, 'writeimpl'): - node.writeimpl(codewriter) - - self._checkpoint('write node implementations') - + def get_entry_point(self, func): assert func is not None self.entrypoint = func @@ -191,14 +154,13 @@ for source in sources: c_sources[source] = True - self.llexterns_header, self.llexterns_functions = \ - generate_llfile(self.db, - self.extern_decls, - self.entrynode, - c_includes, - c_sources, - self.standalone, - codewriter.cconv) + self.llcode = generate_llfile(self.db, + self.extern_decls, + self.entrynode, + c_includes, + c_sources, + self.standalone, + codewriter.cconv) def create_codewriter(self): # prevent running the same function twice in a test @@ -208,24 +170,8 @@ return CodeWriter(f, self.db), filename else: return CodeWriter(f, self.db, cconv='ccc', linkage=''), filename - - def write_extern_decls(self, codewriter): - for c_name, obj in self.extern_decls: - if isinstance(obj, lltype.LowLevelType): - if isinstance(obj, lltype.Ptr): - obj = obj.TO - - l = "%%%s = type %s" % (c_name, self.db.repr_type(obj)) - codewriter.write_lines(l) - def write_extern_impls(self, codewriter): - for c_name, obj in self.extern_decls: - if c_name.startswith("RPyExc_"): - c_name = c_name[1:] - exc_repr = self.db.repr_constant(obj)[1] - write_raise_exc(c_name, exc_repr, codewriter) - - def write_setup_impl(self, codewriter): + def write_startup_impl(self, codewriter): open_decl = "i8* @LLVM_RPython_StartupCode()" codewriter.openfunc(open_decl) for node in self.db.getnodes(): Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Sun Nov 11 23:06:23 2007 @@ -1,11 +1,24 @@ +#include -// append some genc files here manually from python -#ifdef _RPyListOfString_New /* :-( */ -# define HAVE_RPY_LIST_OF_STRING -#endif +char *LLVM_RPython_StartupCode(void); + +#define RPyRaiseSimpleException(exctype, errormsg) raise##exctype(errormsg); + +// XXX abort() this is just to make tests pass. actually it is a million times +// better than it was since it used to basically be a nooop. + +// all of these will go away at some point + +#define FAKE_ERROR(name) \ + int raisePyExc_##name(char *x) { \ + abort(); \ + } #ifdef LL_NEED_MATH + FAKE_ERROR(OverflowError); + FAKE_ERROR(ValueError); #include "c/src/ll_math.h" + #endif #ifdef LL_NEED_STRTOD @@ -13,10 +26,12 @@ #endif #ifdef LL_NEED_STACK + FAKE_ERROR(RuntimeError); #include "c/src/thread.h" #include "c/src/stack.h" #endif + // raw malloc code char *raw_malloc(long size) { return malloc(size); @@ -34,10 +49,7 @@ memset(ptr, 0, size); } -char *LLVM_RPython_StartupCode(); - char *RPython_StartupCode() { - // is there any garbage collection / memory management initialisation __GC_STARTUP_CODE__ @@ -50,6 +62,7 @@ int main(int argc, char *argv[]) { + XXX char *errmsg; int i, exitcode; RPyListOfString *list; Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Sun Nov 11 23:06:23 2007 @@ -1,31 +1,4 @@ -extdeclarations = """ -declare ccc i32 @strlen(i8*) -""" - extfunctions = """ -define internal CC %RPyString* @RPyString_FromString(i8* %s) { - %len = call ccc i32 @strlen(i8* %s) - %rpy = call CC %RPyString* @pypy_RPyString_New__Signed(i32 %len) - %rpystrptr = getelementptr %RPyString* %rpy, i32 0, i32 1, i32 1 - %rpystr = bitcast [0 x i8]* %rpystrptr to i8* - - call ccc void @llvm.memcpyPOSTFIX(i8* %rpystr, i8* %s, WORD %len, WORD 0) - - ret %RPyString* %rpy -} - -define internal CC i8* @RPyString_AsString(%RPyString* %structstring) { - %source1ptr = getelementptr %RPyString* %structstring, i32 0, i32 1, i32 1 - %source1 = bitcast [0 x i8]* %source1ptr to i8* - ret i8* %source1 -} - -define internal CC WORD @RPyString_Size(%RPyString* %structstring) { - %sizeptr = getelementptr %RPyString* %structstring, i32 0, i32 1, i32 0 - %size = load WORD* %sizeptr - ret WORD %size -} - define internal CC double @pypyop_float_abs(double %x) { block0: %cond1 = fcmp ugt double %x, 0.0 @@ -61,28 +34,5 @@ %result = phi i64 [%x, %block0], [%x2, %block1] ret i64 %result } - """ -extfunctions_standalone = """ -""" -from sys import maxint -if maxint != 2**31-1: - extfunctions_standalone += """ -internal CC int %pypy_entry_point(%RPyListOfString* %argv) { - %result = call CC long %pypy_entry_point(%RPyListOfString* %argv) - %tmp = cast long %result to int - ret int %tmp -} - -""" - -def write_raise_exc(c_name, exc_repr, codewriter): - l = """ -define internal CC void @raise%s(i8* %%msg) { - ;%%exception_value = cast %s to %%RPYTHON_EXCEPTION* - ret void -} -""" % (c_name, exc_repr) - codewriter.write_lines(l, patch=True) - From cfbolz at codespeak.net Sun Nov 11 23:34:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Nov 2007 23:34:05 +0100 (CET) Subject: [pypy-svn] r48584 - in pypy/branch/more-unicode-improvements/pypy/objspace/std: . test Message-ID: <20071111223405.1F8D48257@code0.codespeak.net> Author: cfbolz Date: Sun Nov 11 23:34:03 2007 New Revision: 48584 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Log: move unicode.encode to interplevel. add several encoding tests from CPython's tests suite. Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py Sun Nov 11 23:34:03 2007 @@ -1,3 +1,4 @@ +import py import sys @@ -377,3 +378,84 @@ def test_missing_cases(self): # some random cases, which are discovered to not be tested during annotation assert u'xxx'[1:1] == u'' + + # these tests test lots of encodings, so they really belong to the _codecs + # module. however, they test useful unicode methods too + # they are stolen from CPython's unit tests + + def test_codecs_utf7(self): + utfTests = [ + (u'A\u2262\u0391.', 'A+ImIDkQ.'), # RFC2152 example + (u'Hi Mom -\u263a-!', 'Hi Mom -+Jjo--!'), # RFC2152 example + (u'\u65E5\u672C\u8A9E', '+ZeVnLIqe-'), # RFC2152 example + (u'Item 3 is \u00a31.', 'Item 3 is +AKM-1.'), # RFC2152 example + (u'+', '+-'), + (u'+-', '+--'), + (u'+?', '+-?'), + (u'\?', '+AFw?'), + (u'+?', '+-?'), + (ur'\\?', '+AFwAXA?'), + (ur'\\\?', '+AFwAXABc?'), + (ur'++--', '+-+---') + ] + + for (x, y) in utfTests: + assert x.encode('utf-7') == y + + # surrogates not supported + raises(UnicodeError, unicode, '+3ADYAA-', 'utf-7') + + assert unicode('+3ADYAA-', 'utf-7', 'replace') == u'\ufffd' + + def test_codecs_utf8(self): + assert u''.encode('utf-8') == '' + assert u'\u20ac'.encode('utf-8') == '\xe2\x82\xac' + assert u'\ud800\udc02'.encode('utf-8') == '\xf0\x90\x80\x82' + assert u'\ud84d\udc56'.encode('utf-8') == '\xf0\xa3\x91\x96' + assert u'\ud800'.encode('utf-8') == '\xed\xa0\x80' + assert u'\udc00'.encode('utf-8') == '\xed\xb0\x80' + assert (u'\ud800\udc02'*1000).encode('utf-8') == '\xf0\x90\x80\x82'*1000 + assert ( + u'\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + u'\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + u'\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + u'\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + u'\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das' + u' Nunstuck git und'.encode('utf-8') == + '\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81' + '\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3' + '\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe' + '\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' + '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8' + '\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81' + '\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81' + '\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3' + '\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf' + '\xe3\x80\x8cWenn ist das Nunstuck git und' + ) + + # UTF-8 specific decoding tests + assert unicode('\xf0\xa3\x91\x96', 'utf-8') == u'\U00023456' + assert unicode('\xf0\x90\x80\x82', 'utf-8') == u'\U00010002' + assert unicode('\xe2\x82\xac', 'utf-8') == u'\u20ac' + + def test_codecs_errors(self): + # Error handling (encoding) + raises(UnicodeError, u'Andr\202 x'.encode, 'ascii') + raises(UnicodeError, u'Andr\202 x'.encode, 'ascii','strict') + assert u'Andr\202 x'.encode('ascii','ignore') == "Andr x" + assert u'Andr\202 x'.encode('ascii','replace') == "Andr? x" + + # Error handling (decoding) + raises(UnicodeError, unicode, 'Andr\202 x', 'ascii') + raises(UnicodeError, unicode, 'Andr\202 x', 'ascii','strict') + assert unicode('Andr\202 x','ascii','ignore') == u"Andr x" + assert unicode('Andr\202 x','ascii','replace') == u'Andr\uFFFD x' + + # Error handling (unknown character names) + assert "\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx" + + # Error handling (truncated escape sequence) + raises(UnicodeError, "\\".decode, "unicode-escape") + + Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Sun Nov 11 23:34:03 2007 @@ -735,28 +735,29 @@ return W_UnicodeObject(w_new._value.join(parts)) -app = gateway.applevel(r''' -import sys - -def unicode_encode__Unicode_ANY_ANY(unistr, encoding=None, errors=None): - import codecs, sys +def unicode_encode__Unicode_ANY_ANY(space, w_unistr, + w_encoding=None, + w_errors=None): + + from pypy.objspace.std.unicodetype import getdefaultencoding + from pypy.objspace.std.unicodetype import _get_encoding_and_errors + w_codecs = space.getbuiltinmodule("_codecs") + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None: - encoding = sys.getdefaultencoding() - - encoder = codecs.getencoder(encoding) + encoding = getdefaultencoding(space) + w_encode = space.getattr(w_codecs, space.wrap("encode")) if errors is None: - retval, lenght = encoder(unistr) + w_retval = space.call_function(w_encode, w_unistr, space.wrap(encoding)) else: - retval, length = encoder(unistr, errors) - if not isinstance(retval,str): - raise TypeError("encoder did not return a string object (type=%s)" % - type(retval).__name__) - return retval -''') - - - -unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') + w_retval = space.call_function(w_encode, w_unistr, space.wrap(encoding), + space.wrap(errors)) + if not space.is_true(space.isinstance(w_retval, space.w_str)): + raise OperationError( + space.w_TypeError, + space.wrap( + "encoder did not return an unicode object (type=%s)" % + space.type(w_retval).getname(space, '?'))) + return w_retval def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): unistr = w_unistr._value From cfbolz at codespeak.net Sun Nov 11 23:39:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Nov 2007 23:39:43 +0100 (CET) Subject: [pypy-svn] r48585 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071111223943.400008259@code0.codespeak.net> Author: cfbolz Date: Sun Nov 11 23:39:42 2007 New Revision: 48585 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Log: add an XXX Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Sun Nov 11 23:39:42 2007 @@ -231,7 +231,7 @@ w_value = unicode_from_object(space, w_obj) else: w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) - # help the annotator! also the ._value depends on W_UnicodeObject layout + # XXX this is not true when there are different unicode implementations assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) W_UnicodeObject.__init__(w_newobj, w_value._value) From cfbolz at codespeak.net Sun Nov 11 23:51:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Nov 2007 23:51:23 +0100 (CET) Subject: [pypy-svn] r48586 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071111225123.BC7F1825A@code0.codespeak.net> Author: cfbolz Date: Sun Nov 11 23:51:23 2007 New Revision: 48586 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Log: fix error message Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Sun Nov 11 23:51:23 2007 @@ -755,7 +755,7 @@ raise OperationError( space.w_TypeError, space.wrap( - "encoder did not return an unicode object (type=%s)" % + "encoder did not return an string object (type=%s)" % space.type(w_retval).getname(space, '?'))) return w_retval From cfbolz at codespeak.net Mon Nov 12 00:16:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 00:16:12 +0100 (CET) Subject: [pypy-svn] r48588 - in pypy/branch/more-unicode-improvements/pypy/objspace/std: . test Message-ID: <20071111231612.2E7A9824C@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 00:16:11 2007 New Revision: 48588 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_stringobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Log: move string.encode and string.decode to interp-level. share code where possible. Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py Mon Nov 12 00:16:11 2007 @@ -51,7 +51,6 @@ # XXX should this use the default encoding? return _decode_ascii(space, w_self._value) - def _is_generic(space, w_self, fun): v = w_self._value if len(v) == 0: @@ -905,29 +904,25 @@ L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] return ''.join(L) - def str_decode__String_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getdecoder(encoding)(str)[0] - else: - return codecs.getdecoder(encoding)(str, errors)[0] - - def str_encode__String_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getencoder(encoding)(str)[0] - else: - return codecs.getencoder(encoding)(str, errors)[0] ''', filename=__file__) str_translate__String_ANY_ANY = app.interphook('str_translate__String_ANY_ANY') -str_decode__String_ANY_ANY = app.interphook('str_decode__String_ANY_ANY') -str_encode__String_ANY_ANY = app.interphook('str_encode__String_ANY_ANY') + +def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + unicode_from_string, encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + if encoding is None and errors is None: + return unicode_from_string(space, w_string) + return encode_object(space, w_string, encoding, errors) + +def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + #import pdb; pdb.set_trace() + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + return encode_object(space, w_string, encoding, errors) # CPython's logic for deciding if ""%values is # an error (1 value, 0 %-formatters) or not Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_stringobject.py Mon Nov 12 00:16:11 2007 @@ -645,6 +645,10 @@ def test_decode(self): assert 'hello'.decode('rot-13') == 'uryyb' assert 'hello'.decode('string-escape') == 'hello' + + def test_encode(self): + assert 'hello'.encode() == 'hello' + assert type('hello'.encode()) is str def test_hash(self): # check that we have the same hash as CPython for at least 31 bits Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Mon Nov 12 00:16:11 2007 @@ -739,24 +739,12 @@ w_encoding=None, w_errors=None): - from pypy.objspace.std.unicodetype import getdefaultencoding - from pypy.objspace.std.unicodetype import _get_encoding_and_errors - w_codecs = space.getbuiltinmodule("_codecs") + from pypy.objspace.std.unicodetype import getdefaultencoding, \ + _get_encoding_and_errors, encode_object encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None: encoding = getdefaultencoding(space) - w_encode = space.getattr(w_codecs, space.wrap("encode")) - if errors is None: - w_retval = space.call_function(w_encode, w_unistr, space.wrap(encoding)) - else: - w_retval = space.call_function(w_encode, w_unistr, space.wrap(encoding), - space.wrap(errors)) - if not space.is_true(space.isinstance(w_retval, space.w_str)): - raise OperationError( - space.w_TypeError, - space.wrap( - "encoder did not return an string object (type=%s)" % - space.type(w_retval).getname(space, '?'))) + w_retval = encode_object(space, w_unistr, encoding, errors) return w_retval def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Mon Nov 12 00:16:11 2007 @@ -145,7 +145,34 @@ def getdefaultencoding(space): return space.sys.defaultencoding -def unicode_from_encoded_object(space, w_obj, encoding, errors): +def _get_encoding_and_errors(space, w_encoding, w_errors): + if space.is_w(w_encoding, space.w_None): + encoding = None + else: + encoding = space.str_w(w_encoding) + if space.is_w(w_errors, space.w_None): + errors = None + else: + errors = space.str_w(w_errors) + return encoding, errors + +def encode_object(space, w_object, encoding, errors): + w_codecs = space.getbuiltinmodule("_codecs") + w_encode = space.getattr(w_codecs, space.wrap("encode")) + if errors is None: + w_retval = space.call_function(w_encode, w_object, space.wrap(encoding)) + else: + w_retval = space.call_function(w_encode, w_object, space.wrap(encoding), + space.wrap(errors)) + if not space.is_true(space.isinstance(w_retval, space.w_str)): + raise OperationError( + space.w_TypeError, + space.wrap( + "encoder did not return an string object (type=%s)" % + space.type(w_retval).getname(space, '?'))) + return w_retval + +def decode_object(space, w_object, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: encoding = getdefaultencoding(space) @@ -155,6 +182,11 @@ else: w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding), space.wrap(errors)) + return w_retval + + +def unicode_from_encoded_object(space, w_obj, encoding, errors): + w_retval = decode_object(space, w_obj, encoding, errors) if not space.is_true(space.isinstance(w_retval, space.w_unicode)): raise OperationError( space.w_TypeError, @@ -163,7 +195,6 @@ space.type(w_retval).getname(space, '?'))) return w_retval - def unicode_from_object(space, w_obj): if space.is_true(space.isinstance(w_obj, space.w_str)): w_res = w_obj @@ -197,17 +228,6 @@ return unicode_from_object(space, w_str) -def _get_encoding_and_errors(space, w_encoding, w_errors): - if space.is_w(w_encoding, space.w_None): - encoding = None - else: - encoding = space.str_w(w_encoding) - if space.is_w(w_errors, space.w_None): - errors = None - else: - errors = space.str_w(w_errors) - return encoding, errors - def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work From cfbolz at codespeak.net Mon Nov 12 01:40:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 01:40:07 +0100 (CET) Subject: [pypy-svn] r48589 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071112004007.68A9D825E@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 01:40:05 2007 New Revision: 48589 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py Log: move string.translate to interplevel Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py Mon Nov 12 01:40:05 2007 @@ -889,25 +889,28 @@ return space.wrap("".join(buf[:i+1])) # buffer was overallocated, so slice -app = gateway.applevel(r''' - def str_translate__String_ANY_ANY(s, table, deletechars=''): - """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, - which must be a string of length 256""" - - if len(table) != 256: - raise ValueError("translation table must be 256 characters long") - - L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] - return ''.join(L) - -''', filename=__file__) - - -str_translate__String_ANY_ANY = app.interphook('str_translate__String_ANY_ANY') +def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''): + """charfilter - unicode handling is not implemented + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, + which must be a string of length 256""" + + # XXX CPython accepts buffers, too, not sure what we should do + table = space.str_w(w_table) + if len(table) != 256: + raise OperationError( + space.w_ValueError, + space.wrap("translation table must be 256 characters long")) + + string = w_string._value + chars = [] + for char in string: + w_char = W_StringObject.PREBUILT[ord(char)] + if not space.is_true(space.contains(w_deletechars, w_char)): + chars.append(table[ord(char)]) + return W_StringObject(''.join(chars)) def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ From cfbolz at codespeak.net Mon Nov 12 02:01:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 02:01:08 +0100 (CET) Subject: [pypy-svn] r48590 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071112010108.B87648263@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 02:01:08 2007 New Revision: 48590 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Log: tests are good: fix typos Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py Mon Nov 12 02:01:08 2007 @@ -914,11 +914,11 @@ def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ - unicode_from_string, encode_object + unicode_from_string, decode_object encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None and errors is None: return unicode_from_string(space, w_string) - return encode_object(space, w_string, encoding, errors) + return decode_object(space, w_string, encoding, errors) def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): #import pdb; pdb.set_trace() Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Mon Nov 12 02:01:08 2007 @@ -172,7 +172,7 @@ space.type(w_retval).getname(space, '?'))) return w_retval -def decode_object(space, w_object, encoding, errors): +def decode_object(space, w_obj, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: encoding = getdefaultencoding(space) From cfbolz at codespeak.net Mon Nov 12 03:25:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 03:25:44 +0100 (CET) Subject: [pypy-svn] r48591 - in pypy/branch/more-unicode-improvements/pypy/module/_codecs: . test Message-ID: <20071112022544.C24548267@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 03:25:43 2007 New Revision: 48591 Added: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (contents, props changed) Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py Log: start moving some bits of the _codecs module to interplevel Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py Mon Nov 12 03:25:43 2007 @@ -13,15 +13,11 @@ 'escape_encode' : 'app_codecs.escape_encode', 'latin_1_decode' : 'app_codecs.latin_1_decode', 'latin_1_encode' : 'app_codecs.latin_1_encode', - 'lookup' : 'app_codecs.lookup', - 'lookup_error' : 'app_codecs.lookup_error', 'mbcs_decode' : 'app_codecs.mbcs_decode', 'mbcs_encode' : 'app_codecs.mbcs_encode', 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', 'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode', 'readbuffer_encode' : 'app_codecs.readbuffer_encode', - 'register' : 'app_codecs.register', - 'register_error' : 'app_codecs.register_error', 'unicode_escape_decode' : 'app_codecs.unicode_escape_decode', 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', @@ -37,8 +33,21 @@ 'utf_7_encode' : 'app_codecs.utf_7_encode', 'utf_8_decode' : 'app_codecs.utf_8_decode', 'utf_8_encode' : 'app_codecs.utf_8_encode', - 'encode': 'app_codecs.encode', - 'decode': 'app_codecs.decode' + '_register_existing_errors': 'app_codecs._register_existing_errors', } interpleveldefs = { + 'encode': 'interp_codecs.encode', + 'decode': 'interp_codecs.decode', + 'lookup': 'interp_codecs.lookup_codec', + 'lookup_error': 'interp_codecs.lookup_error', + 'register': 'interp_codecs.register_codec', + 'register_error': 'interp_codecs.register_error', } + + def setup_after_space_initialization(self): + "NOT_RPYTHON" + self.space.appexec([], """(): + import _codecs + _codecs._register_existing_errors() + del _codecs._register_existing_errors + """) Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Mon Nov 12 03:25:43 2007 @@ -41,100 +41,6 @@ #from unicodecodec import * import sys -#/* --- Registry ----------------------------------------------------------- */ -codec_search_path = [] -codec_search_cache = {} -codec_error_registry = {} -codec_need_encodings = [True] - -def codec_register( search_function ): - """register(search_function) - - Register a codec search function. Search functions are expected to take - one argument, the encoding name in all lower case letters, and return - a tuple of functions (encoder, decoder, stream_reader, stream_writer). - """ - - if callable(search_function): - codec_search_path.append(search_function) - -register = codec_register - -def codec_lookup(encoding): - """lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) - Looks up a codec tuple in the Python codec registry and returns - a tuple of functions. - """ - if not isinstance(encoding, str): - raise TypeError("Encoding must be a string") - normalized_encoding = encoding.replace(" ", "-").lower() - result = codec_search_cache.get(normalized_encoding, None) - if not result: - if codec_need_encodings: - import encodings - if len(codec_search_path) == 0: - raise LookupError("no codec search functions registered: can't find encoding") - del codec_need_encodings[:] - for search in codec_search_path: - result = search(normalized_encoding) - if result: - if not (type(result) == tuple and len(result) == 4): - raise TypeError("codec search functions must return 4-tuples") - else: - codec_search_cache[normalized_encoding] = result - return result - if not result: - raise LookupError("unknown encoding: %s" % encoding) - return result - - -lookup = codec_lookup - -def encode(v, encoding=None, errors='strict'): - """encode(obj, [encoding[,errors]]) -> object - - Encodes obj using the codec registered for encoding. encoding defaults - to the default encoding. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a ValueError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with - codecs.register_error that can handle ValueErrors. - """ - if encoding == None: - encoding = sys.getdefaultencoding() - if isinstance(encoding, str): - encoder = lookup(encoding)[0] - if encoder and isinstance(errors, str): - res = encoder(v, errors) - return res[0] - else: - raise TypeError("Errors must be a string") - else: - raise TypeError("Encoding must be a string") - -def decode(obj, encoding=None, errors='strict'): - """decode(obj, [encoding[,errors]]) -> object - - Decodes obj using the codec registered for encoding. encoding defaults - to the default encoding. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a ValueError. Other possible values are 'ignore' and 'replace' - as well as any other name registerd with codecs.register_error that is - able to handle ValueErrors. - """ - if encoding == None: - encoding = sys.getdefaultencoding() - if isinstance(encoding, str): - decoder = lookup(encoding)[1] - if decoder and isinstance(errors, str): - res = decoder(obj, errors) - if not isinstance(res, tuple) or len(res) != 2: - raise TypeError("encoder must return a tuple (object, integer)") - return res[0] - else: - raise TypeError("Errors must be a string") - else: - raise TypeError("Encoding must be a string") def latin_1_encode( obj, errors='strict'): """None @@ -500,6 +406,14 @@ raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) +def _register_existing_errors(): + import _codecs + _codecs.register_error("strict", strict_errors) + _codecs.register_error("ignore", ignore_errors) + _codecs.register_error("replace", replace_errors) + _codecs.register_error("xmlcharrefreplace", xmlcharrefreplace_errors) + _codecs.register_error("backslashreplace", backslashreplace_errors) + # ---------------------------------------------------------------------- ##import sys @@ -531,38 +445,7 @@ unicode_latin1 = [None]*256 -def lookup_error(errors): - """lookup_error(errors) -> handler - - Return the error handler for the specified error handling name - or raise a LookupError, if no handler exists under this name. - """ - - try: - err_handler = codec_error_registry[errors] - except KeyError: - raise LookupError("unknown error handler name %s"%errors) - return err_handler - -def register_error(errors, handler): - """register_error(errors, handler) - - Register the specified error handler under the name - errors. handler must be a callable object, that - will be called with an exception instance containing - information about the location of the encoding/decoding - error and must return a (replacement, new position) tuple. - """ - if callable(handler): - codec_error_registry[errors] = handler - else: - raise TypeError("handler must be callable") -register_error("strict", strict_errors) -register_error("ignore", ignore_errors) -register_error("replace", replace_errors) -register_error("xmlcharrefreplace", xmlcharrefreplace_errors) -register_error("backslashreplace", backslashreplace_errors) def SPECIAL(c, encodeO, encodeWS): c = ord(c) @@ -1032,7 +915,8 @@ def unicode_call_errorhandler(errors, encoding, reason, input, startinpos, endinpos, decode=True): - errorHandler = lookup_error(errors) + import _codecs + errorHandler = _codecs.lookup_error(errors) if decode: exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) else: Added: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- (empty file) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Mon Nov 12 03:25:43 2007 @@ -0,0 +1,147 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped +from pypy.interpreter.baseobjspace import W_Root + +class CodecState(object): + def __init__(self, space): + self.codec_search_path = [] + self.codec_search_cache = {} + self.codec_error_registry = {} + self.codec_need_encodings = True + +def register_codec(space, w_search_function): + """register(search_function) + + Register a codec search function. Search functions are expected to take + one argument, the encoding name in all lower case letters, and return + a tuple of functions (encoder, decoder, stream_reader, stream_writer). + """ + #import pdb; pdb.set_trace() + state = space.fromcache(CodecState) + if space.is_true(space.callable(w_search_function)): + state.codec_search_path.append(w_search_function) + else: + raise OperationError( + space.w_TypeError, + space.wrap("argument must be callable")) +register_codec.unwrap_spec = [ObjSpace, W_Root] + + +def lookup_codec(space, encoding): + """lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) + Looks up a codec tuple in the Python codec registry and returns + a tuple of functions. + """ + #import pdb; pdb.set_trace() + state = space.fromcache(CodecState) + normalized_encoding = encoding.replace(" ", "-").lower() + w_result = state.codec_search_cache.get(normalized_encoding, None) + if w_result is not None: + return w_result + if state.codec_need_encodings: + w_import = space.getattr(space.builtin, space.wrap("__import__")) + # registers new codecs + space.call_function(w_import, space.wrap("encodings")) + state.codec_need_encodings = False + if len(state.codec_search_path) == 0: + raise OperationError( + space.w_LookupError, + space.wrap("no codec search functions registered: " + "can't find encoding")) + for w_search in state.codec_search_path: + w_result = space.call_function(w_search, + space.wrap(normalized_encoding)) + if not space.is_w(w_result, space.w_None): + if not (space.is_true(space.is_(space.type(w_result), + space.w_tuple)) and + space.int_w(space.len(w_result)) == 4): + raise OperationError( + space.w_TypeError, + space.wrap("codec search functions must return 4-tuples")) + else: + state.codec_search_cache[normalized_encoding] = w_result + return w_result + raise OperationError( + space.w_LookupError, + space.wrap("unknown encoding: %s" % encoding)) +lookup_codec.unwrap_spec = [ObjSpace, str] + + +def lookup_error(space, errors): + """lookup_error(errors) -> handler + + Return the error handler for the specified error handling name + or raise a LookupError, if no handler exists under this name. + """ + + state = space.fromcache(CodecState) + try: + w_err_handler = state.codec_error_registry[errors] + except KeyError: + raise OperationError( + space.w_LookupError, + space.wrap("unknown error handler name %s" % errors)) + return w_err_handler +lookup_error.unwrap_spec = [ObjSpace, str] + + +def encode(space, w_obj, encoding=NoneNotWrapped, errors='strict'): + """encode(obj, [encoding[,errors]]) -> object + + Encodes obj using the codec registered for encoding. encoding defaults + to the default encoding. errors may be given to set a different error + handling scheme. Default is 'strict' meaning that encoding errors raise + a ValueError. Other possible values are 'ignore', 'replace' and + 'xmlcharrefreplace' as well as any other name registered with + codecs.register_error that can handle ValueErrors. + """ + if encoding is None: + encoding = space.sys.defaultencoding + w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0)) + if space.is_true(w_encoder): + w_res = space.call_function(w_encoder, w_obj, space.wrap(errors)) + return space.getitem(w_res, space.wrap(0)) + else: + assert 0, "XXX, what to do here?" +encode.unwrap_spec = [ObjSpace, W_Root, str, str] + +def decode(space, w_obj, encoding=NoneNotWrapped, errors='strict'): + """decode(obj, [encoding[,errors]]) -> object + + Decodes obj using the codec registered for encoding. encoding defaults + to the default encoding. errors may be given to set a different error + handling scheme. Default is 'strict' meaning that encoding errors raise + a ValueError. Other possible values are 'ignore' and 'replace' + as well as any other name registerd with codecs.register_error that is + able to handle ValueErrors. + """ + if encoding is None: + encoding = sys.getdefaultencoding() + w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1)) + if space.is_true(w_decoder): + w_res = space.call_function(w_decoder, w_obj, space.wrap(errors)) + if (not space.is_true(space.isinstance(w_res, space.w_tuple)) + or space.int_w(space.len(w_res)) != 2): + raise TypeError("encoder must return a tuple (object, integer)") + return space.getitem(w_res, space.wrap(0)) + else: + assert 0, "XXX, what to do here?" +decode.unwrap_spec = [ObjSpace, W_Root, str, str] + +def register_error(space, errors, w_handler): + """register_error(errors, handler) + + Register the specified error handler under the name + errors. handler must be a callable object, that + will be called with an exception instance containing + information about the location of the encoding/decoding + error and must return a (replacement, new position) tuple. + """ + state = space.fromcache(CodecState) + if space.is_true(space.callable(w_handler)): + state.codec_error_registry[errors] = w_handler + else: + raise OperationError( + space.w_TypeError, + space.wrap("handler must be callable")) +register_error.unwrap_spec = [ObjSpace, str, W_Root] Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py Mon Nov 12 03:25:43 2007 @@ -6,6 +6,10 @@ space = gettestobjspace(usemodules=('unicodedata',)) cls.space = space + def test_register_noncallable(self): + import _codecs + raises(TypeError, _codecs.register, 1) + def test_bigU_codecs(self): import sys oldmaxunicode = sys.maxunicode From fijal at codespeak.net Mon Nov 12 13:44:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Nov 2007 13:44:05 +0100 (CET) Subject: [pypy-svn] r48593 - pypy/extradoc/talk/pycon2008 Message-ID: <20071112124405.678C482A1@code0.codespeak.net> Author: fijal Date: Mon Nov 12 13:44:05 2007 New Revision: 48593 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Another refactoring of proxy abstract Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Mon Nov 12 13:44:05 2007 @@ -1,4 +1,5 @@ XXX Understanding PyPy, and How It Let's You Do Things You Have Only Dreamed About. (*) +XXX The PyPy way and the tales of nice features and permissive(?) design ================================================================================ (* for particularly nerdy dreams :-) ) @@ -15,7 +16,7 @@ you mean, but "create transparent object" is a bit vague) * the PyPy Sandbox, which allows to run non-stripped version of - python interpreter in a controlled environment with custom + python interpreter in a controlled environment with a custom safety policy. * The Taint Object Space, which allows programmers to make sure that sensitive @@ -23,23 +24,8 @@ * XXX anything else? lazy evaluation? -* Why our approach is "working by design", rather then "fix as long as - anyone complains", why it's only permitted by our design decisions and - abstraction levels. How our all of the mentioned above examples could - be reduce to simple "proxy operation" concept. - - 'working by design' means something else. it means 'we designed it - to work, which, of course, is true of everything that didn't happen - by accident (and maybe that too, if some people's ideas of God are - are correct). - -* _why_ design matters. _when_ design matters. Some thoughts on when - incremental improvements are not the answer, and why successive - improvements in painting things red will not turn your children's - wagon into a firetruck, despite being the perfect approach to the - problem of 'making what I have red'. - -I am getting all these weird ideas of how a caterpillar becomes a -larger caterpillar is not the same way a caterpillar becomes a -butterfly. Either this is a useful analogy, or I am nuts. - +* Why our approach of doing pervasive changes to semantics, does not + require pervasive changes to the interpreter core all over the place. + Why our design allows us to describe broad class of such changes + as a "proxying operation" and finally, why design matters when you want + to get butterfly out of the caterpillar rather than bigger caterpillar. From cfbolz at codespeak.net Mon Nov 12 15:06:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 15:06:28 +0100 (CET) Subject: [pypy-svn] r48595 - pypy/extradoc/talk/pycon2008 Message-ID: <20071112140628.5317E8293@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 15:06:26 2007 New Revision: 48595 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: mostly typo-level changes. tried to make some sentences more english-sounding, not sure it worked. Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Mon Nov 12 15:06:26 2007 @@ -4,28 +4,27 @@ (* for particularly nerdy dreams :-) ) In this talk, we would like to present our paradigm-shifting architecture -for dynamic langauges and some features which, due to our appraoch -are exclusively available with PyPy. +for dynamic languages and some features which are exclusively available in +PyPy, due to our novel approach. -Topics would include: +Topics include: * The transparent proxy, which allows programmers to create - transparent object of any type in python (including built-in + indistinguishable proxies for any type in Python (including built-in python types like frames), with various usage showcases including - our completely transparent remote access scheme. (XXX: I know what - you mean, but "create transparent object" is a bit vague) + our completely transparent remote access scheme. -* the PyPy Sandbox, which allows to run non-stripped version of - python interpreter in a controlled environment with a custom +* the PyPy Sandbox, which allows to run a non-stripped version of + our Python interpreter in a controlled environment with a custom safety policy. * The Taint Object Space, which allows programmers to make sure that sensitive - information does not cross the I/O barrier. + information does not cross I/O barriers. -* XXX anything else? lazy evaluation? +* XXX anything else? lazy evaluation? Is a bit too much, I think -* Why our approach of doing pervasive changes to semantics, does not - require pervasive changes to the interpreter core all over the place. - Why our design allows us to describe broad class of such changes +* Why our approach of doing pervasive changes to semantics does not + require pervasive changes to the interpreter core. + Why our design allows us to describe a broad class of such changes as a "proxying operation" and finally, why design matters when you want - to get butterfly out of the caterpillar rather than bigger caterpillar. + to get a butterfly out of the caterpillar instead of a bigger caterpillar. From fijal at codespeak.net Mon Nov 12 15:13:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Nov 2007 15:13:18 +0100 (CET) Subject: [pypy-svn] r48596 - pypy/extradoc/talk/pycon2008 Message-ID: <20071112141318.34201829A@code0.codespeak.net> Author: fijal Date: Mon Nov 12 15:13:17 2007 New Revision: 48596 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Kill "anything else" clause Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Mon Nov 12 15:13:17 2007 @@ -21,8 +21,6 @@ * The Taint Object Space, which allows programmers to make sure that sensitive information does not cross I/O barriers. -* XXX anything else? lazy evaluation? Is a bit too much, I think - * Why our approach of doing pervasive changes to semantics does not require pervasive changes to the interpreter core. Why our design allows us to describe a broad class of such changes From cfbolz at codespeak.net Mon Nov 12 15:17:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 15:17:04 +0100 (CET) Subject: [pypy-svn] r48597 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112141704.BFCC9829E@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 15:17:03 2007 New Revision: 48597 Added: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (contents, props changed) pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (contents, props changed) Log: start a new rlib library for unicode handling. so far it contains only an rpython utf-8 decoder Added: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- (empty file) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 15:17:03 2007 @@ -0,0 +1,154 @@ +import sys +from pypy.lang.smalltalk.tool.bitmanipulation import splitter + +MAXUNICODE = sys.maxunicode + + +def raise_unicode_exception(errors, encoding, msg, s, startingpos, endingpos, + decode=True): + if decode: + raise UnicodeDecodeError( + "%s can't decode byte %s in position %s: %s" % ( + encoding, s[startingpos], startingpos, msg)) + else: + XXX + +utf8_code_length = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 +] + +def str_decode_utf8(s, size, errors, final=False, + errorhandler=raise_unicode_exception): + if (size == 0): + return u'', 0 + p = [] + pos = 0 + while pos < size: + ch = s[pos] + ordch1 = ord(ch) + if ordch1 < 0x80: + p += unichr(ordch1) + pos += 1 + continue + + n = utf8_code_length[ordch1] + if (pos + n > size): + if not final: + break + else: + r, pos = errorhandler(errors, "utf8", + "unexpected end of data", s, pos, size) + p += r + if n == 0: + res = errorhandler(errors, "utf8", "unexpected code byte", + s, pos, pos + 1) + p += res[0] + pos = res[1] + elif n == 1: + assert 0, "you can never get here" + elif n == 2: + # 110yyyyy 10zzzzzz ====> 00000000 00000yyy yyzzzzzz + + ordch2 = ord(s[pos+1]) + z, two = splitter[6, 2](ordch2) + y, six = splitter[5, 3](ordch1) + assert six == 6 + if (two != 2): + r, pos = errorhandler(errors, "utf8", "invalid data", + s, pos, pos + 2) + p += r + else: + c = (y << 6) + z + if c < 0x80: + r, pos = errorhandler(errors, "utf8", "illegal encoding", + s, pos, pos + 2) + p += r + else: + p += unichr(c) + pos += n + elif n == 3: + # 1110xxxx 10yyyyyy 10zzzzzz ====> 00000000 xxxxyyyy yyzzzzzz + ordch2 = ord(s[pos+1]) + ordch3 = ord(s[pos+2]) + z, two1 = splitter[6, 2](ordch3) + y, two2 = splitter[6, 2](ordch2) + x, fourteen = splitter[4, 4](ordch1) + assert fourteen == 14 + if (two1 != 2 or two2 != 2): + r, pos = errorhandler(errors, "utf8", "invalid data", + s, pos, pos + 3) + p += r + else: + c = (x << 12) + (y << 6) + z + # Note: UTF-8 encodings of surrogates are considered + # legal UTF-8 sequences; + # XXX For wide builds (UCS-4) we should probably try + # to recombine the surrogates into a single code + # unit. + if c < 0x0800: + r, pos = errorhandler(errors, "utf8", "illegal encoding", + s, pos, pos + 3) + p += r + else: + p += unichr(c) + pos += n + elif n == 4: + # 11110www 10xxxxxx 10yyyyyy 10zzzzzz ====> + # 000wwwxx xxxxyyyy yyzzzzzz + ordch2 = ord(s[pos+1]) + ordch3 = ord(s[pos+2]) + ordch4 = ord(s[pos+3]) + z, two1 = splitter[6, 2](ordch4) + y, two2 = splitter[6, 2](ordch3) + x, two3 = splitter[6, 2](ordch2) + w, thirty = splitter[3, 5](ordch1) + assert thirty == 30 + if (two1 != 2 or two2 != 2 or two3 != 2): + r, pos = errorhandler(errors, "utf8", "invalid data", + s, pos, pos + 4) + p += r + else: + c = (w << 18) + (x << 12) + (y << 6) + z + # minimum value allowed for 4 byte encoding + # maximum value allowed for UTF-16 + if ((c < 0x10000) or (c > 0x10ffff)): + r, pos = errorhandler(errors, "utf8", "illegal encoding", + s, pos, pos + 4) + p += r + else: + # convert to UTF-16 if necessary + if c < MAXUNICODE: + p.append(unichr(c)) + else: + # compute and append the two surrogates: + # translate from 10000..10FFFF to 0..FFFF + c -= 0x10000 + # high surrogate = top 10 bits added to D800 + p.append(unichr(0xD800 + (c >> 10))) + # low surrogate = bottom 10 bits added to DC00 + p.append(unichr(0xDC00 + (c & 0x03FF))) + pos += n + else: + r, pos = errorhandler(errors, "utf8", + "unsupported Unicode code range", + s, pos, pos + n) + p += r + + return u"".join(p), pos + + Added: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- (empty file) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 15:17:03 2007 @@ -0,0 +1,23 @@ +from pypy.rlib import runicode + +class UnicodeTests(object): + def typeequals(self, x, y): + assert x == y + assert type(x) is type(y) + + def checkdecode(self, s, encoding): + decoder = getattr(runicode, "str_decode_%s" % encoding) + trueresult = s.decode(encoding) + result, consumed = decoder(s, len(s), True) + assert consumed == len(s) + self.typeequals(trueresult, result) + +class TestDecoding(UnicodeTests): + + def test_all_ascii(self): + for i in range(128): + self.checkdecode(chr(i), "utf8") + + def test_single_chars(self): + for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: + self.checkdecode(s, "utf8") From cfbolz at codespeak.net Mon Nov 12 15:29:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 15:29:44 +0100 (CET) Subject: [pypy-svn] r48598 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112142944.5052582A9@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 15:29:42 2007 New Revision: 48598 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: decoding latin1 is simple Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 15:29:42 2007 @@ -152,3 +152,11 @@ return u"".join(p), pos +def str_decode_latin1(s, size, errors, final=False, + errorhandler=raise_unicode_exception): + pos = 0 + p = [] + while (pos < size): + p += unichr(ord(s[pos])) + pos += 1 + return u"".join(p), pos Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 15:29:42 2007 @@ -7,17 +7,29 @@ def checkdecode(self, s, encoding): decoder = getattr(runicode, "str_decode_%s" % encoding) - trueresult = s.decode(encoding) + if isinstance(s, str): + trueresult = s.decode(encoding) + else: + trueresult = s + s = s.encode(encoding) result, consumed = decoder(s, len(s), True) assert consumed == len(s) self.typeequals(trueresult, result) + class TestDecoding(UnicodeTests): def test_all_ascii(self): for i in range(128): - self.checkdecode(chr(i), "utf8") + for encoding in "utf8 latin1".split(): + self.checkdecode(chr(i), encoding) + + def test_all_first_256(self): + for i in range(256): + for encoding in "utf8 latin1".split(): + self.checkdecode(unichr(i), encoding) - def test_single_chars(self): + def test_single_chars_utf8(self): for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkdecode(s, "utf8") + From cfbolz at codespeak.net Mon Nov 12 15:36:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 15:36:22 +0100 (CET) Subject: [pypy-svn] r48599 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112143622.E685682AB@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 15:36:22 2007 New Revision: 48599 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: ascii is simple too Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 15:36:22 2007 @@ -154,9 +154,27 @@ def str_decode_latin1(s, size, errors, final=False, errorhandler=raise_unicode_exception): + # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 p = [] while (pos < size): p += unichr(ord(s[pos])) pos += 1 return u"".join(p), pos + + +def str_decode_ascii(s, size, errors, final=False, + errorhandler=raise_unicode_exception): + # ASCII is equivalent to the first 128 ordinals in Unicode. + p = [] + pos = 0 + while pos < len(s): + c = s[pos] + if ord(c) < 128: + p += unichr(ord(c)) + pos += 1 + else: + r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)", + s, pos, pos + 1) + p += r + return u"".join(p), pos Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 15:36:22 2007 @@ -21,7 +21,7 @@ def test_all_ascii(self): for i in range(128): - for encoding in "utf8 latin1".split(): + for encoding in "utf8 latin1 ascii".split(): self.checkdecode(chr(i), encoding) def test_all_first_256(self): From fijal at codespeak.net Mon Nov 12 16:21:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Nov 2007 16:21:17 +0100 (CET) Subject: [pypy-svn] r48600 - in pypy/dist/pypy: rpython rpython/lltypesystem/module rpython/lltypesystem/module/test translator/c translator/c/src Message-ID: <20071112152117.E353E82A5@code0.codespeak.net> Author: fijal Date: Mon Nov 12 16:21:16 2007 New Revision: 48600 Added: pypy/dist/pypy/rpython/lltypesystem/module/test/ pypy/dist/pypy/rpython/lltypesystem/module/test/__init__.py (contents, props changed) pypy/dist/pypy/rpython/lltypesystem/module/test/test_ll_math.py (contents, props changed) Removed: pypy/dist/pypy/translator/c/src/ll_math.h Modified: pypy/dist/pypy/rpython/extfuncregistry.py pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py pypy/dist/pypy/translator/c/extfunc.py pypy/dist/pypy/translator/c/src/g_include.h Log: Move ll_math to rffi. Modified: pypy/dist/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extfuncregistry.py (original) +++ pypy/dist/pypy/rpython/extfuncregistry.py Mon Nov 12 16:21:16 2007 @@ -1,5 +1,6 @@ -# this registry use the new interface for external functions -# all the above declarations in extfunctable should be moved here at some point. +# this registry uses the new interface for external functions +# all the above declarations in extfunctable should be moved here +# at some point. from extfunc import register_external @@ -20,24 +21,18 @@ # the following functions all take one float, return one float # and are part of math.h -simple_math_functions = [ - 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', - 'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh' - ] -for name in simple_math_functions: +for name in ll_math.unary_math_functions: + llimpl = getattr(ll_math, 'll_math_%s' % name, None) register_external(getattr(math, name), [float], float, - "ll_math.ll_math_%s" % name, - sandboxsafe=True) + export_name="ll_math.ll_math_%s" % name, + sandboxsafe=True, llimpl=llimpl) complex_math_functions = [ ('frexp', [float], (float, int)), - ('atan2', [float, float], float), - ('fmod', [float, float], float), ('ldexp', [float, int], float), ('modf', [float], (float, float)), - ('hypot', [float, float], float), - ('pow', [float, float], float), - ] + ] + [(name, [float, float], float) for name in + ll_math.binary_math_functions] for name, args, res in complex_math_functions: func = getattr(math, name) Modified: pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py Mon Nov 12 16:21:16 2007 @@ -1,10 +1,23 @@ import math +import errno +import py from pypy.rpython.lltypesystem import lltype, rffi +from pypy.tool.sourcetools import func_with_new_name math_frexp = rffi.llexternal('frexp', [rffi.DOUBLE, rffi.INTP], rffi.DOUBLE, sandboxsafe=True) math_modf = rffi.llexternal('modf', [rffi.DOUBLE, rffi.DOUBLEP], rffi.DOUBLE, sandboxsafe=True) +math_ldexp = rffi.llexternal('ldexp', [rffi.DOUBLE, rffi.INT], rffi.DOUBLE) + +unary_math_functions = [ + 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', + 'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh' + ] + +binary_math_functions = [ + 'atan2', 'fmod', 'hypot', 'pow' + ] def ll_math_frexp(x): exp_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') @@ -19,3 +32,57 @@ intpart = intpart_p[0] lltype.free(intpart_p, flavor='raw') return (fracpart, intpart) + +def ll_math_ldexp(x, exp): + _error_reset() + r = math_ldexp(x, exp) + _check_error(r) + return r + +def _error_reset(): + rffi.set_errno(0) + +ERANGE = errno.ERANGE +def _check_error(x): + errno = rffi.get_errno() + if errno: + if errno == ERANGE: + if not x: + raise OSError + raise OverflowError("math range error") + else: + raise ValueError("math domain error") + +def new_unary_math_function(name): + c_func = rffi.llexternal(name, [rffi.DOUBLE], rffi.DOUBLE, + sandboxsafe=True, libraries=['m']) + + def ll_math(x): + _error_reset() + r = c_func(x) + _check_error(r) + return r + + return func_with_new_name(ll_math, 'll_math_' + name) + +def new_binary_math_function(name): + c_func = rffi.llexternal(name, [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE, + sandboxsafe=True, libraries=['m']) + + def ll_math(x, y): + _error_reset() + r = c_func(x, y) + _check_error(r) + return r + + return func_with_new_name(ll_math, 'll_math_' + name) + +# the two above are almost the same, but they're C-c C-v not to go mad +# with meta-programming + +for name in unary_math_functions: + globals()['ll_math_' + name] = new_unary_math_function(name) + +for name in binary_math_functions: + globals()['ll_math_' + name] = new_binary_math_function(name) + Added: pypy/dist/pypy/rpython/lltypesystem/module/test/__init__.py ============================================================================== Added: pypy/dist/pypy/rpython/lltypesystem/module/test/test_ll_math.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/lltypesystem/module/test/test_ll_math.py Mon Nov 12 16:21:16 2007 @@ -0,0 +1,44 @@ + +""" Just another bunch of tests for llmath, run on top of llinterp +""" + +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin +from pypy.rpython.lltypesystem.module import ll_math +import math + +# XXX no OORtypeMixin here + +class TestMath(BaseRtypingTest, LLRtypeMixin): + def new_unary_test(name): + def next_test(self): + def f(x): + return getattr(math, name)(x) + assert self.interpret(f, [0.3]) == f(0.3) + return next_test + + def new_binary_test(name): + def next_test(self): + def f(x, y): + return getattr(math, name)(x, y) + assert self.interpret(f, [0.3, 0.4]) == f(0.3, 0.4) + return next_test + + for name in ll_math.unary_math_functions: + func_name = 'test_%s' % (name,) + next_test = new_unary_test(name) + next_test.func_name = func_name + locals()[func_name] = next_test + del next_test + + for name in ll_math.binary_math_functions: + func_name = 'test_%s' % (name,) + next_test = new_binary_test(name) + next_test.func_name = func_name + locals()[func_name] = next_test + del next_test + + def test_ldexp(self): + def f(x, y): + return math.ldexp(x, y) + + assert self.interpret(f, [3.4, 2]) == f(3.4, 2) Modified: pypy/dist/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/dist/pypy/translator/c/extfunc.py Mon Nov 12 16:21:16 2007 @@ -25,26 +25,6 @@ } #______________________________________________________ -# insert 'simple' math functions into EXTERNALs table: - -# XXX: messy, messy, messy -# this interacts in strange ways with node.select_function_code_generators, -# because it fakes to be an ll_* function. - -math_functions = [ - 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', - 'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', - 'pow', 'atan2', 'fmod', 'ldexp', 'hypot' - ] -# frexp and modf have been ported to the new rffi style already - -import math -for name in math_functions: - EXTERNALS['ll_math.ll_math_%s' % name] = 'LL_math_%s' % name - -EXTERNALS['LL_flush_icache'] = 'LL_flush_icache' - -#______________________________________________________ def find_list_of_str(rtyper): for r in rtyper.reprs.itervalues(): @@ -53,7 +33,6 @@ return None def predeclare_common_types(db, rtyper): - from pypy.rpython.lltypesystem.module import ll_math # Common types yield ('RPyString', STR) LIST_OF_STR = find_list_of_str(rtyper) Modified: pypy/dist/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_include.h (original) +++ pypy/dist/pypy/translator/c/src/g_include.h Mon Nov 12 16:21:16 2007 @@ -53,7 +53,6 @@ # include "src/rtyper.h" #ifndef AVR # include "src/ll_os.h" -# include "src/ll_math.h" # include "src/ll_strtod.h" # ifdef RPyExc_thread_error # include "src/ll_thread.h" From cfbolz at codespeak.net Mon Nov 12 16:21:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 16:21:22 +0100 (CET) Subject: [pypy-svn] r48601 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112152122.12F8C82AB@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 16:21:22 2007 New Revision: 48601 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: add utf16 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 16:21:22 2007 @@ -45,7 +45,7 @@ p += unichr(ordch1) pos += 1 continue - + n = utf8_code_length[ordch1] if (pos + n > size): if not final: @@ -67,7 +67,7 @@ ordch2 = ord(s[pos+1]) z, two = splitter[6, 2](ordch2) y, six = splitter[5, 3](ordch1) - assert six == 6 + assert six == 6 if (two != 2): r, pos = errorhandler(errors, "utf8", "invalid data", s, pos, pos + 2) @@ -152,6 +152,118 @@ return u"".join(p), pos +def str_decode_utf16(s, size, errors, final=True, + errorhandler=raise_unicode_exception): + result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, + errorhandler, "native") + return result, length + +def str_decode_utf16be(s, size, errors, final=True, + errorhandler=raise_unicode_exception): + result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, + errorhandler, "big") + return result, length + +def str_decode_utf16le(s, size, errors, final=True, + errorhandler=raise_unicode_exception): + result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, + errorhandler, "little") + return result, length + +def str_decode_utf16_helper(s, size, errors, final=True, + errorhandler=raise_unicode_exception, + byteorder="native"): + + bo = 0 + consumed = 0 + + if sys.byteorder == 'little': + ihi = 1 + ilo = 0 + else: + ihi = 0 + ilo = 1 + + # Check for BOM marks (U+FEFF) in the input and adjust current + # byte order setting accordingly. In native mode, the leading BOM + # mark is skipped, in all other modes, it is copied to the output + # stream as-is (giving a ZWNBSP character). + q = 0 + p = [] + if byteorder == 'native': + if (size >= 2): + bom = (ord(s[ihi]) << 8) | ord(s[ilo]) + if sys.byteorder == 'little': + if (bom == 0xFEFF): + q += 2 + bo = -1 + elif bom == 0xFFFE: + q += 2 + bo = 1 + else: + if bom == 0xFEFF: + q += 2 + bo = 1 + elif bom == 0xFFFE: + q += 2 + bo = -1 + elif byteorder == 'little': + bo = -1 + else: + bo = 1 + if (size == 0): + return u'', 0, bo + if (bo == -1): + # force little endian + ihi = 1 + ilo = 0 + + elif (bo == 1): + # force big endian + ihi = 0 + ilo = 1 + + #XXX I think the errors are not correctly handled here + while (q < len(s)): + # remaining bytes at the end? (size should be even) + if len(s) - q < 2: + if not final: + break + errmsg = "truncated data" + startinpos = q + endinpos = len(s) + errorhandler(errors, 'utf-16', "truncated data", + s, startinpos, endinpos, True) + # CPython ignores the remaining input chars if the callback + # chooses to skip the input. XXX is this sensible? + ch = (ord(s[q + ihi]) << 8) | ord(s[q + ilo]) + q += 2 + if (ch < 0xD800 or ch > 0xDFFF): + p += unichr(ch) + continue + # UTF-16 code pair: + if (q >= len(s)): + errmsg = "unexpected end of data" + errorhandler(errors, 'utf-16', errmsg, s, q - 2, len(s)) + elif (0xD800 <= ch and ch <= 0xDBFF): + ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) + q += 2 + if (0xDC00 <= ch2 and ch2 <= 0xDFFF): + if MAXUNICODE < 65536: + p += unichr(ch) + p += unichr(ch2) + else: + p += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) + continue + else: + errmsg = "illegal UTF-16 surrogate" + errorhandler(errors, 'utf-16', errmsg, s, q - 4, q - 2) + errmsg = "illegal encoding" + startinpos = q-2 + endinpos = startinpos+2 + errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) + return u"".join(p), q, bo + def str_decode_latin1(s, size, errors, final=False, errorhandler=raise_unicode_exception): # latin1 is equivalent to the first 256 ordinals in Unicode. Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 16:21:22 2007 @@ -1,3 +1,4 @@ +import sys, random from pypy.rlib import runicode class UnicodeTests(object): @@ -6,7 +7,7 @@ assert type(x) is type(y) def checkdecode(self, s, encoding): - decoder = getattr(runicode, "str_decode_%s" % encoding) + decoder = getattr(runicode, "str_decode_%s" % encoding.replace("-", "")) if isinstance(s, str): trueresult = s.decode(encoding) else: @@ -26,7 +27,13 @@ def test_all_first_256(self): for i in range(256): - for encoding in "utf8 latin1".split(): + for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): + self.checkdecode(unichr(i), encoding) + + def test_random(self): + for i in range(10000): + uni = unichr(random.randrange(sys.maxunicode)) + for encoding in "utf8 utf16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) def test_single_chars_utf8(self): From fijal at codespeak.net Mon Nov 12 16:45:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Nov 2007 16:45:03 +0100 (CET) Subject: [pypy-svn] r48602 - pypy/extradoc/talk/pycon2008 Message-ID: <20071112154503.D22AE82AE@code0.codespeak.net> Author: fijal Date: Mon Nov 12 16:45:03 2007 New Revision: 48602 Added: pypy/extradoc/talk/pycon2008/tutorial.txt (contents, props changed) Log: Copy tutorial outline from pycon web page Added: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Mon Nov 12 16:45:03 2007 @@ -0,0 +1,12 @@ +* Tutorial title. +* Presenter name(s) & contact information (email, telephone). +* Intended audience (difficulty level, experience required): non-programmers, beginning programmers, advanced users, CPython developers, etc. +* Tutorial format description (lecture, interactive lecture, Q&A session [master class], hands-on, etc.). Be specific. +* Recording: indicate if you are amenable to your tutorial being recorded. "I give permission to record and publish my PyCon tutorial for free distribution." +* Requirements: A list of software and hardware requirements for the tutorial. (Laptop required or recommended? Python installed? Which version? 3rd-party packages required? Provide links & versions.) +* Notes for reviewers. +* Promotional summary (max. 100 words). +* Detailed tutorial outline, for review (include timing). +* Tutorial outline for website publication, if different from the above. +* Presenter bio (one paragraph), written in the third person ("Presenter Name is..."). Include where you live, your job, your projects, books you've written, volunteer activities, family, etc. +* Previous experience: references of presentation/teaching engagements (sessions taught at OSCON, EuroPython, local user groups, at work, etc.). Extra consideration will be given to presenters with prior experience. From cfbolz at codespeak.net Mon Nov 12 18:00:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 18:00:33 +0100 (CET) Subject: [pypy-svn] r48603 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112170033.1D59D82AA@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 18:00:33 2007 New Revision: 48603 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: port encoders to RPython Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 18:00:33 2007 @@ -11,7 +11,12 @@ "%s can't decode byte %s in position %s: %s" % ( encoding, s[startingpos], startingpos, msg)) else: - XXX + raise UnicodeEncodeError( + "%s can't encode byte %s in position %s: %s" % ( + encoding, s[startingpos], startingpos, msg)) + +# ____________________________________________________________ +# unicode decoding utf8_code_length = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -290,3 +295,144 @@ s, pos, pos + 1) p += r return u"".join(p), pos + + +# ____________________________________________________________ +# unicode encoding + + +def unicode_encode_utf8(s, size, errors, errorhandler=raise_unicode_exception): + assert(size >= 0) + p = [] + i = 0 + while i < size: + ch = s[i] + i += 1 + if (ord(ch) < 0x80): + # Encode ASCII + p += chr(ord(ch)) + elif (ord(ch) < 0x0800) : + # Encode Latin-1 + p += chr((0xc0 | (ord(ch) >> 6))) + p += chr((0x80 | (ord(ch) & 0x3f))) + else: + # Encode UCS2 Unicode ordinals + if (ord(ch) < 0x10000): + # Special case: check for high surrogate + if (0xD800 <= ord(ch) and ord(ch) <= 0xDBFF and i != size) : + ch2 = s[i] + # Check for low surrogate and combine the two to + # form a UCS4 value + if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : + ch3 = ((ord(ch) - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 + i += 1 + _encodeUCS4(p, ch) + continue + # Fall through: handles isolated high surrogates + p += (chr((0xe0 | (ord(ch) >> 12)))) + p += (chr((0x80 | ((ord(ch) >> 6) & 0x3f)))) + p += (chr((0x80 | (ord(ch) & 0x3f)))) + continue + else: + _encodeUCS4(p, ord(ch)) + return "".join(p) + +def _encodeUCS4(p, ch): + # Encode UCS4 Unicode ordinals + p += (chr((0xf0 | (ch >> 18)))) + p += (chr((0x80 | ((ch >> 12) & 0x3f)))) + p += (chr((0x80 | ((ch >> 6) & 0x3f)))) + p += (chr((0x80 | (ch & 0x3f)))) + + +def unicode_encode_ucs1_helper(p, size, errors, + errorhandler=raise_unicode_exception, limit=256): + + if limit == 256: + reason = "ordinal not in range(256)" + encoding = "latin-1" + else: + reason = "ordinal not in range(128)" + encoding = "ascii" + + if (size == 0): + return '' + res = [] + pos = 0 + while pos < len(p): + ch = p[pos] + + if ord(ch) < limit: + res += chr(ord(ch)) + pos += 1 + else: + # startpos for collecting unencodable chars + collstart = pos + collend = pos+1 + while collend < len(p) and ord(p[collend]) >= limit: + collend += 1 + x = errorhandler(errors, encoding, reason, p, + collstart, collend, False) + res += str(x[0]) + pos = x[1] + + return "".join(res) + +def unicode_encode_latin1(p, size, errors): + res = unicode_encode_ucs1_helper(p, size, errors, 256) + return res + +def unicode_encode_ascii(p, size, errors): + res = unicode_encode_ucs1_helper(p, size, errors, 128) + return res + + +def _STORECHAR(p, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + p.append(lo) + p.append(hi) + else: + p.append(hi) + p.append(lo) + +def unicode_encode_utf16_helper(s, size, errors, + errorhandler=raise_unicode_exception, + byteorder='little'): + p = [] + if (byteorder == 'native'): + _STORECHAR(p, 0xFEFF, sys.byteorder) + byteorder = sys.byteorder + + if size == 0: + return "" + + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + ch2 = 0 + if (ch >= 0x10000) : + ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) + ch = 0xD800 | ((ch-0x10000) >> 10) + + _STORECHAR(p, ch, byteorder) + if ch2: + _STORECHAR(p, ch2, byteorder) + + return "".join(p) + +def unicode_encode_utf16(s, size, errors, + errorhandler=raise_unicode_exception): + return unicode_encode_utf16_helper(s, size, errors, errorhandler, "native") + + +def unicode_encode_utf16be(s, size, errors, + errorhandler=raise_unicode_exception): + return unicode_encode_utf16_helper(s, size, errors, errorhandler, "big") + + +def unicode_encode_utf16le(s, size, errors, + errorhandler=raise_unicode_exception): + return unicode_encode_utf16_helper(s, size, errors, errorhandler, "little") Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 18:00:33 2007 @@ -17,9 +17,23 @@ assert consumed == len(s) self.typeequals(trueresult, result) + def checkencode(self, s, encoding): + encoder = getattr(runicode, + "unicode_encode_%s" % encoding.replace("-", "")) + if isinstance(s, unicode): + trueresult = s.encode(encoding) + else: + trueresult = s + s = s.decode(encoding) + result = encoder(s, len(s), True) + self.typeequals(trueresult, result) + class TestDecoding(UnicodeTests): + # XXX test bom recognition in utf-16 + # XXX test proper error handling + def test_all_ascii(self): for i in range(128): for encoding in "utf8 latin1 ascii".split(): @@ -30,6 +44,7 @@ for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) + def test_random(self): for i in range(10000): uni = unichr(random.randrange(sys.maxunicode)) @@ -40,3 +55,24 @@ for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkdecode(s, "utf8") + +class TestEncoding(UnicodeTests): + def test_all_ascii(self): + for i in range(128): + for encoding in "utf8 latin1 ascii".split(): + self.checkencode(unichr(i), encoding) + + def test_all_first_256(self): + for i in range(256): + for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): + self.checkencode(unichr(i), encoding) + + def test_random(self): + for i in range(10000): + uni = unichr(random.randrange(sys.maxunicode)) + for encoding in "utf8 utf16 utf-16-be utf-16-le".split(): + self.checkencode(unichr(i), encoding) + + def test_single_chars_utf8(self): + for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: + self.checkencode(s, "utf8") From cfbolz at codespeak.net Mon Nov 12 18:18:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 18:18:43 +0100 (CET) Subject: [pypy-svn] r48604 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112171843.B7BD382B0@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 18:18:42 2007 New Revision: 48604 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: add tests for encoding error handlers, fix problems found Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 18:18:42 2007 @@ -347,10 +347,9 @@ def unicode_encode_ucs1_helper(p, size, errors, errorhandler=raise_unicode_exception, limit=256): - if limit == 256: reason = "ordinal not in range(256)" - encoding = "latin-1" + encoding = "latin1" else: reason = "ordinal not in range(128)" encoding = "ascii" @@ -371,19 +370,18 @@ collend = pos+1 while collend < len(p) and ord(p[collend]) >= limit: collend += 1 - x = errorhandler(errors, encoding, reason, p, - collstart, collend, False) - res += str(x[0]) - pos = x[1] + r, pos = errorhandler(errors, encoding, reason, p, + collstart, collend, False) + res += r return "".join(res) -def unicode_encode_latin1(p, size, errors): - res = unicode_encode_ucs1_helper(p, size, errors, 256) +def unicode_encode_latin1(p, size, errors, errorhandler=raise_unicode_exception): + res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res -def unicode_encode_ascii(p, size, errors): - res = unicode_encode_ucs1_helper(p, size, errors, 128) +def unicode_encode_ascii(p, size, errors, errorhandler=raise_unicode_exception): + res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 18:18:42 2007 @@ -28,6 +28,23 @@ result = encoder(s, len(s), True) self.typeequals(trueresult, result) + def checkencodeerror(self, s, encoding, start, stop): + called = [False] + def errorhandler(errors, enc, msg, t, startingpos, + endingpos, decode): + called[0] = True + assert errors == "foo!" + assert enc == encoding + assert t is s + assert start == startingpos + assert stop == endingpos + assert not decode + return "42424242", stop + encoder = getattr(runicode, + "unicode_encode_%s" % encoding.replace("-", "")) + result = encoder(s, len(s), "foo!", errorhandler) + assert called[0] + assert "42424242" in result class TestDecoding(UnicodeTests): @@ -76,3 +93,9 @@ def test_single_chars_utf8(self): for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkencode(s, "utf8") + + def test_ascii_error(self): + self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) + + def test_latin1_error(self): + self.checkencodeerror(u"abc\uffff\uffff\uffffcde", "latin1", 3, 6) From cfbolz at codespeak.net Mon Nov 12 18:49:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 18:49:48 +0100 (CET) Subject: [pypy-svn] r48606 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112174948.6D15182A8@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 18:49:47 2007 New Revision: 48606 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: tests for decoding errors, fix a small bug in utf8 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 18:49:47 2007 @@ -59,6 +59,8 @@ r, pos = errorhandler(errors, "utf8", "unexpected end of data", s, pos, size) p += r + if (pos + n > size): + break if n == 0: res = errorhandler(errors, "utf8", "unexpected code byte", s, pos, pos + 1) Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 18:49:47 2007 @@ -46,6 +46,31 @@ assert called[0] assert "42424242" in result + def checkdecodeerror(self, s, encoding, start, stop, addstuff=True): + called = [0] + def errorhandler(errors, enc, msg, t, startingpos, + endingpos, decode=True): + called[0] += 1 + if called[0] == 1: + assert errors == "foo!" + assert enc == encoding + assert t is s + assert start == startingpos + assert stop == endingpos + assert decode + return u"42424242", stop + return "", endingpos + decoder = getattr(runicode, + "str_decode_%s" % encoding.replace("-", "")) + if addstuff: + s += "some rest in ascii" + result, _ = decoder(s, len(s), "foo!", True, errorhandler) + assert called[0] > 0 + assert "42424242" in result + if addstuff: + assert result.endswith(u"some rest in ascii") + + class TestDecoding(UnicodeTests): # XXX test bom recognition in utf-16 @@ -61,7 +86,6 @@ for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) - def test_random(self): for i in range(10000): uni = unichr(random.randrange(sys.maxunicode)) @@ -72,6 +96,28 @@ for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkdecode(s, "utf8") + def test_utf8_errors(self): + for s in [# unexpected end of data + "\xd7", "\xd6", "\xeb\x96", "\xf0\x90\x91"]: + self.checkdecodeerror(s, "utf8", 0, len(s), addstuff=False) + + for s in [# unexpected code byte + "\x81", "\xbf", + # invalid data 2 byte + "\xd7\x50", "\xd6\x06", "\xd6\xD6", + # invalid data 3 byte + "\xeb\x56\x95", "\xeb\x06\x95", "\xeb\xD6\x95", + "\xeb\x96\x55", "\xeb\x96\x05", "\xeb\x96\xD5", + # invalid data 4 byte + "\xf0\x50\x91\x93", "\xf0\x00\x91\x93", "\xf0\xd0\x91\x93", + "\xf0\x90\x51\x93", "\xf0\x90\x01\x93", "\xf0\x90\xd1\x93", + "\xf0\x90\x91\x53", "\xf0\x90\x91\x03", "\xf0\x90\x91\xd3", + ]: + self.checkdecodeerror(s, "utf8", 0, len(s), addstuff=True) + + def test_ascii_error(self): + self.checkdecodeerror("abc\xFF\xFF\xFFcde", "ascii", 3, 4) + class TestEncoding(UnicodeTests): def test_all_ascii(self): From cfbolz at codespeak.net Mon Nov 12 19:50:56 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 19:50:56 +0100 (CET) Subject: [pypy-svn] r48609 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112185056.1F8CD82AD@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 19:50:55 2007 New Revision: 48609 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: grr. The error handling of PyPy's utf-16 decoder is broken. Fix the RPython version at least. Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 19:50:55 2007 @@ -195,24 +195,24 @@ # byte order setting accordingly. In native mode, the leading BOM # mark is skipped, in all other modes, it is copied to the output # stream as-is (giving a ZWNBSP character). - q = 0 - p = [] + pos = 0 + result = [] if byteorder == 'native': if (size >= 2): bom = (ord(s[ihi]) << 8) | ord(s[ilo]) if sys.byteorder == 'little': if (bom == 0xFEFF): - q += 2 + pos += 2 bo = -1 elif bom == 0xFFFE: - q += 2 + pos += 2 bo = 1 else: if bom == 0xFEFF: - q += 2 + pos += 2 bo = 1 elif bom == 0xFFFE: - q += 2 + pos += 2 bo = -1 elif byteorder == 'little': bo = -1 @@ -231,45 +231,48 @@ ilo = 1 #XXX I think the errors are not correctly handled here - while (q < len(s)): + while (pos < len(s)): # remaining bytes at the end? (size should be even) - if len(s) - q < 2: + if len(s) - pos < 2: if not final: break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - errorhandler(errors, 'utf-16', "truncated data", - s, startinpos, endinpos, True) - # CPython ignores the remaining input chars if the callback - # chooses to skip the input. XXX is this sensible? - ch = (ord(s[q + ihi]) << 8) | ord(s[q + ilo]) - q += 2 + r, pos = errorhandler(errors, 'utf16', "truncated data", + s, pos, len(s), True) + result.append(r) + if len(s) - pos < 2: + break + ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) + pos += 2 if (ch < 0xD800 or ch > 0xDFFF): - p += unichr(ch) + result += unichr(ch) continue # UTF-16 code pair: - if (q >= len(s)): + if len(s) - pos < 2: + if not final: + break errmsg = "unexpected end of data" - errorhandler(errors, 'utf-16', errmsg, s, q - 2, len(s)) + r, pos = errorhandler(errors, 'utf16', errmsg, s, pos - 2, len(s)) + result.append(r) + if len(s) - pos < 2: + break elif (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) - q += 2 + ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo]) + pos += 2 if (0xDC00 <= ch2 and ch2 <= 0xDFFF): if MAXUNICODE < 65536: - p += unichr(ch) - p += unichr(ch2) + result += unichr(ch) + result += unichr(ch2) else: - p += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) + result += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) continue else: - errmsg = "illegal UTF-16 surrogate" - errorhandler(errors, 'utf-16', errmsg, s, q - 4, q - 2) - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - return u"".join(p), q, bo + r, pos = errorhandler(errors, 'utf16', + "illegal UTF-16 surrogate", + s, pos - 4, pos - 2) + result.append(r) + else: + assert 0, "unreachable" + return u"".join(result), pos, bo def str_decode_latin1(s, size, errors, final=False, errorhandler=raise_unicode_exception): Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 19:50:55 2007 @@ -118,6 +118,23 @@ def test_ascii_error(self): self.checkdecodeerror("abc\xFF\xFF\xFFcde", "ascii", 3, 4) + def test_utf16_errors(self): + # trunkated BOM + for s in ["\xff", "\xfe"]: + self.checkdecodeerror(s, "utf16", 0, len(s), addstuff=False) + + for s in [ + # unexpected end of data ascii + "\xff\xfeF", + # unexpected end of data + '\xff\xfe\xc0\xdb\x00', '\xff\xfe\xc0\xdb', '\xff\xfe\xc0', + ]: + self.checkdecodeerror(s, "utf16", 2, len(s), addstuff=False) + for s in [ + # illegal surrogate + "\xff\xfe\xff\xdb\xff\xff", + ]: + self.checkdecodeerror(s, "utf16", 2, 4, addstuff=False) class TestEncoding(UnicodeTests): def test_all_ascii(self): From cfbolz at codespeak.net Mon Nov 12 20:15:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 20:15:53 +0100 (CET) Subject: [pypy-svn] r48610 - pypy/branch/more-unicode-improvements/pypy/tool Message-ID: <20071112191553.C3E8B8292@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 20:15:51 2007 New Revision: 48610 Added: pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py (contents, props changed) Log: small tool to encode and decode random unicode/byte strings to check encoders and decoders. Finds examples that aren't working in PyPy incredibly fast. Added: pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py ============================================================================== --- (empty file) +++ pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py Mon Nov 12 20:15:51 2007 @@ -0,0 +1,63 @@ +import random, sys +random.seed(42) + +def make_random_encoded_string(length=10, variance=1): + s = [] + s.append(random.choice(["\xff\xfe", "\xfe\xff", ""])) # BOM + for i in range(length + random.randrange(-variance, variance)): + s.append(chr(random.randrange(256))) + return "".join(s) + +def make_random_unicode(length=10, variance=1): + s = [] + for i in range(length + random.randrange(-variance, variance)): + s.append(unichr(random.randrange(sys.maxunicode))) + return "".join(s) + +def check_encode(encoding, s): + try: + s.encode(encoding) + except UnicodeError: + pass + s.encode(encoding, "ignore") + s.encode(encoding, "replace") + +def check_decode(encoding, s): + try: + s.decode(encoding) + except UnicodeError: + pass + s.decode(encoding, "ignore") + s.decode(encoding, "replace") + +def check_with_length(length): + try: + s = make_random_encoded_string(length, 10) + for encoding in all_encodinds: + check_decode(encoding, s) + except Exception, e: + print "decoding:", encoding, repr(s) + try: + s = make_random_unicode(length, 10) + for encoding in all_encodinds: + check_encode(encoding, s) + except Exception, e: + print "encoding:", encoding, repr(s) + + +def main(): + for length in range(0, 1000, 10): + print length + for i in range(100): + check_with_length(length) + length = 1000 + for length in range(1000, 1000000, 1000): + print length + for i in range(100): + check_with_length(length) + +all_encodinds = "utf-8 latin1 ascii utf-16 utf-16-be utf-16-le utf-7".split() + +if __name__ == '__main__': + main() + From cfbolz at codespeak.net Mon Nov 12 20:39:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 20:39:52 +0100 (CET) Subject: [pypy-svn] r48611 - pypy/branch/more-unicode-improvements/pypy/tool Message-ID: <20071112193952.98B2882B2@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 20:39:51 2007 New Revision: 48611 Modified: pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py Log: fix typo (thanks Alexander) Modified: pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py (original) +++ pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py Mon Nov 12 20:39:51 2007 @@ -33,13 +33,13 @@ def check_with_length(length): try: s = make_random_encoded_string(length, 10) - for encoding in all_encodinds: + for encoding in all_encodings: check_decode(encoding, s) except Exception, e: print "decoding:", encoding, repr(s) try: s = make_random_unicode(length, 10) - for encoding in all_encodinds: + for encoding in all_encodings: check_encode(encoding, s) except Exception, e: print "encoding:", encoding, repr(s) @@ -56,7 +56,7 @@ for i in range(100): check_with_length(length) -all_encodinds = "utf-8 latin1 ascii utf-16 utf-16-be utf-16-le utf-7".split() +all_encodings = "utf-8 latin1 ascii utf-16 utf-16-be utf-16-le utf-7".split() if __name__ == '__main__': main() From cfbolz at codespeak.net Mon Nov 12 21:37:26 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 21:37:26 +0100 (CET) Subject: [pypy-svn] r48612 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112203726.82E2E82B2@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 21:37:24 2007 New Revision: 48612 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: change names to be more consistent with CPython Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Mon Nov 12 21:37:24 2007 @@ -37,7 +37,7 @@ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 ] -def str_decode_utf8(s, size, errors, final=False, +def str_decode_utf_8(s, size, errors, final=False, errorhandler=raise_unicode_exception): if (size == 0): return u'', 0 @@ -56,13 +56,13 @@ if not final: break else: - r, pos = errorhandler(errors, "utf8", + r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) p += r if (pos + n > size): break if n == 0: - res = errorhandler(errors, "utf8", "unexpected code byte", + res = errorhandler(errors, "utf-8", "unexpected code byte", s, pos, pos + 1) p += res[0] pos = res[1] @@ -76,13 +76,13 @@ y, six = splitter[5, 3](ordch1) assert six == 6 if (two != 2): - r, pos = errorhandler(errors, "utf8", "invalid data", + r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) p += r else: c = (y << 6) + z if c < 0x80: - r, pos = errorhandler(errors, "utf8", "illegal encoding", + r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 2) p += r else: @@ -97,7 +97,7 @@ x, fourteen = splitter[4, 4](ordch1) assert fourteen == 14 if (two1 != 2 or two2 != 2): - r, pos = errorhandler(errors, "utf8", "invalid data", + r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) p += r else: @@ -108,7 +108,7 @@ # to recombine the surrogates into a single code # unit. if c < 0x0800: - r, pos = errorhandler(errors, "utf8", "illegal encoding", + r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 3) p += r else: @@ -126,7 +126,7 @@ w, thirty = splitter[3, 5](ordch1) assert thirty == 30 if (two1 != 2 or two2 != 2 or two3 != 2): - r, pos = errorhandler(errors, "utf8", "invalid data", + r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) p += r else: @@ -134,7 +134,7 @@ # minimum value allowed for 4 byte encoding # maximum value allowed for UTF-16 if ((c < 0x10000) or (c > 0x10ffff)): - r, pos = errorhandler(errors, "utf8", "illegal encoding", + r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) p += r else: @@ -151,7 +151,7 @@ p.append(unichr(0xDC00 + (c & 0x03FF))) pos += n else: - r, pos = errorhandler(errors, "utf8", + r, pos = errorhandler(errors, "utf-8", "unsupported Unicode code range", s, pos, pos + n) p += r @@ -159,27 +159,27 @@ return u"".join(p), pos -def str_decode_utf16(s, size, errors, final=True, +def str_decode_utf_16(s, size, errors, final=True, errorhandler=raise_unicode_exception): - result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, - errorhandler, "native") + result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, + errorhandler, "native") return result, length -def str_decode_utf16be(s, size, errors, final=True, +def str_decode_utf_16_be(s, size, errors, final=True, errorhandler=raise_unicode_exception): - result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, - errorhandler, "big") + result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, + errorhandler, "big") return result, length -def str_decode_utf16le(s, size, errors, final=True, - errorhandler=raise_unicode_exception): - result, length, byteorder = str_decode_utf16_helper(s, size, errors, final, - errorhandler, "little") +def str_decode_utf_16_le(s, size, errors, final=True, + errorhandler=raise_unicode_exception): + result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, + errorhandler, "little") return result, length -def str_decode_utf16_helper(s, size, errors, final=True, - errorhandler=raise_unicode_exception, - byteorder="native"): +def str_decode_utf_16_helper(s, size, errors, final=True, + errorhandler=raise_unicode_exception, + byteorder="native"): bo = 0 consumed = 0 @@ -236,7 +236,7 @@ if len(s) - pos < 2: if not final: break - r, pos = errorhandler(errors, 'utf16', "truncated data", + r, pos = errorhandler(errors, 'utf-16', "truncated data", s, pos, len(s), True) result.append(r) if len(s) - pos < 2: @@ -251,7 +251,7 @@ if not final: break errmsg = "unexpected end of data" - r, pos = errorhandler(errors, 'utf16', errmsg, s, pos - 2, len(s)) + r, pos = errorhandler(errors, 'utf-16', errmsg, s, pos - 2, len(s)) result.append(r) if len(s) - pos < 2: break @@ -266,7 +266,7 @@ result += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) continue else: - r, pos = errorhandler(errors, 'utf16', + r, pos = errorhandler(errors, 'utf-16', "illegal UTF-16 surrogate", s, pos - 4, pos - 2) result.append(r) @@ -306,7 +306,7 @@ # unicode encoding -def unicode_encode_utf8(s, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception): assert(size >= 0) p = [] i = 0 @@ -400,9 +400,9 @@ p.append(hi) p.append(lo) -def unicode_encode_utf16_helper(s, size, errors, - errorhandler=raise_unicode_exception, - byteorder='little'): +def unicode_encode_utf_16_helper(s, size, errors, + errorhandler=raise_unicode_exception, + byteorder='little'): p = [] if (byteorder == 'native'): _STORECHAR(p, 0xFEFF, sys.byteorder) @@ -426,16 +426,16 @@ return "".join(p) -def unicode_encode_utf16(s, size, errors, - errorhandler=raise_unicode_exception): - return unicode_encode_utf16_helper(s, size, errors, errorhandler, "native") +def unicode_encode_utf_16(s, size, errors, + errorhandler=raise_unicode_exception): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") -def unicode_encode_utf16be(s, size, errors, +def unicode_encode_utf_16_be(s, size, errors, errorhandler=raise_unicode_exception): - return unicode_encode_utf16_helper(s, size, errors, errorhandler, "big") + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") -def unicode_encode_utf16le(s, size, errors, - errorhandler=raise_unicode_exception): - return unicode_encode_utf16_helper(s, size, errors, errorhandler, "little") +def unicode_encode_utf_16_le(s, size, errors, + errorhandler=raise_unicode_exception): + return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Mon Nov 12 21:37:24 2007 @@ -6,8 +6,15 @@ assert x == y assert type(x) is type(y) + def getdecoder(self, encoding): + return getattr(runicode, "str_decode_%s" % encoding.replace("-", "_")) + + def getencoder(self, encoding): + return getattr(runicode, + "unicode_encode_%s" % encoding.replace("-", "_")) + def checkdecode(self, s, encoding): - decoder = getattr(runicode, "str_decode_%s" % encoding.replace("-", "")) + decoder = self.getdecoder(encoding) if isinstance(s, str): trueresult = s.decode(encoding) else: @@ -18,8 +25,7 @@ self.typeequals(trueresult, result) def checkencode(self, s, encoding): - encoder = getattr(runicode, - "unicode_encode_%s" % encoding.replace("-", "")) + encoder = self.getencoder(encoding) if isinstance(s, unicode): trueresult = s.encode(encoding) else: @@ -40,8 +46,7 @@ assert stop == endingpos assert not decode return "42424242", stop - encoder = getattr(runicode, - "unicode_encode_%s" % encoding.replace("-", "")) + encoder = self.getencoder(encoding) result = encoder(s, len(s), "foo!", errorhandler) assert called[0] assert "42424242" in result @@ -60,8 +65,7 @@ assert decode return u"42424242", stop return "", endingpos - decoder = getattr(runicode, - "str_decode_%s" % encoding.replace("-", "")) + decoder = self.getdecoder(encoding) if addstuff: s += "some rest in ascii" result, _ = decoder(s, len(s), "foo!", True, errorhandler) @@ -78,28 +82,28 @@ def test_all_ascii(self): for i in range(128): - for encoding in "utf8 latin1 ascii".split(): + for encoding in "utf-8 latin1 ascii".split(): self.checkdecode(chr(i), encoding) def test_all_first_256(self): for i in range(256): - for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 latin1 utf-16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) def test_random(self): for i in range(10000): uni = unichr(random.randrange(sys.maxunicode)) - for encoding in "utf8 utf16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 utf-16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) def test_single_chars_utf8(self): for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: - self.checkdecode(s, "utf8") + self.checkdecode(s, "utf-8") def test_utf8_errors(self): for s in [# unexpected end of data "\xd7", "\xd6", "\xeb\x96", "\xf0\x90\x91"]: - self.checkdecodeerror(s, "utf8", 0, len(s), addstuff=False) + self.checkdecodeerror(s, "utf-8", 0, len(s), addstuff=False) for s in [# unexpected code byte "\x81", "\xbf", @@ -113,7 +117,7 @@ "\xf0\x90\x51\x93", "\xf0\x90\x01\x93", "\xf0\x90\xd1\x93", "\xf0\x90\x91\x53", "\xf0\x90\x91\x03", "\xf0\x90\x91\xd3", ]: - self.checkdecodeerror(s, "utf8", 0, len(s), addstuff=True) + self.checkdecodeerror(s, "utf-8", 0, len(s), addstuff=True) def test_ascii_error(self): self.checkdecodeerror("abc\xFF\xFF\xFFcde", "ascii", 3, 4) @@ -121,7 +125,7 @@ def test_utf16_errors(self): # trunkated BOM for s in ["\xff", "\xfe"]: - self.checkdecodeerror(s, "utf16", 0, len(s), addstuff=False) + self.checkdecodeerror(s, "utf-16", 0, len(s), addstuff=False) for s in [ # unexpected end of data ascii @@ -129,33 +133,33 @@ # unexpected end of data '\xff\xfe\xc0\xdb\x00', '\xff\xfe\xc0\xdb', '\xff\xfe\xc0', ]: - self.checkdecodeerror(s, "utf16", 2, len(s), addstuff=False) + self.checkdecodeerror(s, "utf-16", 2, len(s), addstuff=False) for s in [ # illegal surrogate "\xff\xfe\xff\xdb\xff\xff", ]: - self.checkdecodeerror(s, "utf16", 2, 4, addstuff=False) + self.checkdecodeerror(s, "utf-16", 2, 4, addstuff=False) class TestEncoding(UnicodeTests): def test_all_ascii(self): for i in range(128): - for encoding in "utf8 latin1 ascii".split(): + for encoding in "utf-8 latin1 ascii".split(): self.checkencode(unichr(i), encoding) def test_all_first_256(self): for i in range(256): - for encoding in "utf8 latin1 utf16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 latin1 utf-16 utf-16-be utf-16-le".split(): self.checkencode(unichr(i), encoding) def test_random(self): for i in range(10000): uni = unichr(random.randrange(sys.maxunicode)) - for encoding in "utf8 utf16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 utf-16 utf-16-be utf-16-le".split(): self.checkencode(unichr(i), encoding) def test_single_chars_utf8(self): for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: - self.checkencode(s, "utf8") + self.checkencode(s, "utf-8") def test_ascii_error(self): self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) From rxe at codespeak.net Mon Nov 12 21:39:00 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 12 Nov 2007 21:39:00 +0100 (CET) Subject: [pypy-svn] r48613 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071112203900.99AC282B4@code0.codespeak.net> Author: rxe Date: Mon Nov 12 21:39:00 2007 New Revision: 48613 Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py Log: failing test was because of a dirty cache Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_extfunc.py Mon Nov 12 21:39:00 2007 @@ -15,8 +15,6 @@ assert abs(f()-fn()) < 10.0 def test_external_function_ll_time_clock(): - if sys.platform == 'linux2': - py.test.skip('fix me') import time def fn(): return time.clock() From rxe at codespeak.net Mon Nov 12 21:43:19 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 12 Nov 2007 21:43:19 +0100 (CET) Subject: [pypy-svn] r48614 - pypy/dist/pypy/translator/llvm Message-ID: <20071112204319.89D3582B8@code0.codespeak.net> Author: rxe Date: Mon Nov 12 21:43:19 2007 New Revision: 48614 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/structnode.py Log: pretty print pbcs Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Mon Nov 12 21:43:19 2007 @@ -3,7 +3,7 @@ class ArrayNode(ConstantNode): __slots__ = "db value arraytype".split() - prefix = '@arrayinstance' + prefix = '@a_isnt' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) @@ -30,7 +30,12 @@ def get_arrayvalue(self): items = self.value.items l = len(items) - r = "[%s]" % ", ".join([self.db.repr_constant(v)[1] for v in items]) + reprs = [self.db.repr_constant(v)[1] for v in items] + line = ", ".join(reprs) + if len(line) < 70: + r = "[%s]" % line + else: + r = "[%s]" % ",\n\t ".join(reprs) return l, r def get_typerepr(self): @@ -48,8 +53,8 @@ ref = self.db.get_childref(p, c) ref = "bitcast(%s* %s to %s*)" % (self.get_typerepr(), - ref, - typeval) + ref, + typeval) return ref def get_pbcref(self, toptr): @@ -71,11 +76,11 @@ typeval = self.db.repr_type(self.arraytype) # first length is logical, second is physical - value = "%s %s, [%s x %s] %s" % (self.db.get_machine_word(), - self.get_length(), - physicallen, - typeval, - arrayrepr) + value = "%s %s, [%s x %s]\n\t%s" % (self.db.get_machine_word(), + self.get_length(), + physicallen, + typeval, + arrayrepr) s = "%s {%s}" % (self.get_typerepr(), value) return s Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Mon Nov 12 21:43:19 2007 @@ -87,5 +87,7 @@ p, c = lltype.parentlink(self.value) if p is None: codewriter.globalinstance(self.ref, self.constantvalue()) + codewriter.newline() + codewriter.newline() Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Mon Nov 12 21:43:19 2007 @@ -19,7 +19,7 @@ """ __slots__ = "db value structtype _get_ref_cache _get_types".split() - prefix = '@structinstance_' + prefix = '@s_inst_' def __init__(self, db, value): self.db = db @@ -89,12 +89,15 @@ def constantvalue(self): """ Returns the constant representation for this node. """ values = self._getvalues() - all_values = ",\n ".join(values) - return "%s {\n %s\n }\n" % (self.get_typerepr(), all_values) - + if len(values) > 3: + all_values = ",\n\t".join(values) + return "%s {\n\t%s }" % (self.get_typerepr(), all_values) + else: + all_values = ", ".join(values) + return "%s { %s }" % (self.get_typerepr(), all_values) class FixedSizeArrayNode(StructNode): - prefix = '@fixarrayinstance_' + prefix = '@fa_inst_' def __init__(self, db, struct): super(FixedSizeArrayNode, self).__init__(db, struct) @@ -148,6 +151,8 @@ a series of embedded structs, which has as its last element an array. """ + prefix = '@sv_inst_' + def _getvalues(self): values = [] for name, T in self._get_types[:-1]: From cfbolz at codespeak.net Mon Nov 12 22:23:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 22:23:28 +0100 (CET) Subject: [pypy-svn] r48615 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112212328.6FCA082B6@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 22:23:27 2007 New Revision: 48615 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Log: plug the RPython decoders and encoders into the _codecs module. Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py Mon Nov 12 22:23:27 2007 @@ -4,15 +4,11 @@ appleveldefs = { '__doc__' : 'app_codecs.__doc__', '__name__' : 'app_codecs.__name__', - 'ascii_decode' : 'app_codecs.ascii_decode', - 'ascii_encode' : 'app_codecs.ascii_encode', 'charbuffer_encode' : 'app_codecs.charbuffer_encode', 'charmap_decode' : 'app_codecs.charmap_decode', 'charmap_encode' : 'app_codecs.charmap_encode', 'escape_decode' : 'app_codecs.escape_decode', 'escape_encode' : 'app_codecs.escape_encode', - 'latin_1_decode' : 'app_codecs.latin_1_decode', - 'latin_1_encode' : 'app_codecs.latin_1_encode', 'mbcs_decode' : 'app_codecs.mbcs_decode', 'mbcs_encode' : 'app_codecs.mbcs_encode', 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', @@ -22,17 +18,9 @@ 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', - 'utf_16_be_decode' : 'app_codecs.utf_16_be_decode', - 'utf_16_be_encode' : 'app_codecs.utf_16_be_encode', - 'utf_16_decode' : 'app_codecs.utf_16_decode', - 'utf_16_encode' : 'app_codecs.utf_16_encode', - 'utf_16_ex_decode' : 'app_codecs.utf_16_ex_decode', - 'utf_16_le_decode' : 'app_codecs.utf_16_le_decode', - 'utf_16_le_encode' : 'app_codecs.utf_16_le_encode', 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'utf_8_decode' : 'app_codecs.utf_8_decode', - 'utf_8_encode' : 'app_codecs.utf_8_encode', + 'utf_16_ex_decode' : 'app_codecs.utf_16_ex_decode', '_register_existing_errors': 'app_codecs._register_existing_errors', } interpleveldefs = { @@ -42,6 +30,20 @@ 'lookup_error': 'interp_codecs.lookup_error', 'register': 'interp_codecs.register_codec', 'register_error': 'interp_codecs.register_error', + + # encoders and decoders + 'ascii_decode' : 'interp_codecs.ascii_decode', + 'ascii_encode' : 'interp_codecs.ascii_encode', + 'latin_1_decode' : 'interp_codecs.latin_1_decode', + 'latin_1_encode' : 'interp_codecs.latin_1_encode', + 'utf_8_decode' : 'interp_codecs.utf_8_decode', + 'utf_8_encode' : 'interp_codecs.utf_8_encode', + 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', + 'utf_16_be_encode' : 'interp_codecs.utf_16_be_encode', + 'utf_16_decode' : 'interp_codecs.utf_16_decode', + 'utf_16_encode' : 'interp_codecs.utf_16_encode', + 'utf_16_le_decode' : 'interp_codecs.utf_16_le_decode', + 'utf_16_le_encode' : 'interp_codecs.utf_16_le_encode', } def setup_after_space_initialization(self): @@ -49,5 +51,4 @@ self.space.appexec([], """(): import _codecs _codecs._register_existing_errors() - del _codecs._register_existing_errors """) Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Mon Nov 12 22:23:27 2007 @@ -8,6 +8,52 @@ self.codec_search_cache = {} self.codec_error_registry = {} self.codec_need_encodings = True + self.error_handler = self.make_errorhandler(space) + + def make_errorhandler(self, space): + def unicode_call_errorhandler(errors, encoding, reason, input, + startinpos, endinpos, decode=True): + + w_errorhandler = lookup_error(space, errors) + if decode: + w_cls = space.w_UnicodeDecodeError + else: + w_cls = space.w_UnicodeEncodeError + w_exc = space.call_function( + w_cls, + space.wrap(encoding), + space.wrap(input), + space.wrap(startinpos), + space.wrap(endingpos), + space.wrap(reason)) + w_res = space.call_function(w_errorhandler, w_exc) + try: + w_replace, w_newpos = space.unpacktuple(w_res, 2) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise OperationError( + space.w_TypeError, + space.wrap("encoding error handler must return " + "(unicode, int) tuple, not %s" % ( + space.str_w(space.repr(w_res))))) + newpos = space.int_w(w_newpos) + if (newpos < 0): + newpos = len(input) + newpos + if newpos < 0 or newpos > len(input): + raise OperationError( + space.w_IndexError, + space.wrap("position %d from error handler " + "out of bounds" % newpos)) + if decode: + replace = space.unicode_w(w_replace) + return replace, newpos + else: + replace = space.str_w(w_replace) + return replace, newpos + unicode_call_errorhandler._annspecialcase_ = "specialize:arg(6)" + return unicode_call_errorhandler + def register_codec(space, w_search_function): """register(search_function) @@ -95,6 +141,7 @@ 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle ValueErrors. """ + #import pdb; pdb.set_trace() if encoding is None: encoding = space.sys.defaultencoding w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0)) @@ -145,3 +192,50 @@ space.w_TypeError, space.wrap("handler must be callable")) register_error.unwrap_spec = [ObjSpace, str, W_Root] + +# ____________________________________________________________ +# delegation to runicode + +from pypy.rlib import runicode + +def make_encoder_wrapper(name): + rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) + def wrap_encoder(space, uni, errors="strict"): + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + result = func(uni, len(uni), errors, state.error_handler) + return space.newtuple([space.wrap(result), space.wrap(len(result))]) + wrap_encoder.unwrap_spec = [ObjSpace, unicode, str] + globals()[name] = wrap_encoder + +def make_decoder_wrapper(name): + rname = "str_decode_%s" % (name.replace("_decode", ""), ) + def wrap_decoder(space, string, errors="strict", w_final=True): + final = space.is_true(w_final) + state = space.fromcache(CodecState) + func = getattr(runicode, rname) + result, consumed = func(string, len(string), errors, + final, state.error_handler) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + wrap_decoder.unwrap_spec = [ObjSpace, str, str, W_Root] + globals()[name] = wrap_decoder + +for encoders in [ + "ascii_encode", + "latin_1_encode", + "utf_8_encode", + "utf_16_encode", + "utf_16_be_encode", + "utf_16_le_encode", + ]: + make_encoder_wrapper(encoders) + +for decoders in [ + "ascii_decode", + "latin_1_decode", + "utf_8_decode", + "utf_16_decode", + "utf_16_be_decode", + "utf_16_le_decode", + ]: + make_decoder_wrapper(decoders) From cfbolz at codespeak.net Mon Nov 12 22:34:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 22:34:10 +0100 (CET) Subject: [pypy-svn] r48616 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112213410.1C74482BE@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 22:34:09 2007 New Revision: 48616 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Log: kill kill kill kill kill kill kill kill the applevel versions (they are buggy anyway) Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Mon Nov 12 22:34:09 2007 @@ -42,12 +42,6 @@ import sys -def latin_1_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeLatin1(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) # XXX MBCS codec might involve ctypes ? def mbcs_decode(): """None @@ -67,16 +61,6 @@ v = s[1:-1] return v, len(v) -def utf_8_decode( data, errors='strict', final=False): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed = PyUnicode_DecodeUTF8Stateful(data, len(data), errors, final) - res = u''.join(res) - return res, consumed - def raw_unicode_escape_decode( data, errors='strict'): """None """ @@ -98,23 +82,6 @@ res = ''.join(res) return res, len(res) -def latin_1_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = u''.join(res) - return res, len(res) - -def utf_16_decode( data, errors='strict', final=False): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) - return res, consumed - def unicode_escape_decode( data, errors='strict'): """None """ @@ -123,13 +90,6 @@ return res, len(res) -def ascii_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeASCII(data, len(data), errors) - res = u''.join(res) - return res, len(res) - def charmap_encode(obj, errors='strict', mapping='latin-1'): """None """ @@ -289,20 +249,6 @@ ## len(obj)) -def ascii_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeASCII(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) - -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') - res = ''.join(res) - return res, len(res) - def raw_unicode_escape_encode( obj, errors='strict'): """None """ @@ -310,47 +256,6 @@ res = ''.join(res) return res, len(res) -def utf_8_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF8(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) - -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') - res = ''.join(res) - return res, len(res) - -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') - res = ''.join(res) - return res, len(res) - -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = u''.join(res) - return res, consumed - -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = u''.join(res) - return res, consumed - def strict_errors(exc): if isinstance(exc, Exception): raise exc @@ -714,40 +619,6 @@ p += p[1] return p -def PyUnicode_DecodeASCII(s, size, errors): - -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : - return [unichr(ord(s))] - if (size == 0): - return [u''] #unicode('') - p = [] - pos = 0 - while pos < len(s): - c = s[pos] - if ord(c) < 128: - p += unichr(ord(c)) - pos += 1 - else: - - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) - p += [unichr(ord(x)) for x in res[0]] - pos = res[1] - return p - -def PyUnicode_EncodeASCII(p, size, errors): - - return unicode_encode_ucs1(p, size, errors, 128) - -def PyUnicode_AsASCIIString(unistr): - - if not type(unistr) == unicode: - raise TypeError - return PyUnicode_EncodeASCII(unicode(unistr), - len(unicode), - None) def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): @@ -871,40 +742,6 @@ else: return [hi, lo] -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): - -# /* Offsets from p for storing byte pairs in the right order. */ - - - p = [] - bom = sys.byteorder - if (byteorder == 'native'): - - bom = sys.byteorder - p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' - - - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) - - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) - - return p - def PyUnicode_DecodeMBCS(s, size, errors): pass @@ -932,239 +769,6 @@ else: raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) -def PyUnicode_DecodeUTF8(s, size, errors): - return PyUnicode_DecodeUTF8Stateful(s, size, errors, False) - -## /* Map UTF-8 encoded prefix byte to sequence length. zero means -## illegal prefix. see RFC 2279 for details */ -utf8_code_length = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 -] - -def PyUnicode_DecodeUTF8Stateful(s, size, errors, final): - - consumed = 0 - if (size == 0): - if not final: - consumed = 0 - return u'', consumed - p = [] - pos = 0 - while pos < size: - ch = s[pos] - if ord(ch) < 0x80: - p += ch - pos += 1 - continue - - n = utf8_code_length[ord(ch)] - startinpos = pos - if (startinpos + n > size): - if not final: - break - else: - errmsg = "unexpected end of data" - endinpos = size - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - if n == 0: - errmsg = "unexpected code byte" - endinpos = startinpos+1 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - elif n == 1: - errmsg = "internal error" - endinpos = startinpos+1 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - elif n == 2: - if ((ord(s[pos+1]) & 0xc0) != 0x80): - errmsg = "invalid data" - endinpos = startinpos+2 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos]) & 0x1f) << 6) + (ord(s[pos+1]) & 0x3f) - if c < 0x80: - errmsg = "illegal encoding" - endinpos = startinpos+2 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - p += unichr(c) - pos += n - #break - elif n == 3: - if ((ord(s[pos+1]) & 0xc0) != 0x80 or - (ord(s[pos+2]) & 0xc0) != 0x80): - errmsg = "invalid data" - endinpos = startinpos+3 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos]) & 0x0f) << 12) + \ - ((ord(s[pos+1]) & 0x3f) << 6) +\ - (ord(s[pos+2]) & 0x3f) - -## /* Note: UTF-8 encodings of surrogates are considered -## legal UTF-8 sequences; -## -## XXX For wide builds (UCS-4) we should probably try -## to recombine the surrogates into a single code -## unit. -## */ - if c < 0x0800: - errmsg = "illegal encoding" - endinpos = startinpos+3 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - p += unichr(c) - pos += n - elif n == 4: -## case 4: - if ((ord(s[pos+1]) & 0xc0) != 0x80 or - (ord(s[pos+2]) & 0xc0) != 0x80 or - (ord(s[pos+3]) & 0xc0) != 0x80): - - errmsg = "invalid data" - startinpos = pos - endinpos = startinpos+4 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos+0]) & 0x7) << 18) + ((ord(s[pos+1]) & 0x3f) << 12) +\ - ((ord(s[pos+2]) & 0x3f) << 6) + (ord(s[pos+3]) & 0x3f) - #/* validate and convert to UTF-16 */ - if ((c < 0x10000) or (c > 0x10ffff)): - #/* minimum value allowed for 4 byte encoding */ - #/* maximum value allowed for UTF-16 */ - - errmsg = "illegal encoding" - startinpos = pos - endinpos = startinpos+4 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: -#ifdef Py_UNICODE_WIDE - if c < sys.maxunicode: - p += unichr(c) - pos += n - else: -## /* compute and append the two surrogates: */ -## /* translate from 10000..10FFFF to 0..FFFF */ - c -= 0x10000 - #/* high surrogate = top 10 bits added to D800 */ - p += unichr(0xD800 + (c >> 10)) - #/* low surrogate = bottom 10 bits added to DC00 */ - p += unichr(0xDC00 + (c & 0x03FF)) - pos += n - else: -## default: -## /* Other sizes are only needed for UCS-4 */ - errmsg = "unsupported Unicode code range" - startinpos = pos - endinpos = startinpos+n - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - - #continue - - if not final: - consumed = pos - return p, pos # consumed - -def PyUnicode_EncodeUTF8(s, size, errors): - - #assert(s != None) - assert(size >= 0) - p = [] - i = 0 - while i < size: - ch = s[i] - i += 1 - if (ord(ch) < 0x80): -## /* Encode ASCII */ - p += chr(ord(ch)) - elif (ord(ch) < 0x0800) : -## /* Encode Latin-1 */ - p += chr((0xc0 | (ord(ch) >> 6))) - p += chr((0x80 | (ord(ch) & 0x3f))) - else: -## /* Encode UCS2 Unicode ordinals */ - if (ord(ch) < 0x10000): -## /* Special case: check for high surrogate */ - if (0xD800 <= ord(ch) and ord(ch) <= 0xDBFF and i != size) : - ch2 = s[i] -## /* Check for low surrogate and combine the two to -## form a UCS4 value */ - if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : - ch3 = ((ord(ch) - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 - i += 1 - p.extend(encodeUCS4(ch3)) - continue -## /* Fall through: handles isolated high surrogates */ - p += (chr((0xe0 | (ord(ch) >> 12)))) - p += (chr((0x80 | ((ord(ch) >> 6) & 0x3f)))) - p += (chr((0x80 | (ord(ch) & 0x3f)))) - continue - else: - p.extend(encodeUCS4(ord(ch))) - return p - -def encodeUCS4(ch): -## /* Encode UCS4 Unicode ordinals */ - p = [] - p += (chr((0xf0 | (ch >> 18)))) - p += (chr((0x80 | ((ch >> 12) & 0x3f)))) - p += (chr((0x80 | ((ch >> 6) & 0x3f)))) - p += (chr((0x80 | (ch & 0x3f)))) - return p #/* --- Latin-1 Codec ------------------------------------------------------ */ From cfbolz at codespeak.net Mon Nov 12 23:00:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 23:00:03 +0100 (CET) Subject: [pypy-svn] r48617 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112220003.0BE0682BD@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 23:00:03 2007 New Revision: 48617 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Log: yet another decoder moved to use the runicode version Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py Mon Nov 12 23:00:03 2007 @@ -20,7 +20,6 @@ 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'utf_16_ex_decode' : 'app_codecs.utf_16_ex_decode', '_register_existing_errors': 'app_codecs._register_existing_errors', } interpleveldefs = { @@ -44,6 +43,7 @@ 'utf_16_encode' : 'interp_codecs.utf_16_encode', 'utf_16_le_decode' : 'interp_codecs.utf_16_le_decode', 'utf_16_le_encode' : 'interp_codecs.utf_16_le_encode', + 'utf_16_ex_decode' : 'interp_codecs.utf_16_ex_decode', } def setup_after_space_initialization(self): Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Mon Nov 12 23:00:03 2007 @@ -150,22 +150,6 @@ res = u''.join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ - if byteorder == 0: - bm = 'native' - elif byteorder == -1: - bm = 'little' - else: - bm = 'big' - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) - return res, consumed, byteorder - # XXX needs error messages when the input is invalid def escape_decode(data, errors='strict'): """None Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Mon Nov 12 23:00:03 2007 @@ -239,3 +239,25 @@ "utf_16_le_decode", ]: make_decoder_wrapper(decoders) + + +def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False): + """None + """ + final = space.is_true(w_final) + state = space.fromcache(CodecState) + if byteorder == 0: + byteorder = 'native' + elif byteorder == -1: + byteorder = 'little' + else: + byteorder = 'big' + consumed = len(data) + if final: + consumed = 0 + res, consumed, byteorder = runicode.str_decode_utf_16_helper( + data, len(data), errors, final, state.error_handler, byteorder) + return space.newtuple([space.wrap(res), space.wrap(consumed), + space.wrap(byteorder)]) +utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] + From cfbolz at codespeak.net Mon Nov 12 23:02:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 23:02:18 +0100 (CET) Subject: [pypy-svn] r48618 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112220218.9EADF82C5@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 23:02:18 2007 New Revision: 48618 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Log: this is no longer needed either Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Mon Nov 12 23:02:18 2007 @@ -604,129 +604,6 @@ return p -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - - bo = 0 #/* assume native ordering by default */ - consumed = 0 - errmsg = "" - - if sys.byteorder == 'little': - ihi = 1 - ilo = 0 - else: - ihi = 0 - ilo = 1 - - - #/* Unpack UTF-16 encoded data */ - -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ - q = 0 - p = [] - if byteorder == 'native': - if (size >= 2): - bom = (ord(s[ihi]) << 8) | ord(s[ilo]) -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): - q += 2 - bo = -1 - elif bom == 0xFFFE: - q += 2 - bo = 1 - else: - if bom == 0xFEFF: - q += 2 - bo = 1 - elif bom == 0xFFFE: - q += 2 - bo = -1 - elif byteorder == 'little': - bo = -1 - else: - bo = 1 - - if (size == 0): - return [u''], 0, bo - - if (bo == -1): - #/* force LE */ - ihi = 1 - ilo = 0 - - elif (bo == 1): - #/* force BE */ - ihi = 0 - ilo = 1 - - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): - if not final: - break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): - p += unichr(ch) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) - q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += unichr(ch) - p += unichr(ch2) - else: - p += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) - #endif - continue - - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - return p, q, bo - -# moved out of local scope, especially because it didn't -# have any nested variables. - -def STORECHAR(CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - return [lo, hi] - else: - return [hi, lo] - - def PyUnicode_DecodeMBCS(s, size, errors): pass From cfbolz at codespeak.net Mon Nov 12 23:12:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 23:12:00 +0100 (CET) Subject: [pypy-svn] r48619 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112221200.3EFC782D3@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 23:12:00 2007 New Revision: 48619 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Log: kill kill kill some more code Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py Mon Nov 12 23:12:00 2007 @@ -331,9 +331,6 @@ 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, ] -unicode_latin1 = [None]*256 - - def SPECIAL(c, encodeO, encodeWS): @@ -631,54 +628,6 @@ raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) -#/* --- Latin-1 Codec ------------------------------------------------------ */ - -def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] - pos = 0 - p = [] - while (pos < size): - p += unichr(ord(s[pos])) - pos += 1 - return p - -def unicode_encode_ucs1(p, size, errors, limit): - - if limit == 256: - reason = "ordinal not in range(256)" - encoding = "latin-1" - else: - reason = "ordinal not in range(128)" - encoding = "ascii" - - if (size == 0): - return [''] - res = [] - pos = 0 - while pos < len(p): - #for ch in p: - ch = p[pos] - - if ord(ch) < limit: - res += chr(ord(ch)) - pos += 1 - else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 - while collend < len(p) and ord(p[collend]) >= limit: - collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += str(x[0]) - pos = x[1] - - return res - -def PyUnicode_EncodeLatin1(p, size, errors): - res = unicode_encode_ucs1(p, size, errors, 256) - return res hexdigits = [hex(i)[-1] for i in range(16)]+[hex(i)[-1].upper() for i in range(10, 16)] @@ -872,7 +821,8 @@ # /* Default to Latin-1 */ if mapping == 'latin-1': - return PyUnicode_EncodeLatin1(p, size, errors) + import _codecs + return _codecs.latin_1_encode(p, size, errors) if (size == 0): return '' inpos = 0 @@ -897,7 +847,8 @@ ## /* Default to Latin-1 */ if (mapping == None): - return PyUnicode_DecodeLatin1(s, size, errors) + import _codecs + return _codecs.latin_1_decode(s, size, errors) if (size == 0): return u'' From ericvrp at codespeak.net Mon Nov 12 23:27:43 2007 From: ericvrp at codespeak.net (ericvrp at codespeak.net) Date: Mon, 12 Nov 2007 23:27:43 +0100 (CET) Subject: [pypy-svn] r48620 - pypy/dist/pypy/translator/llvm Message-ID: <20071112222743.DACD582D4@code0.codespeak.net> Author: ericvrp Date: Mon Nov 12 23:27:37 2007 New Revision: 48620 Modified: pypy/dist/pypy/translator/llvm/arraynode.py Log: fix typo Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Mon Nov 12 23:27:37 2007 @@ -3,7 +3,7 @@ class ArrayNode(ConstantNode): __slots__ = "db value arraytype".split() - prefix = '@a_isnt' + prefix = '@a_inst' def __init__(self, db, value): assert isinstance(lltype.typeOf(value), lltype.Array) From cfbolz at codespeak.net Mon Nov 12 23:45:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Nov 2007 23:45:55 +0100 (CET) Subject: [pypy-svn] r48621 - in pypy/branch/more-unicode-improvements/pypy/interpreter: . test Message-ID: <20071112224555.65A7981B6@code0.codespeak.net> Author: cfbolz Date: Mon Nov 12 23:45:55 2007 New Revision: 48621 Modified: pypy/branch/more-unicode-improvements/pypy/interpreter/gateway.py pypy/branch/more-unicode-improvements/pypy/interpreter/test/test_gateway.py Log: allow unicode in the unwrap_spec Modified: pypy/branch/more-unicode-improvements/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/interpreter/gateway.py (original) +++ pypy/branch/more-unicode-improvements/pypy/interpreter/gateway.py Mon Nov 12 23:45:55 2007 @@ -164,7 +164,7 @@ app_sig.varargname = argname[2:] def visit__object(self, typ, app_sig): - if typ not in (int, str, float, r_longlong): + if typ not in (int, str, float, unicode, r_longlong): assert False, "unsupported basic type in unwrap_spec" self.checked_space_method(typ.__name__, app_sig) @@ -210,8 +210,8 @@ self.run_args.append(self.scopenext()) def visit__object(self, typ): - if typ not in (int, str, float, r_longlong): - assert False, "unsupported basic type in uwnrap_spec" + if typ not in (int, str, float, unicode, r_longlong): + assert False, "unsupported basic type in unwrap_spec" if typ is r_int is r_longlong: name = 'r_longlong' else: @@ -327,7 +327,7 @@ raise FastFuncNotSupported def visit__object(self, typ): - if typ not in (int, str, float, r_longlong): + if typ not in (int, str, float, unicode, r_longlong): assert False, "unsupported basic type in uwnrap_spec" self.unwrap.append("space.%s_w(%s)" % (typ.__name__, self.nextarg())) Modified: pypy/branch/more-unicode-improvements/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/more-unicode-improvements/pypy/interpreter/test/test_gateway.py Mon Nov 12 23:45:55 2007 @@ -287,6 +287,27 @@ raises(gateway.OperationError,space.call_function,w_app_g3_f,w(None)) raises(gateway.OperationError,space.call_function,w_app_g3_f,w("foo")) + def test_interp2app_unwrap_spec_unicode(self): + space = self.space + w = space.wrap + def g3_u(space, uni): + return space.wrap(len(uni)) + app_g3_u = gateway.interp2app_temp(g3_u, + unwrap_spec=[gateway.ObjSpace, + unicode]) + w_app_g3_u = space.wrap(app_g3_u) + assert self.space.eq_w( + space.call_function(w_app_g3_u, w(u"foo")), + w(3)) + assert self.space.eq_w( + space.call_function(w_app_g3_u, w("baz")), + w(3)) + raises(gateway.OperationError, space.call_function, w_app_g3_u, + w(None)) + raises(gateway.OperationError, space.call_function, w_app_g3_u, + w(42)) + + def test_interp2app_unwrap_spec_func(self): space = self.space w = space.wrap From cfbolz at codespeak.net Tue Nov 13 00:10:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 00:10:00 +0100 (CET) Subject: [pypy-svn] r48623 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071112231000.02A5382CB@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 00:10:00 2007 New Revision: 48623 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: change latin1 to latin_1 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 00:10:00 2007 @@ -274,7 +274,7 @@ assert 0, "unreachable" return u"".join(result), pos, bo -def str_decode_latin1(s, size, errors, final=False, +def str_decode_latin_1(s, size, errors, final=False, errorhandler=raise_unicode_exception): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 @@ -354,7 +354,7 @@ errorhandler=raise_unicode_exception, limit=256): if limit == 256: reason = "ordinal not in range(256)" - encoding = "latin1" + encoding = "latin-1" else: reason = "ordinal not in range(128)" encoding = "ascii" @@ -381,7 +381,7 @@ return "".join(res) -def unicode_encode_latin1(p, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Tue Nov 13 00:10:00 2007 @@ -82,12 +82,12 @@ def test_all_ascii(self): for i in range(128): - for encoding in "utf-8 latin1 ascii".split(): + for encoding in "utf-8 latin-1 ascii".split(): self.checkdecode(chr(i), encoding) def test_all_first_256(self): for i in range(256): - for encoding in "utf-8 latin1 utf-16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 latin-1 utf-16 utf-16-be utf-16-le".split(): self.checkdecode(unichr(i), encoding) def test_random(self): @@ -143,12 +143,12 @@ class TestEncoding(UnicodeTests): def test_all_ascii(self): for i in range(128): - for encoding in "utf-8 latin1 ascii".split(): + for encoding in "utf-8 latin-1 ascii".split(): self.checkencode(unichr(i), encoding) def test_all_first_256(self): for i in range(256): - for encoding in "utf-8 latin1 utf-16 utf-16-be utf-16-le".split(): + for encoding in "utf-8 latin-1 utf-16 utf-16-be utf-16-le".split(): self.checkencode(unichr(i), encoding) def test_random(self): @@ -165,4 +165,4 @@ self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) def test_latin1_error(self): - self.checkencodeerror(u"abc\uffff\uffff\uffffcde", "latin1", 3, 6) + self.checkencodeerror(u"abc\uffff\uffff\uffffcde", "latin-1", 3, 6) From cfbolz at codespeak.net Tue Nov 13 00:10:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 00:10:08 +0100 (CET) Subject: [pypy-svn] r48624 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112231008.4BBB082D4@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 00:10:08 2007 New Revision: 48624 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Log: a sanity check Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Tue Nov 13 00:10:08 2007 @@ -200,6 +200,7 @@ def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) + assert hasattr(runicode, rname) def wrap_encoder(space, uni, errors="strict"): state = space.fromcache(CodecState) func = getattr(runicode, rname) @@ -210,6 +211,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) + assert hasattr(runicode, rname) def wrap_decoder(space, string, errors="strict", w_final=True): final = space.is_true(w_final) state = space.fromcache(CodecState) From cfbolz at codespeak.net Tue Nov 13 00:19:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 00:19:04 +0100 (CET) Subject: [pypy-svn] r48625 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112231904.8154580FF@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 00:19:03 2007 New Revision: 48625 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Log: typo Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Tue Nov 13 00:19:03 2007 @@ -163,7 +163,7 @@ able to handle ValueErrors. """ if encoding is None: - encoding = sys.getdefaultencoding() + encoding = space.sys.defaultencoding w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1)) if space.is_true(w_decoder): w_res = space.call_function(w_decoder, w_obj, space.wrap(errors)) From cfbolz at codespeak.net Tue Nov 13 00:27:54 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 00:27:54 +0100 (CET) Subject: [pypy-svn] r48626 - pypy/branch/more-unicode-improvements/pypy/rlib Message-ID: <20071112232754.F16C882CA@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 00:27:50 2007 New Revision: 48626 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Log: the annotator doesn't like reading from the sys module directly Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 00:27:50 2007 @@ -2,6 +2,7 @@ from pypy.lang.smalltalk.tool.bitmanipulation import splitter MAXUNICODE = sys.maxunicode +BYTEORDER = sys.byteorder def raise_unicode_exception(errors, encoding, msg, s, startingpos, endingpos, @@ -184,7 +185,7 @@ bo = 0 consumed = 0 - if sys.byteorder == 'little': + if BYTEORDER == 'little': ihi = 1 ilo = 0 else: @@ -200,7 +201,7 @@ if byteorder == 'native': if (size >= 2): bom = (ord(s[ihi]) << 8) | ord(s[ilo]) - if sys.byteorder == 'little': + if BYTEORDER == 'little': if (bom == 0xFEFF): pos += 2 bo = -1 @@ -405,8 +406,8 @@ byteorder='little'): p = [] if (byteorder == 'native'): - _STORECHAR(p, 0xFEFF, sys.byteorder) - byteorder = sys.byteorder + _STORECHAR(p, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER if size == 0: return "" From cfbolz at codespeak.net Tue Nov 13 00:43:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 00:43:35 +0100 (CET) Subject: [pypy-svn] r48627 - pypy/branch/more-unicode-improvements/pypy/module/_codecs Message-ID: <20071112234335.94B0482A1@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 00:43:34 2007 New Revision: 48627 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Log: more typos Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Tue Nov 13 00:43:34 2007 @@ -12,7 +12,7 @@ def make_errorhandler(self, space): def unicode_call_errorhandler(errors, encoding, reason, input, - startinpos, endinpos, decode=True): + startpos, endpos, decode=True): w_errorhandler = lookup_error(space, errors) if decode: @@ -23,8 +23,8 @@ w_cls, space.wrap(encoding), space.wrap(input), - space.wrap(startinpos), - space.wrap(endingpos), + space.wrap(startpos), + space.wrap(endpos), space.wrap(reason)) w_res = space.call_function(w_errorhandler, w_exc) try: From fijal at codespeak.net Tue Nov 13 09:39:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 09:39:56 +0100 (CET) Subject: [pypy-svn] r48629 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113083956.5203482B3@code0.codespeak.net> Author: fijal Date: Tue Nov 13 09:39:56 2007 New Revision: 48629 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Update abstract in order to fulfill the pycon requirements. Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Tue Nov 13 09:39:56 2007 @@ -3,11 +3,25 @@ ================================================================================ (* for particularly nerdy dreams :-) ) +Summary (max 100 words, for the website): + +In this talk we would like to outline pypy's unique features, namely +sandboxing, transparent proxy, our distribution prototype and our security +prototype, the taint objspace. We would also like to summarize our design +decision which make it possible for such features to be non-pervasive to the +core interpreter codebase. Those features also works out of the box on +different pypy target platforms (C/POSIX, .NET, JVM) from the single source +interpreter. + +Description and outline: + In this talk, we would like to present our paradigm-shifting architecture for dynamic languages and some features which are exclusively available in PyPy, due to our novel approach. -Topics include: +Talk outline: + +* Quick recap of our motivation for creating pypy. * The transparent proxy, which allows programmers to create indistinguishable proxies for any type in Python (including built-in From fijal at codespeak.net Tue Nov 13 09:40:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 09:40:20 +0100 (CET) Subject: [pypy-svn] r48630 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113084020.D021582B3@code0.codespeak.net> Author: fijal Date: Tue Nov 13 09:40:20 2007 New Revision: 48630 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Rewording Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Tue Nov 13 09:40:20 2007 @@ -1,5 +1,5 @@ XXX Understanding PyPy, and How It Let's You Do Things You Have Only Dreamed About. (*) -XXX The PyPy way and the tales of nice features and permissive(?) design +XXX The PyPy way and the tales of nice features and pleasant design ================================================================================ (* for particularly nerdy dreams :-) ) From fijal at codespeak.net Tue Nov 13 09:43:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 09:43:04 +0100 (CET) Subject: [pypy-svn] r48631 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113084304.D0DAB82B5@code0.codespeak.net> Author: fijal Date: Tue Nov 13 09:43:04 2007 New Revision: 48631 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Update the description. I think it's important to tell this. Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Tue Nov 13 09:43:04 2007 @@ -17,7 +17,9 @@ In this talk, we would like to present our paradigm-shifting architecture for dynamic languages and some features which are exclusively available in -PyPy, due to our novel approach. +PyPy, due to our novel approach. Also thanks to our design, those features +are available on all of our target platforms (C/POSIX, .NET, JVM) and are +independent on other decisions (garbage collector, stackless, JIT, etc.) Talk outline: From cfbolz at codespeak.net Tue Nov 13 10:05:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 10:05:41 +0100 (CET) Subject: [pypy-svn] r48632 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071113090541.85B5D82AE@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 10:05:41 2007 New Revision: 48632 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: test + fix Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 10:05:41 2007 @@ -332,7 +332,7 @@ if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : ch3 = ((ord(ch) - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 i += 1 - _encodeUCS4(p, ch) + _encodeUCS4(p, ord(ch)) continue # Fall through: handles isolated high surrogates p += (chr((0xe0 | (ord(ch) >> 12)))) Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Tue Nov 13 10:05:41 2007 @@ -158,8 +158,11 @@ self.checkencode(unichr(i), encoding) def test_single_chars_utf8(self): + # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkencode(s, "utf-8") + # check replacing of two surrogates by single char while encoding + self.checkencode(u"\ud800\udc10", "utf-8") def test_ascii_error(self): self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) From fijal at codespeak.net Tue Nov 13 10:10:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 10:10:53 +0100 (CET) Subject: [pypy-svn] r48633 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113091053.C2FEB82CB@code0.codespeak.net> Author: fijal Date: Tue Nov 13 10:10:53 2007 New Revision: 48633 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Inteded audience paragraph. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Tue Nov 13 10:10:53 2007 @@ -1,6 +1,18 @@ -* Tutorial title. +Hands on PyPy - how to modify your virtual machine in 15 minutes +================================================================ + * Presenter name(s) & contact information (email, telephone). + * Intended audience (difficulty level, experience required): non-programmers, beginning programmers, advanced users, CPython developers, etc. + +Intended audience +================= + +This tutorial is intended for people who has a desire to modify python's +virtual machine, being uncomfortable with some design decisions, want an +extra feature or just a wish to experiment. Python fluency, including +prior experience with meta-programming will be expected from atendees. + * Tutorial format description (lecture, interactive lecture, Q&A session [master class], hands-on, etc.). Be specific. * Recording: indicate if you are amenable to your tutorial being recorded. "I give permission to record and publish my PyCon tutorial for free distribution." * Requirements: A list of software and hardware requirements for the tutorial. (Laptop required or recommended? Python installed? Which version? 3rd-party packages required? Provide links & versions.) From cfbolz at codespeak.net Tue Nov 13 10:50:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 10:50:38 +0100 (CET) Subject: [pypy-svn] r48635 - pypy/branch/more-unicode-improvements/pypy/rlib/test Message-ID: <20071113095038.256AA82D2@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 10:50:37 2007 New Revision: 48635 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: make the test more reliable: don't let CPython's unmarshaller renormalize the string when reloading it from a pyc file. Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Tue Nov 13 10:50:37 2007 @@ -161,8 +161,14 @@ # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: self.checkencode(s, "utf-8") + + def test_utf8_surrogates(self): # check replacing of two surrogates by single char while encoding - self.checkencode(u"\ud800\udc10", "utf-8") + # make sure that the string itself is not marshalled + u = u"\ud800" + for i in range(4): + u += u"\udc00" + self.checkencode(u, "utf-8") def test_ascii_error(self): self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) From cfbolz at codespeak.net Tue Nov 13 10:51:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 10:51:09 +0100 (CET) Subject: [pypy-svn] r48636 - pypy/branch/more-unicode-improvements/pypy/rlib Message-ID: <20071113095109.EB18F82D5@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 10:51:09 2007 New Revision: 48636 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Log: fix the test plus a bit of niceification: call ord(ch) only once. Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 10:51:09 2007 @@ -313,34 +313,35 @@ i = 0 while i < size: ch = s[i] + ordch = ord(ch) i += 1 - if (ord(ch) < 0x80): + if (ordch < 0x80): # Encode ASCII - p += chr(ord(ch)) - elif (ord(ch) < 0x0800) : + p += chr(ordch) + elif (ordch < 0x0800) : # Encode Latin-1 - p += chr((0xc0 | (ord(ch) >> 6))) - p += chr((0x80 | (ord(ch) & 0x3f))) + p += chr((0xc0 | (ordch >> 6))) + p += chr((0x80 | (ordch & 0x3f))) else: # Encode UCS2 Unicode ordinals - if (ord(ch) < 0x10000): + if (ordch < 0x10000): # Special case: check for high surrogate - if (0xD800 <= ord(ch) and ord(ch) <= 0xDBFF and i != size) : + if (0xD800 <= ordch and ordch <= 0xDBFF and i != size) : ch2 = s[i] # Check for low surrogate and combine the two to # form a UCS4 value if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : - ch3 = ((ord(ch) - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 + ch3 = ((ordch - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 i += 1 - _encodeUCS4(p, ord(ch)) + _encodeUCS4(p, ch3) continue # Fall through: handles isolated high surrogates - p += (chr((0xe0 | (ord(ch) >> 12)))) - p += (chr((0x80 | ((ord(ch) >> 6) & 0x3f)))) - p += (chr((0x80 | (ord(ch) & 0x3f)))) + p += (chr((0xe0 | (ordch >> 12)))) + p += (chr((0x80 | ((ordch >> 6) & 0x3f)))) + p += (chr((0x80 | (ordch & 0x3f)))) continue else: - _encodeUCS4(p, ord(ch)) + _encodeUCS4(p, ordch) return "".join(p) def _encodeUCS4(p, ch): From antocuni at codespeak.net Tue Nov 13 11:01:21 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 13 Nov 2007 11:01:21 +0100 (CET) Subject: [pypy-svn] r48638 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20071113100121.DC77882D3@code0.codespeak.net> Author: antocuni Date: Tue Nov 13 11:01:21 2007 New Revision: 48638 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/test/test_string.py pypy/dist/pypy/translator/jvm/test/test_unicode.py Log: proper handling of unicode strings for genjvm Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Tue Nov 13 11:01:21 2007 @@ -1100,7 +1100,7 @@ if value == ootype.null(TYPE): self.emit(ACONST_NULL) else: - self.load_string(str(value._str)) + self.load_string(value._str) else: assert False, 'Unknown constant type: %s' % TYPE Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Tue Nov 13 11:01:21 2007 @@ -304,23 +304,15 @@ return "False"; } - public static void _append_char(StringBuffer sb, char c) { - if (c == '"') - sb.append("\\\""); - else - sb.append(c); + private static String format_char(char c) { + String res = "\\x"; + if (c <= 0x0F) res = res + "0"; + res = res + Integer.toHexString(c); + return res; } public static String escaped_char(char c) { - StringBuffer sb = new StringBuffer(); - sb.append('"'); - _append_char(sb, c); - sb.append('"'); - return sb.toString(); - } - - public static String escaped_unichar(char c) { - return "u" + escaped_char(c); + return "'" + format_char(c) + "'"; } public static String escaped_string(String b) { @@ -330,14 +322,37 @@ sb.append('"'); for (int i = 0; i < b.length(); i++) { char c = b.charAt(i); - _append_char(sb, c); + sb.append(format_char(c)); } sb.append('"'); return sb.toString(); } + private static String format_unichar(char c) { + String res = "\\u"; + if (c <= 0xF) res = res + "0"; + if (c <= 0xFF) res = res + "0"; + if (c <= 0xFFF) res = res + "0"; + res = res + Integer.toHexString(c); + return res; + } + + public static String escaped_unichar(char c) + { + return "u'" + format_unichar(c) + "'"; + } + public static String escaped_unicode(String b) { - return "u" + escaped_string(b); + if (b == null) + return "None"; + StringBuffer sb = new StringBuffer(); + sb.append("u'"); + for (int i = 0; i < b.length(); i++) { + char c = b.charAt(i); + sb.append(format_unichar(c)); + } + sb.append("'"); + return sb.toString(); } // used in running unit tests Modified: pypy/dist/pypy/translator/jvm/test/test_string.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_string.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_string.py Tue Nov 13 11:01:21 2007 @@ -24,3 +24,10 @@ # but we don't bother to make runtest understand how to parse that, # so we just skip the test. py.test.skip("test fails in JVM specific way") + + def test_string_constant(self): + const = ''.join(map(chr, range(0, 256))) + def fn(): + return const + res = self.interpret(fn, []) + assert res == const Modified: pypy/dist/pypy/translator/jvm/test/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_unicode.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_unicode.py Tue Nov 13 11:01:21 2007 @@ -9,12 +9,24 @@ EMPTY_STRING_HASH = 0 def test_unichar_const(self): + def fn(): + return u'\u03b1' + assert self.interpret(fn, []) == u'\u03b1' + + def test_unichar_eq(self): py.test.skip("JVM doesn't support unicode for command line arguments") - test_unichar_eq = test_unichar_const - test_unichar_ord = test_unichar_const - test_unichar_hash = test_unichar_const - test_char_unichar_eq = test_unichar_const - test_char_unichar_eq_2 = test_unichar_const + test_unichar_ord = test_unichar_eq + test_unichar_hash = test_unichar_eq + test_char_unichar_eq = test_unichar_eq + test_char_unichar_eq_2 = test_unichar_eq def test_getitem_exc(self): py.test.skip('fixme!') + + def test_unicode_constant(self): + const = u''.join(map(unichr, range(0, 256))) + const = const + u'\ufffd' + def fn(): + return const + res = self.interpret(fn, []) + assert res == const From cfbolz at codespeak.net Tue Nov 13 11:13:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 11:13:13 +0100 (CET) Subject: [pypy-svn] r48639 - pypy/branch/more-unicode-improvements/pypy/rlib Message-ID: <20071113101313.1D27D82D2@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 11:13:12 2007 New Revision: 48639 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Log: be a bit more consistent in variable names Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 11:13:12 2007 @@ -312,36 +312,35 @@ p = [] i = 0 while i < size: - ch = s[i] - ordch = ord(ch) + ch = ord(s[i]) i += 1 - if (ordch < 0x80): + if (ch < 0x80): # Encode ASCII - p += chr(ordch) - elif (ordch < 0x0800) : + p += chr(ch) + elif (ch < 0x0800) : # Encode Latin-1 - p += chr((0xc0 | (ordch >> 6))) - p += chr((0x80 | (ordch & 0x3f))) + p += chr((0xc0 | (ch >> 6))) + p += chr((0x80 | (ch & 0x3f))) else: # Encode UCS2 Unicode ordinals - if (ordch < 0x10000): + if (ch < 0x10000): # Special case: check for high surrogate - if (0xD800 <= ordch and ordch <= 0xDBFF and i != size) : - ch2 = s[i] + if (0xD800 <= ch and ch <= 0xDBFF and i != size) : + ch2 = ord(s[i]) # Check for low surrogate and combine the two to # form a UCS4 value - if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : - ch3 = ((ordch - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 + if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : + ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 i += 1 _encodeUCS4(p, ch3) continue # Fall through: handles isolated high surrogates - p += (chr((0xe0 | (ordch >> 12)))) - p += (chr((0x80 | ((ordch >> 6) & 0x3f)))) - p += (chr((0x80 | (ordch & 0x3f)))) + p += (chr((0xe0 | (ch >> 12)))) + p += (chr((0x80 | ((ch >> 6) & 0x3f)))) + p += (chr((0x80 | (ch & 0x3f)))) continue else: - _encodeUCS4(p, ordch) + _encodeUCS4(p, ch) return "".join(p) def _encodeUCS4(p, ch): From fijal at codespeak.net Tue Nov 13 11:52:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 11:52:20 +0100 (CET) Subject: [pypy-svn] r48641 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113105220.17E6382E1@code0.codespeak.net> Author: fijal Date: Tue Nov 13 11:52:19 2007 New Revision: 48641 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Some update on the tutorial. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Tue Nov 13 11:52:19 2007 @@ -3,19 +3,37 @@ * Presenter name(s) & contact information (email, telephone). -* Intended audience (difficulty level, experience required): non-programmers, beginning programmers, advanced users, CPython developers, etc. - Intended audience ================= This tutorial is intended for people who has a desire to modify python's virtual machine, being uncomfortable with some design decisions, want an -extra feature or just a wish to experiment. Python fluency, including -prior experience with meta-programming will be expected from atendees. +extra feature, fast modules not written in C or just a wish to experiment. +Python fluency, including prior experience with meta-programming will be +expected from atendees. No knowledge of C is needed nor required, although +prior knowledge of any virtual machine (for example) CPython might be +useful. + +Tutorial format description +=========================== + +XXX rough sketch +* half an hour of introduction and quick codebase skimming +* one hour of modifying pypy's python interpreter (what exactly XXX) +* an hour and a half of hands-on session, including pre-prepared + topics. + +Recording +========= + +I give permission to record and publish my PyCon tutorial for free +distribution. + +Requirements +============ + +XXX prepare -* Tutorial format description (lecture, interactive lecture, Q&A session [master class], hands-on, etc.). Be specific. -* Recording: indicate if you are amenable to your tutorial being recorded. "I give permission to record and publish my PyCon tutorial for free distribution." -* Requirements: A list of software and hardware requirements for the tutorial. (Laptop required or recommended? Python installed? Which version? 3rd-party packages required? Provide links & versions.) * Notes for reviewers. * Promotional summary (max. 100 words). * Detailed tutorial outline, for review (include timing). From fijal at codespeak.net Tue Nov 13 12:26:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 12:26:07 +0100 (CET) Subject: [pypy-svn] r48643 - pypy/dist/pypy/translator/c Message-ID: <20071113112607.EB8C782CF@code0.codespeak.net> Author: fijal Date: Tue Nov 13 12:26:06 2007 New Revision: 48643 Modified: pypy/dist/pypy/translator/c/extfunc.py Log: Revert flush_icache removal Modified: pypy/dist/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/dist/pypy/translator/c/extfunc.py Tue Nov 13 12:26:06 2007 @@ -22,6 +22,7 @@ ll_stackless.ll_stackless_stack_frames_depth: 'LL_stackless_stack_frames_depth', ll_stack.ll_stack_unwind: 'LL_stack_unwind', ll_stack.ll_stack_too_big: 'LL_stack_too_big', + 'LL_flush_icache': 'LL_flush_icache', } #______________________________________________________ From cfbolz at codespeak.net Tue Nov 13 12:38:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 12:38:01 +0100 (CET) Subject: [pypy-svn] r48644 - pypy/branch/more-unicode-improvements/pypy/rlib Message-ID: <20071113113801.576B582D2@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 12:38:00 2007 New Revision: 48644 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Log: more speaking var names. don't use += for list appending Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 12:38:00 2007 @@ -42,13 +42,13 @@ errorhandler=raise_unicode_exception): if (size == 0): return u'', 0 - p = [] + result = [] pos = 0 while pos < size: ch = s[pos] ordch1 = ord(ch) if ordch1 < 0x80: - p += unichr(ordch1) + result.append(unichr(ordch1)) pos += 1 continue @@ -59,14 +59,13 @@ else: r, pos = errorhandler(errors, "utf-8", "unexpected end of data", s, pos, size) - p += r + result.append(r) if (pos + n > size): break if n == 0: - res = errorhandler(errors, "utf-8", "unexpected code byte", - s, pos, pos + 1) - p += res[0] - pos = res[1] + r, pos = errorhandler(errors, "utf-8", "unexpected code byte", + s, pos, pos + 1) + result.append(r) elif n == 1: assert 0, "you can never get here" elif n == 2: @@ -79,15 +78,15 @@ if (two != 2): r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 2) - p += r + result.append(r) else: c = (y << 6) + z if c < 0x80: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 2) - p += r + result.append(r) else: - p += unichr(c) + result.append(unichr(c)) pos += n elif n == 3: # 1110xxxx 10yyyyyy 10zzzzzz ====> 00000000 xxxxyyyy yyzzzzzz @@ -100,7 +99,7 @@ if (two1 != 2 or two2 != 2): r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 3) - p += r + result.append(r) else: c = (x << 12) + (y << 6) + z # Note: UTF-8 encodings of surrogates are considered @@ -111,9 +110,9 @@ if c < 0x0800: r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 3) - p += r + result.append(r) else: - p += unichr(c) + result.append(unichr(c)) pos += n elif n == 4: # 11110www 10xxxxxx 10yyyyyy 10zzzzzz ====> @@ -129,7 +128,7 @@ if (two1 != 2 or two2 != 2 or two3 != 2): r, pos = errorhandler(errors, "utf-8", "invalid data", s, pos, pos + 4) - p += r + result.append(r) else: c = (w << 18) + (x << 12) + (y << 6) + z # minimum value allowed for 4 byte encoding @@ -137,27 +136,27 @@ if ((c < 0x10000) or (c > 0x10ffff)): r, pos = errorhandler(errors, "utf-8", "illegal encoding", s, pos, pos + 4) - p += r + result.append(r) else: # convert to UTF-16 if necessary if c < MAXUNICODE: - p.append(unichr(c)) + result.append(unichr(c)) else: # compute and append the two surrogates: # translate from 10000..10FFFF to 0..FFFF c -= 0x10000 # high surrogate = top 10 bits added to D800 - p.append(unichr(0xD800 + (c >> 10))) + result.append(unichr(0xD800 + (c >> 10))) # low surrogate = bottom 10 bits added to DC00 - p.append(unichr(0xDC00 + (c & 0x03FF))) + result.append(unichr(0xDC00 + (c & 0x03FF))) pos += n else: r, pos = errorhandler(errors, "utf-8", "unsupported Unicode code range", s, pos, pos + n) - p += r + result.append(r) - return u"".join(p), pos + return u"".join(result), pos def str_decode_utf_16(s, size, errors, final=True, @@ -245,7 +244,7 @@ ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo]) pos += 2 if (ch < 0xD800 or ch > 0xDFFF): - result += unichr(ch) + result.append(unichr(ch)) continue # UTF-16 code pair: if len(s) - pos < 2: @@ -261,10 +260,11 @@ pos += 2 if (0xDC00 <= ch2 and ch2 <= 0xDFFF): if MAXUNICODE < 65536: - result += unichr(ch) - result += unichr(ch2) + result.append(unichr(ch)) + result.append(unichr(ch2)) else: - result += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) + result.append(unichr((((ch & 0x3FF)<<10) | + (ch2 & 0x3FF)) + 0x10000)) continue else: r, pos = errorhandler(errors, 'utf-16', @@ -279,28 +279,28 @@ errorhandler=raise_unicode_exception): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 - p = [] + result = [] while (pos < size): - p += unichr(ord(s[pos])) + result.append(unichr(ord(s[pos]))) pos += 1 - return u"".join(p), pos + return u"".join(result), pos def str_decode_ascii(s, size, errors, final=False, errorhandler=raise_unicode_exception): # ASCII is equivalent to the first 128 ordinals in Unicode. - p = [] + result = [] pos = 0 while pos < len(s): c = s[pos] if ord(c) < 128: - p += unichr(ord(c)) + result.append(unichr(ord(c))) pos += 1 else: r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1) - p += r - return u"".join(p), pos + result.append(r) + return u"".join(result), pos # ____________________________________________________________ @@ -309,18 +309,18 @@ def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception): assert(size >= 0) - p = [] + result = [] i = 0 while i < size: ch = ord(s[i]) i += 1 if (ch < 0x80): # Encode ASCII - p += chr(ch) + result.append(chr(ch)) elif (ch < 0x0800) : # Encode Latin-1 - p += chr((0xc0 | (ch >> 6))) - p += chr((0x80 | (ch & 0x3f))) + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) else: # Encode UCS2 Unicode ordinals if (ch < 0x10000): @@ -332,23 +332,23 @@ if (0xDC00 <= ch2 and ch2 <= 0xDFFF) : ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000 i += 1 - _encodeUCS4(p, ch3) + _encodeUCS4(result, ch3) continue # Fall through: handles isolated high surrogates - p += (chr((0xe0 | (ch >> 12)))) - p += (chr((0x80 | ((ch >> 6) & 0x3f)))) - p += (chr((0x80 | (ch & 0x3f)))) + result.append((chr((0xe0 | (ch >> 12))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) continue else: - _encodeUCS4(p, ch) - return "".join(p) + _encodeUCS4(result, ch) + return "".join(result) -def _encodeUCS4(p, ch): +def _encodeUCS4(result, ch): # Encode UCS4 Unicode ordinals - p += (chr((0xf0 | (ch >> 18)))) - p += (chr((0x80 | ((ch >> 12) & 0x3f)))) - p += (chr((0x80 | ((ch >> 6) & 0x3f)))) - p += (chr((0x80 | (ch & 0x3f)))) + result.append((chr((0xf0 | (ch >> 18))))) + result.append((chr((0x80 | ((ch >> 12) & 0x3f))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) def unicode_encode_ucs1_helper(p, size, errors, @@ -362,13 +362,13 @@ if (size == 0): return '' - res = [] + result = [] pos = 0 while pos < len(p): ch = p[pos] if ord(ch) < limit: - res += chr(ord(ch)) + result.append(chr(ord(ch))) pos += 1 else: # startpos for collecting unencodable chars @@ -378,9 +378,9 @@ collend += 1 r, pos = errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += r + result.append(r) - return "".join(res) + return "".join(result) def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) @@ -391,22 +391,22 @@ return res -def _STORECHAR(p, CH, byteorder): +def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) lo = chr((CH) & 0xff) if byteorder == 'little': - p.append(lo) - p.append(hi) + result.append(lo) + result.append(hi) else: - p.append(hi) - p.append(lo) + result.append(hi) + result.append(lo) def unicode_encode_utf_16_helper(s, size, errors, errorhandler=raise_unicode_exception, byteorder='little'): - p = [] + result = [] if (byteorder == 'native'): - _STORECHAR(p, 0xFEFF, BYTEORDER) + _STORECHAR(result, 0xFEFF, BYTEORDER) byteorder = BYTEORDER if size == 0: @@ -421,11 +421,11 @@ ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) ch = 0xD800 | ((ch-0x10000) >> 10) - _STORECHAR(p, ch, byteorder) + _STORECHAR(result, ch, byteorder) if ch2: - _STORECHAR(p, ch2, byteorder) + _STORECHAR(result, ch2, byteorder) - return "".join(p) + return "".join(result) def unicode_encode_utf_16(s, size, errors, errorhandler=raise_unicode_exception): From rxe at codespeak.net Tue Nov 13 12:57:12 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 13 Nov 2007 12:57:12 +0100 (CET) Subject: [pypy-svn] r48645 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071113115712.B50F582CB@code0.codespeak.net> Author: rxe Date: Tue Nov 13 12:57:12 2007 New Revision: 48645 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: add myself Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Tue Nov 13 12:57:12 2007 @@ -17,6 +17,7 @@ Maciej Fijalkowski 19-26 Laura and Jacob's Armin Rigo lives thereish Alexander Schremmer 19-26 Laura and Jacob's +Richard Emslie 19-26 Laura and Jacob's ==================== ============== ===================== @@ -32,7 +33,6 @@ Eric van Riet paap ? ? Niko Matsakis ? ? Leonardo Santagada ? ? -Richard Emslie ? ? Aurelien Campeas ? ? Alexandre Fayolle ? ? Lene Wagner ? ? From fijal at codespeak.net Tue Nov 13 13:18:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 13:18:18 +0100 (CET) Subject: [pypy-svn] r48646 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113121818.7784C82D3@code0.codespeak.net> Author: fijal Date: Tue Nov 13 13:18:16 2007 New Revision: 48646 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Update. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Tue Nov 13 13:18:16 2007 @@ -11,17 +11,14 @@ extra feature, fast modules not written in C or just a wish to experiment. Python fluency, including prior experience with meta-programming will be expected from atendees. No knowledge of C is needed nor required, although -prior knowledge of any virtual machine (for example) CPython might be +prior knowledge of any virtual machine (for example CPython) might be useful. Tutorial format description =========================== -XXX rough sketch -* half an hour of introduction and quick codebase skimming -* one hour of modifying pypy's python interpreter (what exactly XXX) -* an hour and a half of hands-on session, including pre-prepared - topics. +A brief introduction to pypy codebase and concepts, then hands-on session +with pre-prepared tasks. Recording ========= @@ -34,9 +31,33 @@ XXX prepare +Promotional summary +=================== + +Ever missed feature in your Python interpreter? Ever wondered how to write +extensions in Python rather than C or Pyrex? Ever wondered how pypy works? + +This tutorial will give you detailed description how the PyPy python +interpreter is constructed and how to change it to meet your needs. +It'll also contain simple changes to the interpreter codebase as live coding +session. Afterwards will be welcomed to experiment in groups on they're own +on implementing prepared interpreter changes. + +Attendees are welcome to bring their own laptops. + +Outline +======= + +XXX expand +* half an hour of introduction and quick codebase skimming +* one hour of modifying pypy's python interpreter (what exactly XXX) +* an hour and a half of hands-on session, including pre-prepared + topics. + * Notes for reviewers. -* Promotional summary (max. 100 words). -* Detailed tutorial outline, for review (include timing). + * Tutorial outline for website publication, if different from the above. + * Presenter bio (one paragraph), written in the third person ("Presenter Name is..."). Include where you live, your job, your projects, books you've written, volunteer activities, family, etc. + * Previous experience: references of presentation/teaching engagements (sessions taught at OSCON, EuroPython, local user groups, at work, etc.). Extra consideration will be given to presenters with prior experience. From exarkun at codespeak.net Tue Nov 13 13:20:47 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 13 Nov 2007 13:20:47 +0100 (CET) Subject: [pypy-svn] r48647 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113122047.760B782D3@code0.codespeak.net> Author: exarkun Date: Tue Nov 13 13:20:47 2007 New Revision: 48647 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: spelling/grammar edits, a couple notes for areas where minor improvements might be warranted Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Tue Nov 13 13:20:47 2007 @@ -1,36 +1,36 @@ -XXX Understanding PyPy, and How It Let's You Do Things You Have Only Dreamed About. (*) +XXX Understanding PyPy, and How It Lets You Do Things You Have Only Dreamed About. (*) XXX The PyPy way and the tales of nice features and pleasant design ================================================================================ (* for particularly nerdy dreams :-) ) Summary (max 100 words, for the website): -In this talk we would like to outline pypy's unique features, namely -sandboxing, transparent proxy, our distribution prototype and our security -prototype, the taint objspace. We would also like to summarize our design -decision which make it possible for such features to be non-pervasive to the +In this talk we will outline PyPy's unique features, namely +sandboxing, transparent proxying, our distribution prototype and our security +prototype, the taint objspace (XXX "objspace" is obscure - might want to say something a non-pypy developer can understand here). We would also like to summarize our design +decisions which make it possible for such features to be non-pervasive to the core interpreter codebase. Those features also works out of the box on -different pypy target platforms (C/POSIX, .NET, JVM) from the single source -interpreter. +different PyPy target platforms (C/POSIX, .NET, JVM) from the single +interpreter source. Description and outline: In this talk, we would like to present our paradigm-shifting architecture for dynamic languages and some features which are exclusively available in -PyPy, due to our novel approach. Also thanks to our design, those features +PyPy due to our novel approach. Also thanks to our design, those features are available on all of our target platforms (C/POSIX, .NET, JVM) and are -independent on other decisions (garbage collector, stackless, JIT, etc.) +independent of other decisions (garbage collector, stackless, JIT, etc.) Talk outline: -* Quick recap of our motivation for creating pypy. +* Quick recap of our motivation for creating PyPy. * The transparent proxy, which allows programmers to create indistinguishable proxies for any type in Python (including built-in python types like frames), with various usage showcases including our completely transparent remote access scheme. -* the PyPy Sandbox, which allows to run a non-stripped version of +* the PyPy Sandbox, which allows one to run a non-stripped (XXX not clear what "non-stripped" means) version of our Python interpreter in a controlled environment with a custom safety policy. @@ -41,4 +41,4 @@ require pervasive changes to the interpreter core. Why our design allows us to describe a broad class of such changes as a "proxying operation" and finally, why design matters when you want - to get a butterfly out of the caterpillar instead of a bigger caterpillar. + to get a butterfly out of a caterpillar instead of a bigger caterpillar. From lac at codespeak.net Tue Nov 13 15:28:33 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Tue, 13 Nov 2007 15:28:33 +0100 (CET) Subject: [pypy-svn] r48648 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113142833.1B21482AB@code0.codespeak.net> Author: lac Date: Tue Nov 13 15:28:31 2007 New Revision: 48648 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: I think this is more like what you wanted. If not, revert. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Tue Nov 13 15:28:31 2007 @@ -1,18 +1,25 @@ Hands on PyPy - how to modify your virtual machine in 15 minutes ================================================================ +XXX We need a sexier title. + * Presenter name(s) & contact information (email, telephone). Intended audience ================= -This tutorial is intended for people who has a desire to modify python's -virtual machine, being uncomfortable with some design decisions, want an -extra feature, fast modules not written in C or just a wish to experiment. -Python fluency, including prior experience with meta-programming will be -expected from atendees. No knowledge of C is needed nor required, although -prior knowledge of any virtual machine (for example CPython) might be -useful. +This tutorial is intended for people who desire to modify python's +virtual machine. + +Perhaps you are uncomfortable with some design decisions? want an +extra feature? desire fast modules but don't want to write them +in C? or just wish to experiment? + +Attendees will be expected to be fluent in Python, and have prior +experience with meta-programming (in any language). No knowledge of C +is needed nor required, although prior knowledge of any virtual +machine (for example CPython) might be useful. If you are wondering +'what's a virtual machine' then this tutorial is not for you. Tutorial format description =========================== From cfbolz at codespeak.net Tue Nov 13 15:34:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 15:34:32 +0100 (CET) Subject: [pypy-svn] r48649 - in pypy/branch/more-unicode-improvements/pypy/rlib: . test Message-ID: <20071113143432.0FE5282DA@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 15:34:32 2007 New Revision: 48649 Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Log: fix bug in utf-16-decoder Modified: pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py Tue Nov 13 15:34:32 2007 @@ -9,12 +9,10 @@ decode=True): if decode: raise UnicodeDecodeError( - "%s can't decode byte %s in position %s: %s" % ( - encoding, s[startingpos], startingpos, msg)) + encoding, s[startingpos], startingpos, endingpos, msg) else: raise UnicodeEncodeError( - "%s can't encode byte %s in position %s: %s" % ( - encoding, s[startingpos], startingpos, msg)) + encoding, s[startingpos], startingpos, endingpos, msg) # ____________________________________________________________ # unicode decoding @@ -272,7 +270,10 @@ s, pos - 4, pos - 2) result.append(r) else: - assert 0, "unreachable" + r, pos = errorhandler(errors, 'utf-16', + "illegal encoding", + s, pos - 2, pos) + result.append(r) return u"".join(result), pos, bo def str_decode_latin_1(s, size, errors, final=False, Modified: pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py Tue Nov 13 15:34:32 2007 @@ -1,3 +1,4 @@ +import py import sys, random from pypy.rlib import runicode @@ -140,6 +141,12 @@ ]: self.checkdecodeerror(s, "utf-16", 2, 4, addstuff=False) + def test_utf16_bugs(self): + s = '\x80-\xe9\xdeL\xa3\x9b' + py.test.raises(UnicodeDecodeError, runicode.str_decode_utf_16_le, + s, len(s), True) + + class TestEncoding(UnicodeTests): def test_all_ascii(self): for i in range(128): @@ -175,3 +182,14 @@ def test_latin1_error(self): self.checkencodeerror(u"abc\uffff\uffff\uffffcde", "latin-1", 3, 6) + +class TestTranslation(object): + def test_utf8(self): + from pypy.rpython.test.test_llinterp import interpret + def f(x): + s1 = "\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93" * x + u = runicode.str_decode_utf_8(s1, len(s1), True) + s2 = runicode.unicode_encode_utf_8(u, len(u), True) + return s1 == s2 + res = interpret(f, [2]) + assert res From cfbolz at codespeak.net Tue Nov 13 16:08:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 16:08:05 +0100 (CET) Subject: [pypy-svn] r48650 - in pypy/branch/more-unicode-improvements/pypy/objspace/std: . test Message-ID: <20071113150805.EDF3382B8@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 16:08:04 2007 New Revision: 48650 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Log: refactor repr__Unicode, which seems to be written with the assumption that RPython strings didn't have overalloction. also fix a bug in the process. Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py Tue Nov 13 16:08:04 2007 @@ -458,4 +458,6 @@ # Error handling (truncated escape sequence) raises(UnicodeError, "\\".decode, "unicode-escape") - + def test_repr_bug(self): + assert (repr(u'\U00090418\u027d\U000582b9\u54c3\U000fcb6e') == + "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'") Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Tue Nov 13 16:08:04 2007 @@ -848,36 +848,23 @@ quote = '"' else: quote = '\'' - result = ['\0'] * (3 + size*6) - result[0] = 'u' - result[1] = quote - i = 2 + result = ['u', quote] j = 0 while j= 0x10000: # Resize if needed - if i + 12 > len(result): - result.extend(['\0'] * 100) - result[i] = '\\' - result[i + 1] = "U" - result[i + 2] = hexdigits[(code >> 28) & 0xf] - result[i + 3] = hexdigits[(code >> 24) & 0xf] - result[i + 4] = hexdigits[(code >> 20) & 0xf] - result[i + 5] = hexdigits[(code >> 16) & 0xf] - result[i + 6] = hexdigits[(code >> 12) & 0xf] - result[i + 7] = hexdigits[(code >> 8) & 0xf] - result[i + 8] = hexdigits[(code >> 4) & 0xf] - result[i + 9] = hexdigits[(code >> 0) & 0xf] - i += 10 + result.extend(['\\', "U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue if code >= 0xD800 and code < 0xDC00: @@ -886,70 +873,60 @@ code2 = ord(ch2) if code2 >= 0xDC00 and code2 <= 0xDFFF: code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 - if i + 12 > len(result): - result.extend(['\0'] * 100) - result[i] = '\\' - result[i + 1] = "U" - result[i + 2] = hexdigits[(code >> 28) & 0xf] - result[i + 3] = hexdigits[(code >> 24) & 0xf] - result[i + 4] = hexdigits[(code >> 20) & 0xf] - result[i + 5] = hexdigits[(code >> 16) & 0xf] - result[i + 6] = hexdigits[(code >> 12) & 0xf] - result[i + 7] = hexdigits[(code >> 8) & 0xf] - result[i + 8] = hexdigits[(code >> 4) & 0xf] - result[i + 9] = hexdigits[(code >> 0) & 0xf] - i += 10 + result.extend(["U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 2 continue if code >= 0x100: - result[i] = '\\' - result[i + 1] = "u" - result[i + 2] = hexdigits[(code >> 12) & 0xf] - result[i + 3] = hexdigits[(code >> 8) & 0xf] - result[i + 4] = hexdigits[(code >> 4) & 0xf] - result[i + 5] = hexdigits[(code >> 0) & 0xf] - i += 6 + result.extend(['\\', "u", + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue if code == ord('\\') or code == ord(quote): - result[i] = '\\' - result[i + 1] = chr(code) - i += 2 + result.append('\\') + result.append(chr(code)) j += 1 continue if code == ord('\t'): - result[i] = '\\' - result[i + 1] = "t" - i += 2 + result.append('\\') + result.append('t') j += 1 continue if code == ord('\r'): - result[i] = '\\' - result[i + 1] = "r" - i += 2 + result.append('\\') + result.append('r') j += 1 continue if code == ord('\n'): - result[i] = '\\' - result[i + 1] = "n" + result.append('\\') + result.append('n') i += 2 j += 1 continue if code < ord(' ') or code >= 0x7f: - result[i] = '\\' - result[i + 1] = "x" - result[i + 2] = hexdigits[(code >> 4) & 0xf] - result[i + 3] = hexdigits[(code >> 0) & 0xf] - i += 4 + result.extend(['\\', "x", + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue - result[i] = chr(code) - i += 1 + result.append(chr(code)) j += 1 - result[i] = quote - i += 1 - return space.wrap(''.join(result[:i])) + result.append(quote) + return space.wrap(''.join(result)) def mod__Unicode_ANY(space, w_format, w_values): From fijal at codespeak.net Tue Nov 13 16:11:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 16:11:11 +0100 (CET) Subject: [pypy-svn] r48651 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113151111.ED3BA82D2@code0.codespeak.net> Author: fijal Date: Tue Nov 13 16:11:11 2007 New Revision: 48651 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Resolve one XXX Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Tue Nov 13 16:11:11 2007 @@ -30,9 +30,9 @@ python types like frames), with various usage showcases including our completely transparent remote access scheme. -* the PyPy Sandbox, which allows one to run a non-stripped (XXX not clear what "non-stripped" means) version of - our Python interpreter in a controlled environment with a custom - safety policy. +* the PyPy Sandbox, which allows one to run a full python + interpreter (including free imports) in a controlled environment with a + custom safety policy, restrained on calls to C level. * The Taint Object Space, which allows programmers to make sure that sensitive information does not cross I/O barriers. From cfbolz at codespeak.net Tue Nov 13 16:32:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 16:32:39 +0100 (CET) Subject: [pypy-svn] r48652 - in pypy/branch/more-unicode-improvements/pypy/objspace/std: . test Message-ID: <20071113153239.7043882D6@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 16:32:38 2007 New Revision: 48652 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Log: bug + fix Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py Tue Nov 13 16:32:38 2007 @@ -461,3 +461,5 @@ def test_repr_bug(self): assert (repr(u'\U00090418\u027d\U000582b9\u54c3\U000fcb6e') == "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'") + assert (repr(u'\n') == + "u'\\n'") Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Tue Nov 13 16:32:38 2007 @@ -913,7 +913,6 @@ if code == ord('\n'): result.append('\\') result.append('n') - i += 2 j += 1 continue if code < ord(' ') or code >= 0x7f: From cfbolz at codespeak.net Tue Nov 13 17:00:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 17:00:01 +0100 (CET) Subject: [pypy-svn] r48653 - pypy/dist/pypy/doc Message-ID: <20071113160001.83A6582D1@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 17:00:01 2007 New Revision: 48653 Modified: pypy/dist/pypy/doc/news.txt Log: news entry about the pypy status blog Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Tue Nov 13 17:00:01 2007 @@ -14,6 +14,14 @@ .. _`iCalendar format`: webcal://pypycal.sabi.net///calendars/PyPy.ics .. _eventhistory: eventhistory.html +PyPy blog started +================= + +A few days ago some of the PyPy developers started a `PyPy Status Blog`_. Let's +see how this works out. *(November 13th, 2007)* + +.. _`PyPy Status Blog`: http://morepypy.blogspot.com + PyPy/Squeak Sprint in Bern finished =================================== From fijal at codespeak.net Tue Nov 13 17:12:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:12:38 +0100 (CET) Subject: [pypy-svn] r48654 - pypy/dist/pypy/doc Message-ID: <20071113161238.0664182EB@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:12:38 2007 New Revision: 48654 Modified: pypy/dist/pypy/doc/confrest.py Log: Remove link to the EU page Modified: pypy/dist/pypy/doc/confrest.py ============================================================================== --- pypy/dist/pypy/doc/confrest.py (original) +++ pypy/dist/pypy/doc/confrest.py Tue Nov 13 17:12:38 2007 @@ -21,8 +21,6 @@ class_="menu"), html.a("contact", href=self.get_doclink("contact.html"), class_="menu"), " ", - html.a("EU/project", - href="http://pypy.org/", class_="menu"), " ", " ", id="menubar") def get_doclink(self, target): From fijal at codespeak.net Tue Nov 13 17:12:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:12:56 +0100 (CET) Subject: [pypy-svn] r48655 - pypy/dist/pypy/doc Message-ID: <20071113161256.F013B82EB@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:12:56 2007 New Revision: 48655 Modified: pypy/dist/pypy/doc/getting-started.txt Log: Point to samuele pypy-c tests rather than py.py tests Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Tue Nov 13 17:12:56 2007 @@ -61,7 +61,7 @@ assume that you have a py.py; with precompiled binaries, you need to pick one with the matching features compiled in. -.. _`98% of CPythons core language regression tests`: http://codespeak.net/~hpk/pypy-testresult/ +.. _`98% of CPythons core language regression tests`: http://www2.openend.se/~pedronis/pypy-c-test/allworkingmodules/summary.html .. _`pypy-1.0.0.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.bz2 .. _`pypy-1.0.0.zip`: http://codespeak.net/download/pypy/pypy-1.0.0.zip .. _`pypy-1.0.0.tar.gz`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.gz From fijal at codespeak.net Tue Nov 13 17:18:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:18:32 +0100 (CET) Subject: [pypy-svn] r48656 - pypy/dist/pypy/doc Message-ID: <20071113161832.3FF3082D4@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:18:31 2007 New Revision: 48656 Modified: pypy/dist/pypy/doc/getting-started.txt Log: Make note bold. Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Tue Nov 13 17:18:31 2007 @@ -117,10 +117,10 @@ test_all.py is just another name for `py.test`_ which is the testing tool that we are using and enhancing for PyPy. -Note that running all the tests takes a very long time, and +**Note that running all the tests takes a very long time, and enormous amounts of memory if you are trying to run them all in the same process; test_all.py is only suitable to run a -subset of them at a time. To run them all we have an +subset of them at a time.** To run them all we have an `autotest driver`_ that executes the tests directory by directory and produces pages like the following one: From cfbolz at codespeak.net Tue Nov 13 17:19:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 17:19:06 +0100 (CET) Subject: [pypy-svn] r48657 - pypy/dist/pypy/doc Message-ID: <20071113161906.6A40582F1@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 17:19:05 2007 New Revision: 48657 Modified: pypy/dist/pypy/doc/index.txt Log: put a link to eu page to index.txt, now that it is gone in the navigation Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Tue Nov 13 17:19:05 2007 @@ -42,6 +42,11 @@ Project Documentation ===================================== +PyPy was funded by the EU for several years. See the `web site of the EU +project`_ for more details. + +.. _`web site of the EU project`: http://pypy.org + architecture_ gives a complete view of PyPy's basic design. `coding guide`_ helps you to write code for PyPy (especially also describes From fijal at codespeak.net Tue Nov 13 17:19:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:19:13 +0100 (CET) Subject: [pypy-svn] r48658 - pypy/dist/pypy/doc Message-ID: <20071113161913.4B2D482F5@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:19:12 2007 New Revision: 48658 Modified: pypy/dist/pypy/doc/getting-started.txt Log: Python 2.3 is not supported in many places. Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Tue Nov 13 17:19:12 2007 @@ -156,7 +156,7 @@ The py.py interpreter +++++++++++++++++++++ -To start interpreting Python with PyPy, use Python 2.3 or greater:: +To start interpreting Python with PyPy, use Python 2.4 or greater:: cd pypy python bin/py.py From fijal at codespeak.net Tue Nov 13 17:39:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:39:42 +0100 (CET) Subject: [pypy-svn] r48659 - pypy/dist/pypy/doc Message-ID: <20071113163942.3A3B382EB@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:39:41 2007 New Revision: 48659 Modified: pypy/dist/pypy/doc/getting-started.txt Log: Update to the current state. The thing is that right now it starts from "download 1.0 release" and then explains current state from svn. We should discourage 1.0 Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Tue Nov 13 17:39:41 2007 @@ -293,71 +293,13 @@ of around 200 lines of code. Since the 0.8.0 release you can `translate PyPy with the thunk object space`_. -Logic programming -+++++++++++++++++ - -People familiar with logic programming languages will be interested to -know that PyPy optionally supports logic variables and constraint-based -programming. Among the many interesting features of logic programming --- like unification -- this subsumes the thunk object space by -providing a more extensive way to deal with laziness. +More +++++ -Try it out:: - - cd pypy - python bin/py.py -o logic +If you know more about pypy-exclusive features, choose one from the following, +go to `objspace proxies`_ document. - >>>> X = newvar() # a logic variable - >>>> bind(X, 42) # give it a value - >>>> assert X / 2 == 21 # then use it - >>>> assert type(X) is int - - >>>> X, Y, Z = newvar(), newvar(), newvar() # three logic vars - >>>> unify({'hello': Y, 'world': Z}, X) # a complex unification - >>>> bind(Y, 5) # then give values to Y - >>>> bind(Z, 7) # ... and Z - >>>> X - {'hello': 5, 'world': 7} - - >>>> bind(Z, 8) - RuntimeError: Cannot bind twice - -Read more about `Logic Object space features`_. - -.. _`Logic Object space features`: howto-logicobjspace.html - - -Aspect Oriented Programming -+++++++++++++++++++++++++++ - -PyPy provides an experimental `Aspect Oriented Programming` facility -in the 'aop' module. Be aware that this module works on the abstract -syntax tree level, and will leave traces in the byte-compiled ('.pyc') -files. Be sure to remove these files after playing with the module if -you do not intend to really use the aspect oriented programming in -your code. Even better, use the '--no-objspace-usepycfiles' command -line option, so that the byte-compiled modules are not written to -disk. - -Here is an example:: - - cd pypy - python bin/py.py --no-objspace-usepycfiles - - >>>> from aop import Aspect, before, PointCut - >>>> class DemoAspect: - .... __metaclass__ = Aspect - .... @before(PointCut(func="^open$").call()) - .... def before_open(self, joinpoint): - .... print "opening", joinpoint.arguments() - .... - >>>> aspect = DemoAspect() - >>>> f = open("/tmp/toto.txt", 'w') - -To read more about this, try the `aop module documentation`_. - -.. _`Aspect Oriented Programming`: http://aosd.net -.. _`aop module documentation`: aspect_oriented_programming.html +.. _`objspace proxies`: objspace-proxies.html Running the tests +++++++++++++++++ @@ -389,11 +331,10 @@ or if you have `installed py.test`_ then you simply say:: - py.test + py.test -E -from the lib-python/2.4.1/test directory. You need to have a checkout of the -testresult directory. Running one of the above commands tells you how to -proceed. +from the lib-python/2.4.1/test directory. Running one of the above +commands tells you how to proceed. .. _`installed py.test`: https://codespeak.net/py/current/doc/download.html @@ -414,15 +355,8 @@ sufficiently static Python programs into low-level code. To be able to use it you need to: - * Download and install Pygame_ and ctypes_ if you do not already have them. - - * Have an internet connection. The flowgraph viewer connects to - codespeak.net and lets it convert the flowgraph by a patched version of - `Dot Graphviz`_ that does not crash. This is only needed if you want to - look at the flowgraphs. - - * Use Python-2.4 for using translation tools because python2.5 - is (as of revision 39130) not fully supported. + * Download and install Pygame_ and `Dot Graphviz`_ if you do not + already have them. To start the interactive translator shell do:: @@ -502,8 +436,8 @@ +++++++++++++++++++++++++++++++++++++++++++++ The JavaScript backend is still experimental but was heavily improved -during last -years `Google summer of code`_. It contains some rudimentary support +during 2006 +`Google summer of code`_. It contains support for the document object model and a good integration with PyPy's unittesting framework. Code can be tested with the `Spidermonkey`_ commandline JavaScript interpreter in addition to a multitude of JavaScript capable browsers. @@ -557,11 +491,6 @@ Translating RPystone to JVM code ++++++++++++++++++++++++++++++++ -Although the JVM backend cannot yet translate the entire interpreter, -you can use it with the interactive shell for small functions, and -with the ``pypy/goal/translate.py`` for some larger benchmarks such as -RPystone. - Using the interactive shell with the JVM is basically the same as any other backend. For example, you might enter: @@ -637,9 +566,9 @@ By default the translation process will try to use the `Boehm-Demers-Weiser garbage collector`_ for the translated interpreter (Use ``--gc=framework`` to use our own exact mark-n-sweep implementation which -at the moment is slower but doesn't have external dependencies). Otherwise, be sure -to install Boehm before starting the translation (e.g. by running -``apt-get install libgc-dev`` on Debian). +at the moment is slower but doesn't have external dependencies). Otherwise, be +sure io install Boehm before starting the translation (e.g. by running +``apt-get install libgc-dev`` on Debian or Ubuntu). This whole process will take some time and quite a lot of memory (although it might just work on a machine with 512MB of RAM). If the translation is @@ -665,7 +594,7 @@ mainly the same basic options as CPython. In addition, ``pypy-c --info`` prints the translation options that where used to produce this particular executable. This executable contains a lot of things that are hard-coded -for your particular system (including paths), so it's not really meant to +for your particular system, so it's not really meant to be installed or redistributed at the moment. If you exit the interpreter you get a pygame window with all the flow graphs @@ -798,45 +727,6 @@ loaded when accessing .NET namespaces as they were Python modules, as IronPython does. -Translate using the Build Tool -++++++++++++++++++++++++++++++ - -If you don't have access to the required resources to build PyPy yourself, -or want to build it on a platform you don't have access to, you can use -the 'build tool', a set of components that allow translating and -compiling PyPy on remote servers. The build tool provides a meta server -that manages a set of connected build servers; clients can request a build -by specifying target options and Subversion revision and path, and the meta -server will do its best to get the request fulfilled. -Currently you need to have a codespeak account to be able to use the build tool. - -The meta server is made available on codespeak.net and there should always be -a number of build servers running. If you want to request a compilation you need -to checkout the build tool sources from: - - https://codespeak.net/svn/pypy/build/buildtool - -Then you can issue:: - - $ ./bin/startcompile.py [options] - -in the root of the 'buildtool' package replacing '[options]' with -the options you desire and '' with your email address. Type:: - - $ ./bin/startcompile.py --help - -to see a list of available options. - -To donate build server time (we'd be eternally thankful! ;) just run:: - - $ ./bin/buildserver.py - -without arguments. Do mind that you need to be a registered user on -'codespeak.net', and that a decent machine with enough RAM is required -(about 1-2 GB is recommended). - -Note: currently Windows is not supported. - .. _`start reading sources`: Where to start reading the sources @@ -891,6 +781,9 @@ there is a file rxxxx.py that contains the low level functions needed for this type. +* `pypy/rlib`_ contains the RPython standard library, things that you can + use from rpython. + .. _optionaltool: Additional Tools for running (and hacking) PyPy @@ -913,24 +806,13 @@ CTypes (highly recommended) ++++++++++++++++++++++++++++ -`ctypes`_ (version 0.9.9.6 or later) is required if you want to translate PyPy -to C. See the `download page of ctypes`_. +`ctypes`_ (version 0.9.9.6 or later) is required if you want to run low-level +tests. See the `download page of ctypes`_. .. _`download page of ctypes`: http://sourceforge.net/project/showfiles.php?group_id=71702 .. _`ctypes`: http://starship.python.net/crew/theller/ctypes/ - -CLISP -+++++++ - -The CLISP backend is optional and not quite up-to-date with the rest of -PyPy. Still there are a few examples you can try our backend out on. -Here is a link to a LISP implementation that should basically work: - - http://clisp.cons.org/ - - -.. _`py.test`: +.. _`py.test`: py.test and the py lib +++++++++++++++++++++++ @@ -973,7 +855,7 @@ .. _`CLI backend`: cli-backend.html .. _clr: clr-module.html -.. _Dot Graphviz: http://www.graphviz.org/ +.. _`Dot Graphviz`: http://www.graphviz.org/ .. _Pygame: http://www.pygame.org/ .. _pyopcode.py: http://codespeak.net/svn/pypy/dist/pypy/interpreter/pyopcode.py .. _eval.py: http://codespeak.net/svn/pypy/dist/pypy/interpreter/eval.py From fijal at codespeak.net Tue Nov 13 17:51:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 17:51:15 +0100 (CET) Subject: [pypy-svn] r48660 - pypy/dist/pypy/doc Message-ID: <20071113165115.B34AC82EE@code0.codespeak.net> Author: fijal Date: Tue Nov 13 17:51:15 2007 New Revision: 48660 Modified: pypy/dist/pypy/doc/getting-started.txt Log: Review the graphviz-on-codespeak section Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Tue Nov 13 17:51:15 2007 @@ -353,10 +353,17 @@ The translator is a tool based on the PyPy interpreter which can translate sufficiently static Python programs into low-level code. To be able to use it -you need to: +you need to (if you want to look at the flowgraphs, which you obviously +should): - * Download and install Pygame_ and `Dot Graphviz`_ if you do not - already have them. + * Download and install Pygame_. + + * Download and install `Dot Graphviz`_. + +or: + + * Have an internet connection. The flowgraph viewer connects to + codespeak.net and lets it convert the flowgraph by a graphviz server. To start the interactive translator shell do:: From fijal at codespeak.net Tue Nov 13 18:42:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Nov 2007 18:42:10 +0100 (CET) Subject: [pypy-svn] r48661 - pypy/extradoc/talk/pycon2008 Message-ID: <20071113174210.6C87682CB@code0.codespeak.net> Author: fijal Date: Tue Nov 13 18:42:09 2007 New Revision: 48661 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Quick outline Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Tue Nov 13 18:42:09 2007 @@ -38,6 +38,16 @@ XXX prepare +Note for Reviewers +================== + +As this is a very advanced topic, we don't expect too many people to show +up. In case the tutorial would be accepted and there would be less than 10 +people showing up, we can do the tutorial for free. + +Also, in case there would be CPython developement tutorial, it would be +nice if those two will not collide, if possible. + Promotional summary =================== @@ -55,13 +65,20 @@ Outline ======= -XXX expand -* half an hour of introduction and quick codebase skimming -* one hour of modifying pypy's python interpreter (what exactly XXX) -* an hour and a half of hands-on session, including pre-prepared - topics. +* Introduction into pypy - 15 minutes + + * Motivation - why Python in Python + + * RPython - the restricted python + + * Objectspace concept + + * Backends - how we can compile to all platforms + +* Scheme over codebase - 15 minutes -* Notes for reviewers. +XXX example, think what that could be. Much of touching pypy, + not much doing algorithms IMHO * Tutorial outline for website publication, if different from the above. From cfbolz at codespeak.net Tue Nov 13 19:30:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 19:30:19 +0100 (CET) Subject: [pypy-svn] r48662 - in pypy/branch/more-unicode-improvements/pypy/module/_codecs: . test Message-ID: <20071113183019.B997582F7@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 19:30:18 2007 New Revision: 48662 Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py Log: oops, raise an app-level, not an interplevel error Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py Tue Nov 13 19:30:18 2007 @@ -169,7 +169,9 @@ w_res = space.call_function(w_decoder, w_obj, space.wrap(errors)) if (not space.is_true(space.isinstance(w_res, space.w_tuple)) or space.int_w(space.len(w_res)) != 2): - raise TypeError("encoder must return a tuple (object, integer)") + raise OperationError( + space.w_TypeError, + space.wrap("encoder must return a tuple (object, integer)")) return space.getitem(w_res, space.wrap(0)) else: assert 0, "XXX, what to do here?" Modified: pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py Tue Nov 13 19:30:18 2007 @@ -261,6 +261,20 @@ assert '\\253'.decode('string_escape') == chr(0253) assert '\\312'.decode('string_escape') == chr(0312) + def test_decode_utf8_different_case(self): constant = u"a" assert constant.encode("utf-8") == constant.encode("UTF-8") + + def test_codec_wrong_result(self): + import _codecs + def search_function(encoding): + def f(input, errors="strict"): + return 42 + print encoding + if encoding == 'test.mytestenc': + return (f, f, None, None) + return None + _codecs.register(search_function) + raises(TypeError, "hello".decode, "test.mytestenc") + raises(TypeError, u"hello".encode, "test.mytestenc") From antocuni at codespeak.net Tue Nov 13 21:13:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 13 Nov 2007 21:13:00 +0100 (CET) Subject: [pypy-svn] r48663 - pypy/dist/pypy/translator Message-ID: <20071113201300.EF51682EC@code0.codespeak.net> Author: antocuni Date: Tue Nov 13 21:12:59 2007 New Revision: 48663 Modified: pypy/dist/pypy/translator/driver.py Log: automatically create a .jar and a corresponding script when using jvm backend Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Nov 13 21:12:59 2007 @@ -748,11 +748,49 @@ # restore original os values if hasattr(self, 'old_cli_defs'): unpatch_os(self.old_cli_defs) - self.log.info("Compiled JVM source") + if self.standalone and self.exe_name: + self.copy_jvm_jar() task_compile_jvm = taskdef(task_compile_jvm, ['source_jvm'], 'Compiling JVM source') + def copy_jvm_jar(self): + from py.compat import subprocess + basename = self.exe_name % self.get_info() + root = udir.join('pypy') + manifest = self.create_manifest(root) + classlist = self.create_classlist(root) + jarfile = py.path.local(basename + '.jar') + self.log.info('Creating jar file') + oldpath = root.chdir() + subprocess.call(['jar', 'cmf', str(manifest), str(jarfile), '@'+str(classlist)]) + oldpath.chdir() + + # create a convenience script + newexename = basename + f = file(newexename, 'w') + f.write("""#!/bin/bash +java -jar $0.jar "$@" +""") + f.close() + os.chmod(newexename, 0755) + + def create_manifest(self, root): + filename = root.join('manifest.txt') + manifest = filename.open('w') + manifest.write('Main-class: pypy.Main\n\n') + manifest.close() + return filename + + def create_classlist(self, root): + filename = root.join('classlist.txt') + classlist = filename.open('w') + classfiles = root.visit('*.class', True) + for classfile in classfiles: + print >> classlist, classfile.relto(root) + classlist.close() + return filename + def task_run_jvm(self): pass task_run_jvm = taskdef(task_run_jvm, ['compile_jvm'], From antocuni at codespeak.net Tue Nov 13 21:17:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 13 Nov 2007 21:17:43 +0100 (CET) Subject: [pypy-svn] r48664 - pypy/dist/pypy/translator/goal Message-ID: <20071113201743.6EF0482EF@code0.codespeak.net> Author: antocuni Date: Tue Nov 13 21:17:41 2007 New Revision: 48664 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: remove pypy-cli, add pypy-cli-faassen and pypy-jvm-faassen to nightly benchmarks Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Tue Nov 13 21:17:41 2007 @@ -187,7 +187,8 @@ c--gc=semispace--_faassen c--gc=generation--_faassen c--_objspace-std-withrope - cli + cli--_faassen + jvm--_faassen """.split('\n') if backend.strip() and not backend.strip().startswith('#')] print time.ctime() for backend in backends: From cfbolz at codespeak.net Tue Nov 13 21:22:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 21:22:19 +0100 (CET) Subject: [pypy-svn] r48665 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071113202219.69F5682FA@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 21:22:19 2007 New Revision: 48665 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Log: go through a bit less machinery when mixing strings and unicode strings in the same operation. Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py Tue Nov 13 21:22:19 2007 @@ -26,6 +26,12 @@ def unwrap(w_self, space): # for testing return w_self._value + + def create_if_subclassed(w_self): + if type(w_self) is W_UnicodeObject: + return w_self + return W_UnicodeObject(w_self._value) + W_UnicodeObject.EMPTY = W_UnicodeObject(u'') registerimplementation(W_UnicodeObject) @@ -59,7 +65,8 @@ # string-to-unicode delegation def delegate_String2Unicode(space, w_str): - w_uni = space.call_function(space.w_unicode, w_str) + from pypy.objspace.std.unicodetype import unicode_from_string + w_uni = unicode_from_string(space, w_str) assert isinstance(w_uni, W_UnicodeObject) # help the annotator! return w_uni @@ -92,17 +99,20 @@ return W_UnicodeObject(w_left._value + w_right._value) def add__String_Unicode(space, w_left, w_right): - return space.add(space.call_function(space.w_unicode, w_left) , w_right) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.add(unicode_from_string(space, w_left) , w_right) add__Rope_Unicode = add__String_Unicode def add__Unicode_String(space, w_left, w_right): - return space.add(w_left, space.call_function(space.w_unicode, w_right)) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.add(w_left, unicode_from_string(space, w_right)) add__Unicode_Rope = add__Unicode_String def contains__String_Unicode(space, w_container, w_item): - return space.contains(space.call_function(space.w_unicode, w_container), w_item ) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.contains(unicode_from_string(space, w_container), w_item ) contains__Rope_Unicode = contains__String_Unicode @@ -311,8 +321,9 @@ def unicode_strip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 1, 1) def unicode_strip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'strip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_strip__Unicode_Rope = unicode_strip__Unicode_String def unicode_lstrip__Unicode_None(space, w_self, w_chars): @@ -320,8 +331,9 @@ def unicode_lstrip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 1, 0) def unicode_lstrip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'lstrip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_lstrip__Unicode_Rope = unicode_lstrip__Unicode_String @@ -330,8 +342,9 @@ def unicode_rstrip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 0, 1) def unicode_rstrip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'rstrip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_rstrip__Unicode_Rope = unicode_rstrip__Unicode_String @@ -481,7 +494,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() leftpad = padding // 2 + (padding & width & 1) result = [fillchar] * width for i in range(len(self)): @@ -494,7 +507,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [fillchar] * width for i in range(len(self)): result[i] = self[i] @@ -506,7 +519,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [fillchar] * width for i in range(len(self)): result[padding + i] = self[i] @@ -519,7 +532,7 @@ return W_UnicodeObject(u'0' * width) padding = width - len(self) if padding <= 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [u'0'] * width for i in range(len(self)): result[padding + i] = self[i] @@ -948,47 +961,58 @@ from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.ropeobject import W_RopeObject def str_strip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'strip', w_chars) str_strip__Rope_Unicode = str_strip__String_Unicode def str_lstrip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'lstrip', w_chars) str_lstrip__Rope_Unicode = str_lstrip__String_Unicode def str_rstrip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rstrip', w_chars) str_rstrip__Rope_Unicode = str_rstrip__String_Unicode def str_count__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'count', w_substr, w_start, w_end) str_count__Rope_Unicode_ANY_ANY = str_count__String_Unicode_ANY_ANY def str_find__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'find', w_substr, w_start, w_end) str_find__Rope_Unicode_ANY_ANY = str_find__String_Unicode_ANY_ANY def str_rfind__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rfind', w_substr, w_start, w_end) str_rfind__Rope_Unicode_ANY_ANY = str_rfind__String_Unicode_ANY_ANY def str_index__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'index', w_substr, w_start, w_end) str_index__Rope_Unicode_ANY_ANY = str_index__String_Unicode_ANY_ANY def str_rindex__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rindex', w_substr, w_start, w_end) str_rindex__Rope_Unicode_ANY_ANY = str_rindex__String_Unicode_ANY_ANY def str_replace__String_Unicode_Unicode_ANY(space, w_self, w_old, w_new, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'replace', w_old, w_new, w_maxsplit) str_replace__Rope_Unicode_Unicode_ANY = str_replace__String_Unicode_Unicode_ANY def str_split__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'split', w_delim, w_maxsplit) str_split__Rope_Unicode_ANY = str_split__String_Unicode_ANY def str_rsplit__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rsplit', w_delim, w_maxsplit) str_rsplit__Rope_Unicode_ANY = str_rsplit__String_Unicode_ANY register_all(vars(), stringtype) Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Tue Nov 13 21:22:19 2007 @@ -219,7 +219,7 @@ from pypy.objspace.std.unicodeobject import W_UnicodeObject encoding = getdefaultencoding(space) if encoding != 'ascii': - return unicode_from_object(space, w_str) + return unicode_from_encoded_object(space, w_str, encoding, "strict") s = space.str_w(w_str) try: return W_UnicodeObject(s.decode("ascii")) From cfbolz at codespeak.net Tue Nov 13 21:22:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Nov 2007 21:22:42 +0100 (CET) Subject: [pypy-svn] r48666 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071113202242.9635182FE@code0.codespeak.net> Author: cfbolz Date: Tue Nov 13 21:22:42 2007 New Revision: 48666 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/ropeobject.py Log: move these operations on ropes to interplevel Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/ropeobject.py Tue Nov 13 21:22:42 2007 @@ -1030,44 +1030,49 @@ return W_RopeObject(rope.rope_from_charlist(buf[:i+1])) - -app = gateway.applevel(r''' - def str_translate__Rope_ANY_ANY(s, table, deletechars=''): - """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, - which must be a string of length 256""" - - if len(table) != 256: - raise ValueError("translation table must be 256 characters long") - - L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] - return ''.join(L) - - def str_decode__Rope_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getdecoder(encoding)(str)[0] - else: - return codecs.getdecoder(encoding)(str, errors)[0] - - def str_encode__Rope_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getencoder(encoding)(str)[0] - else: - return codecs.getencoder(encoding)(str, errors)[0] -''', filename=__file__) - -str_translate__Rope_ANY_ANY = app.interphook('str_translate__Rope_ANY_ANY') -str_decode__Rope_ANY_ANY = app.interphook('str_decode__Rope_ANY_ANY') -str_encode__Rope_ANY_ANY = app.interphook('str_encode__Rope_ANY_ANY') +def str_translate__Rope_ANY_ANY(space, w_string, w_table, w_deletechars=''): + """charfilter - unicode handling is not implemented + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, + which must be a string of length 256""" + + # XXX CPython accepts buffers, too, not sure what we should do + table = space.str_w(w_table) + if len(table) != 256: + raise OperationError( + space.w_ValueError, + space.wrap("translation table must be 256 characters long")) + + node = w_string._node + chars = [] + iter = rope.CharIterator(node) + while 1: + try: + c = iter.next() + i += 1 + w_char = W_RopeObject.PREBUILT[ord(char)] + if not space.is_true(space.contains(w_deletechars, w_char)): + chars.append(table[ord(char)]) + except StopIteration: + break + return W_RopeObject(rope.rope_from_charlist(chars)) + +def str_decode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + unicode_from_string, decode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + if encoding is None and errors is None: + return unicode_from_string(space, w_string) + return decode_object(space, w_string, encoding, errors) + +def str_encode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + return encode_object(space, w_string, encoding, errors) + # methods of the iterator From fijal at codespeak.net Wed Nov 14 01:17:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:17:17 +0100 (CET) Subject: [pypy-svn] r48668 - pypy/dist/pypy/rpython Message-ID: <20071114001717.C7F7A830C@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:17:15 2007 New Revision: 48668 Modified: pypy/dist/pypy/rpython/extfunc.py Log: Preserve libraries and libdirs as well Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Wed Nov 14 01:17:15 2007 @@ -92,7 +92,7 @@ class BaseLazyRegistering(object): __metaclass__ = LazyRegisteringMeta - __ATTRIBUTES = ['includes', 'include_dirs'] + __ATTRIBUTES = ['includes', 'include_dirs', 'libraries', 'library_dirs'] def configure(self, CConfig): classes_seen = self.__dict__.setdefault('__classes_seen', {}) From fijal at codespeak.net Wed Nov 14 01:23:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:23:56 +0100 (CET) Subject: [pypy-svn] r48669 - in pypy/dist/pypy: rpython rpython/lltypesystem/module rpython/module rpython/module/test translator/c/src Message-ID: <20071114002356.0B6D4830D@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:23:55 2007 New Revision: 48669 Removed: pypy/dist/pypy/rpython/lltypesystem/module/ll_strtod.py Modified: pypy/dist/pypy/rpython/extfuncregistry.py pypy/dist/pypy/rpython/extfunctable.py pypy/dist/pypy/rpython/module/ll_strtod.py pypy/dist/pypy/rpython/module/test/test_ll_strtod.py pypy/dist/pypy/translator/c/src/ll_strtod.h Log: Move ll_strtod to rpython. Because the C level helpers are ok, just make sure they don't touch the RPython interface. Otherwise just call them via rffi helpers Modified: pypy/dist/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extfuncregistry.py (original) +++ pypy/dist/pypy/rpython/extfuncregistry.py Wed Nov 14 01:23:55 2007 @@ -69,3 +69,7 @@ llimpl = func_with_new_name(func, name) register_external(func, args, res, 'll_os_path.ll_%s' % name, llimpl=llimpl, sandboxsafe=True) + +# -------------------- strtod functions ---------------------- + +from pypy.rpython.module import ll_strtod Modified: pypy/dist/pypy/rpython/extfunctable.py ============================================================================== --- pypy/dist/pypy/rpython/extfunctable.py (original) +++ pypy/dist/pypy/rpython/extfunctable.py Wed Nov 14 01:23:55 2007 @@ -163,13 +163,6 @@ ntpath.isabs = isabs # ___________________________________________________________ -# string->float helper -from pypy.rlib import rarithmetic -declare(rarithmetic.parts_to_float, float, 'll_strtod/parts_to_float') -# float->string helper -declare(rarithmetic.formatd, str, 'll_strtod/formatd') - -# ___________________________________________________________ # stackless from pypy.rlib import rstack declare(rstack.stack_frames_depth, int, 'll_stackless/stack_frames_depth') Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Wed Nov 14 01:23:55 2007 @@ -1,3 +1,54 @@ +import py +from pypy.rpython.extfunc import BaseLazyRegistering, extdef, registering +from pypy.rlib import rarithmetic +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.tool.cbuild import cache_c_module +from pypy.tool.autopath import pypydir +class CConfig: + _includes_ = ['ll_strtod.h'] +class RegisterStrtod(BaseLazyRegistering): + def __init__(self): + # HACK HACK HACK + # we need to have some sane way of doing stuff below + # problem: we don't have a way to call things in our header files + from pypy.tool.udir import udir + c_file = udir.join('test_strtod.c') + c_file.write(py.code.Source(""" + #include + """)) + cache_c_module([c_file], '_ll_strtod') + self._libraries_ = [str(py.path.local(pypydir).join('_cache', + '_ll_strtod.so'))] + self.configure(CConfig) + + @registering(rarithmetic.formatd) + def register_formatd(self): + ll_strtod = self.llexternal('LL_strtod_formatd', + [rffi.CCHARP, rffi.DOUBLE], rffi.CCHARP, + sandboxsafe=True, threadsafe=False) + + def llimpl(fmt, x): + res = ll_strtod(fmt, x) + return rffi.charp2str(res) + + return extdef([str, float], str, 'll_strtod.ll_strtod_formatd', + llimpl=llimpl) + + @registering(rarithmetic.parts_to_float) + def register_parts_to_float(self): + ll_parts_to_float = self.llexternal('LL_strtod_parts_to_float', + [rffi.CCHARP] * 4, rffi.DOUBLE, + sandboxsafe=True, + threadsafe=False) + + def llimpl(sign, beforept, afterpt, exponent): + res = ll_parts_to_float(sign, beforept, afterpt, exponent) + if res == -1 and rffi.get_errno() == 42: + raise ValueError("Wrong literal for float") + return res + + return extdef([str, str, str, str], float, + 'll_strtod.ll_strtod_parts_to_float', llimpl=llimpl) Modified: pypy/dist/pypy/rpython/module/test/test_ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_strtod.py Wed Nov 14 01:23:55 2007 @@ -1,11 +1,18 @@ import py -class BaseTest(object): - +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib import rarithmetic + +class BaseTestStrtod(BaseRtypingTest): + def test_formatd(self): + def f(y): + return rarithmetic.formatd("%.2f", y) + + assert self.ll_to_string(self.interpret(f, [3.0])) == f(3.0) + def test_parts_to_float(self): - #py.test.skip("wip") - Support = self.Support - Impl = self.Implementation + def f(a, b, c, d): + return rarithmetic.parts_to_float(a, b, c, d) data = [ (("","1","","") , 1.0), @@ -17,20 +24,8 @@ ] for parts, val in data: - assert Impl.ll_strtod_parts_to_float(*map(Support.to_rstr, parts)) == val - - - def test_formatd(self): - Support = self.Support - Impl = self.Implementation - - res = Impl.ll_strtod_formatd(Support.to_rstr("%.2f"), 1.5) - assert Support.from_rstr(res) == "1.50" + args = [self.string_to_ll(i) for i in parts] + assert self.interpret(f, args) == val -class TestLL(BaseTest): - from pypy.rpython.module.support import LLSupport as Support - from pypy.rpython.lltypesystem.module.ll_strtod import Implementation - -class TestOO(BaseTest): - from pypy.rpython.module.support import OOSupport as Support - from pypy.rpython.ootypesystem.module.ll_strtod import Implementation +class TestLLStrtod(BaseTestStrtod, LLRtypeMixin): + pass Modified: pypy/dist/pypy/translator/c/src/ll_strtod.h ============================================================================== --- pypy/dist/pypy/translator/c/src/ll_strtod.h (original) +++ pypy/dist/pypy/translator/c/src/ll_strtod.h Wed Nov 14 01:23:55 2007 @@ -1,12 +1,15 @@ #include #include - +#include +#include +#include +#include /* prototypes */ -double LL_strtod_parts_to_float(RPyString *sign, RPyString *beforept, - RPyString *afterpt, RPyString *exponent); -RPyString *LL_strtod_formatd(RPyString *fmt, double x); +double LL_strtod_parts_to_float(char *sign, char *beforept, + char *afterpt, char *exponent); +char *LL_strtod_formatd(char *fmt, double x); /* implementations */ @@ -14,10 +17,10 @@ #ifndef PYPY_NOT_MAIN_FILE double LL_strtod_parts_to_float( - RPyString *sign, - RPyString *beforept, - RPyString *afterpt, - RPyString *exponent) + char *sign, + char *beforept, + char *afterpt, + char *exponent) { char *fail_pos; struct lconv *locale_data; @@ -25,7 +28,7 @@ int decimal_point_len; double x; char *last; - char *expo = RPyString_AsString(exponent); + char *expo = exponent; int buf_size; char *s; @@ -37,20 +40,20 @@ decimal_point = locale_data->decimal_point; decimal_point_len = strlen(decimal_point); - buf_size = RPyString_Size(sign) + - RPyString_Size(beforept) + + buf_size = strlen(sign) + + strlen(beforept) + decimal_point_len + - RPyString_Size(afterpt) + + strlen(afterpt) + 1 /* e */ + strlen(expo) + 1 /* asciiz */ ; s = (char*)malloc(buf_size); - strcpy(s, RPyString_AsString(sign)); - strcat(s, RPyString_AsString(beforept)); + strcpy(s, sign); + strcat(s, beforept); strcat(s, decimal_point); - strcat(s, RPyString_AsString(afterpt)); + strcat(s, afterpt); strcat(s, "e"); strcat(s, expo); @@ -61,7 +64,7 @@ fail_pos = last; if (fail_pos == s || *fail_pos != '\0' || fail_pos != last) { free(s); - RPyRaiseSimpleException(PyExc_ValueError, "invalid float literal"); + errno = 42; // just because return -1.0; } if (x == 0.0) { /* maybe a denormal value, ask for atof behavior */ @@ -72,12 +75,12 @@ return x; } +char buffer[120]; /* this should be enough, from PyString_Format code */ +int buflen = 120; -RPyString *LL_strtod_formatd(RPyString *fmt, double x) { - char buffer[120]; /* this should be enough, from PyString_Format code */ - int buflen = 120; +char* LL_strtod_formatd(char *fmt, double x) { int res; - res = snprintf(buffer, buflen, RPyString_AsString(fmt), x); + res = snprintf(buffer, buflen, fmt, x); if (res <= 0 || res >= buflen) { strcpy(buffer, "??.?"); /* should not occur */ } else { @@ -117,7 +120,7 @@ } - return RPyString_FromString(buffer); + return buffer; } #endif /* PYPY_NOT_MAIN_FILE */ From fijal at codespeak.net Wed Nov 14 01:32:40 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:32:40 +0100 (CET) Subject: [pypy-svn] r48670 - pypy/dist/pypy/translator/c/src Message-ID: <20071114003240.1317B8312@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:32:39 2007 New Revision: 48670 Modified: pypy/dist/pypy/translator/c/src/ll_strtod.h Log: Make sure we don't include this file twice Modified: pypy/dist/pypy/translator/c/src/ll_strtod.h ============================================================================== --- pypy/dist/pypy/translator/c/src/ll_strtod.h (original) +++ pypy/dist/pypy/translator/c/src/ll_strtod.h Wed Nov 14 01:32:39 2007 @@ -1,3 +1,6 @@ +#ifndef LL_STRTOD_H +#define LL_STRTOD_H + #include #include #include @@ -5,6 +8,7 @@ #include #include + /* prototypes */ double LL_strtod_parts_to_float(char *sign, char *beforept, @@ -124,3 +128,4 @@ } #endif /* PYPY_NOT_MAIN_FILE */ +#endif From fijal at codespeak.net Wed Nov 14 01:32:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:32:57 +0100 (CET) Subject: [pypy-svn] r48671 - pypy/dist/pypy/rpython/module Message-ID: <20071114003257.91A528315@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:32:56 2007 New Revision: 48671 Modified: pypy/dist/pypy/rpython/module/ll_strtod.py Log: Ooops Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Wed Nov 14 01:32:56 2007 @@ -7,7 +7,7 @@ from pypy.tool.autopath import pypydir class CConfig: - _includes_ = ['ll_strtod.h'] + _includes_ = ['src/ll_strtod.h'] class RegisterStrtod(BaseLazyRegistering): def __init__(self): From fijal at codespeak.net Wed Nov 14 01:41:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:41:49 +0100 (CET) Subject: [pypy-svn] r48672 - pypy/dist/pypy/translator/c Message-ID: <20071114004149.490F98313@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:41:48 2007 New Revision: 48672 Modified: pypy/dist/pypy/translator/c/extfunc.py Log: Don't do LLEXTERNALS here Modified: pypy/dist/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/dist/pypy/translator/c/extfunc.py Wed Nov 14 01:41:48 2007 @@ -7,17 +7,12 @@ from pypy.rpython.lltypesystem import rlist from pypy.rpython.module import ll_time, ll_os from pypy.rpython.module import ll_stackless, ll_stack -from pypy.rpython.lltypesystem.module import ll_strtod # table of functions hand-written in src/ll_*.h # Note about *.im_func: The annotator and the rtyper expect direct # references to functions, so we cannot insert classmethods here. EXTERNALS = { - ll_strtod.Implementation.ll_strtod_parts_to_float: - 'LL_strtod_parts_to_float', - ll_strtod.Implementation.ll_strtod_formatd: - 'LL_strtod_formatd', ll_stackless.ll_stackless_switch: 'LL_stackless_switch', ll_stackless.ll_stackless_stack_frames_depth: 'LL_stackless_stack_frames_depth', ll_stack.ll_stack_unwind: 'LL_stack_unwind', From fijal at codespeak.net Wed Nov 14 01:42:40 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 01:42:40 +0100 (CET) Subject: [pypy-svn] r48673 - in pypy/dist/pypy/rpython: lltypesystem ootypesystem/module Message-ID: <20071114004240.E8C548315@code0.codespeak.net> Author: fijal Date: Wed Nov 14 01:42:40 2007 New Revision: 48673 Removed: pypy/dist/pypy/rpython/ootypesystem/module/ll_strtod.py Modified: pypy/dist/pypy/rpython/lltypesystem/ll_str.py pypy/dist/pypy/rpython/lltypesystem/rstr.py Log: Simplify ll_float_str, good :) Modified: pypy/dist/pypy/rpython/lltypesystem/ll_str.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll_str.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll_str.py Wed Nov 14 01:42:40 2007 @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem.lltype import GcArray, Array, Char, malloc -from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.rarithmetic import r_uint, formatd CHAR_ARRAY = GcArray(Char) @@ -115,7 +115,5 @@ return result def ll_float_str(repr, f): - from pypy.rpython.lltypesystem.module.ll_strtod import Implementation - from pypy.rpython.lltypesystem.rstr import percent_f - return Implementation.ll_strtod_formatd(percent_f, f) + return formatd("%f", f) Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Wed Nov 14 01:42:40 2007 @@ -899,4 +899,3 @@ unboxed_instance_str_prefix = string_repr.convert_const("") -percent_f = string_repr.convert_const("%f") From cfbolz at codespeak.net Wed Nov 14 11:05:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 11:05:27 +0100 (CET) Subject: [pypy-svn] r48674 - pypy/branch/more-unicode-improvements/pypy/objspace/std Message-ID: <20071114100527.0A6728263@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 11:05:27 2007 New Revision: 48674 Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Log: unfail some string tests Modified: pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py Wed Nov 14 11:05:27 2007 @@ -159,6 +159,8 @@ def encode_object(space, w_object, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") w_encode = space.getattr(w_codecs, space.wrap("encode")) + if encoding is None: + encoding = getdefaultencoding(space) if errors is None: w_retval = space.call_function(w_encode, w_object, space.wrap(encoding)) else: From antocuni at codespeak.net Wed Nov 14 12:45:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 14 Nov 2007 12:45:00 +0100 (CET) Subject: [pypy-svn] r48675 - pypy/dist/pypy/translator/goal Message-ID: <20071114114500.BC7968232@code0.codespeak.net> Author: antocuni Date: Wed Nov 14 12:44:59 2007 New Revision: 48675 Modified: pypy/dist/pypy/translator/goal/bench-unix.py Log: use the size of the jar file as the size of pypy-jvm Modified: pypy/dist/pypy/translator/goal/bench-unix.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-unix.py (original) +++ pypy/dist/pypy/translator/goal/bench-unix.py Wed Nov 14 12:44:59 2007 @@ -153,6 +153,13 @@ exesize = os.path.getsize(os.path.join(dirname, 'main.exe')) except OSError: exesize = 'XXX' + elif 'pypy-jvm' in exename: + jarname = exename + '.jar' + codesize = 'N/A' + try: + exesize = os.path.getsize(jarname) + except OSError: + exesize = 'XXX' else: codesize = os.popen('size "%s" | tail -n1 | cut -f1'%(exename,)).read().strip() exesize = os.path.getsize(exe) From fijal at codespeak.net Wed Nov 14 15:18:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 15:18:27 +0100 (CET) Subject: [pypy-svn] r48677 - pypy/extradoc/talk/pycon2008 Message-ID: <20071114141827.9710D8259@code0.codespeak.net> Author: fijal Date: Wed Nov 14 15:18:25 2007 New Revision: 48677 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Resolve XXX Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Wed Nov 14 15:18:25 2007 @@ -7,7 +7,7 @@ In this talk we will outline PyPy's unique features, namely sandboxing, transparent proxying, our distribution prototype and our security -prototype, the taint objspace (XXX "objspace" is obscure - might want to say something a non-pypy developer can understand here). We would also like to summarize our design +prototype, the taint prototype. We would also like to summarize our design decisions which make it possible for such features to be non-pervasive to the core interpreter codebase. Those features also works out of the box on different PyPy target platforms (C/POSIX, .NET, JVM) from the single From lac at codespeak.net Wed Nov 14 16:02:07 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Wed, 14 Nov 2007 16:02:07 +0100 (CET) Subject: [pypy-svn] r48679 - pypy/extradoc/talk/pycon2008 Message-ID: <20071114150207.821048101@code0.codespeak.net> Author: lac Date: Wed Nov 14 16:02:03 2007 New Revision: 48679 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: minor tweaks. fijal, its good. Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Wed Nov 14 16:02:03 2007 @@ -5,29 +5,37 @@ Summary (max 100 words, for the website): -In this talk we will outline PyPy's unique features, namely +In this talk we will outline some of PyPy's unique features, namely sandboxing, transparent proxying, our distribution prototype and our security -prototype, the taint prototype. We would also like to summarize our design -decisions which make it possible for such features to be non-pervasive to the -core interpreter codebase. Those features also works out of the box on -different PyPy target platforms (C/POSIX, .NET, JVM) from the single -interpreter source. +prototype (also known as the taint prototype). We would also like to +present and summarise the design +decisions which makes it so easy to implement such features. + +One additional benefit of our design decisions is that low-level +decisions do not permeate the codebase. Thus, these features are not +platform-dependent. They work out of the box on different PyPy target +platforms (C/POSIX, .NET, JVM) from a single interpreter source. Description and outline: In this talk, we would like to present our paradigm-shifting architecture for dynamic languages and some features which are exclusively available in -PyPy due to our novel approach. Also thanks to our design, those features +PyPy due to our novel approach. These features are available on all of our target platforms (C/POSIX, .NET, JVM) and are independent of other decisions (garbage collector, stackless, JIT, etc.) +The result is an unprecidented ability to mix-and-match low-level +internal implementation details so that we can really serve you a +'Python as you like it'. Talk outline: * Quick recap of our motivation for creating PyPy. +demos including + * The transparent proxy, which allows programmers to create indistinguishable proxies for any type in Python (including built-in - python types like frames), with various usage showcases including + python types like frames), with various demos including one for our completely transparent remote access scheme. * the PyPy Sandbox, which allows one to run a full python @@ -37,8 +45,12 @@ * The Taint Object Space, which allows programmers to make sure that sensitive information does not cross I/O barriers. + +We conclude with a little bit of philosophy. + * Why our approach of doing pervasive changes to semantics does not require pervasive changes to the interpreter core. Why our design allows us to describe a broad class of such changes as a "proxying operation" and finally, why design matters when you want - to get a butterfly out of a caterpillar instead of a bigger caterpillar. + to get a butterfly out of a caterpillar instead of just a bigger, + healthier caterpillar. From fijal at codespeak.net Wed Nov 14 16:14:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 16:14:48 +0100 (CET) Subject: [pypy-svn] r48680 - pypy/extradoc/talk/pycon2008 Message-ID: <20071114151448.A94EA8286@code0.codespeak.net> Author: fijal Date: Wed Nov 14 16:14:48 2007 New Revision: 48680 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Add a demo paragraph Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Wed Nov 14 16:14:48 2007 @@ -31,8 +31,6 @@ * Quick recap of our motivation for creating PyPy. -demos including - * The transparent proxy, which allows programmers to create indistinguishable proxies for any type in Python (including built-in python types like frames), with various demos including one for @@ -45,6 +43,9 @@ * The Taint Object Space, which allows programmers to make sure that sensitive information does not cross I/O barriers. +All the features mentioned above would be presented together +with immediate usecase live demos how these features work and how +to use them in the real-world applications. We conclude with a little bit of philosophy. From fijal at codespeak.net Wed Nov 14 17:19:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 17:19:46 +0100 (CET) Subject: [pypy-svn] r48681 - pypy/extradoc/talk/pycon2008 Message-ID: <20071114161946.0BAA68263@code0.codespeak.net> Author: fijal Date: Wed Nov 14 17:19:45 2007 New Revision: 48681 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Drop in some random ideas about examples Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Wed Nov 14 17:19:45 2007 @@ -38,6 +38,11 @@ XXX prepare +* pypy checkout, codespeak account + +* pygame, ctypes or python2.5, libgc (boehm) - all not necessary, + but very useful + Note for Reviewers ================== @@ -75,10 +80,33 @@ * Backends - how we can compile to all platforms -* Scheme over codebase - 15 minutes +* Skim over codebase - 15 minutes + +* Implementing example modification to the VM - 1h + + * break + +* Hands-on pypy, 1.5h + +Example ++++++++ + +We would like to present one or two examples during the talk and +offer some tasks for people to experiment with. The exact details about +the examples will be prepared later, especially because given current +pypy developement pace, a lot of things might be implemented till march. + +One of the following might be especially good for the purpose of talk: + +* (Re)implement the thunk objspace, the way to "freeze" computation as + long as they're not needed. + +* Implement specialized sort of tuples, like pairs, indistinguishable from + normal tuples from the application level code. + +* Implement dict.keys() as a lazily computed list. + -XXX example, think what that could be. Much of touching pypy, - not much doing algorithms IMHO * Tutorial outline for website publication, if different from the above. From pedronis at codespeak.net Wed Nov 14 17:48:39 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 14 Nov 2007 17:48:39 +0100 (CET) Subject: [pypy-svn] r48683 - pypy/extradoc/talk/roadshow-google Message-ID: <20071114164839.36A698286@code0.codespeak.net> Author: pedronis Date: Wed Nov 14 17:48:38 2007 New Revision: 48683 Added: pypy/extradoc/talk/roadshow-google/technical.txt - copied, changed from r48681, pypy/extradoc/talk/roadshow-ibm/talk.txt Log: copy the technical part of the ibm talk here for the jit part of it From cfbolz at codespeak.net Wed Nov 14 17:56:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 17:56:32 +0100 (CET) Subject: [pypy-svn] r48684 - pypy/dist/pypy/objspace/test Message-ID: <20071114165632.ED854828C@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 17:56:32 2007 New Revision: 48684 Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py Log: two tests showing the brokenness of the thunk object space Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_thunkobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_thunkobjspace.py Wed Nov 14 17:56:32 2007 @@ -57,13 +57,19 @@ assert id(x) == id(y) == idy def test_double_become(self): + skip("fix me") from __pypy__ import thunk, become - x = [] - y = [] - z = [] + x = [1] + y = [2] + z = [3] become(x, y) become(y, z) assert x is y is z + a = [] + a.extend(x) + a.extend(y) + a.extend(z) + assert a == [3, 3, 3] def test_double_become2(self): from __pypy__ import thunk, become @@ -147,3 +153,10 @@ x = [] become(x, x) assert str(x) == "[]" + + def test_thunk_special_method(self): + skip("fix me") + from __pypy__ import thunk + x = thunk(lambda : 42) + assert 1 .__add__(x) == 43 + From jacob at codespeak.net Wed Nov 14 18:25:42 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Wed, 14 Nov 2007 18:25:42 +0100 (CET) Subject: [pypy-svn] r48685 - pypy/extradoc/talk/roadshow-google Message-ID: <20071114172542.2FDF381E2@code0.codespeak.net> Author: jacob Date: Wed Nov 14 18:25:40 2007 New Revision: 48685 Added: pypy/extradoc/talk/roadshow-google/Pypy_architecture.pdf (contents, props changed) Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp Log: Modified talk, added pdf version. Modified: pypy/extradoc/talk/roadshow-google/Pypy architecture.odp ============================================================================== Binary files. No diff available. Added: pypy/extradoc/talk/roadshow-google/Pypy_architecture.pdf ============================================================================== Binary file. No diff available. From cfbolz at codespeak.net Wed Nov 14 19:25:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 19:25:35 +0100 (CET) Subject: [pypy-svn] r48686 - pypy/dist/pypy/translator/c/test Message-ID: <20071114182535.B747181DF@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 19:25:34 2007 New Revision: 48686 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: this passes these days Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Wed Nov 14 19:25:34 2007 @@ -874,5 +874,3 @@ class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" should_be_moving = True - def test_many_ids(self): - py.test.skip("fails!") From cfbolz at codespeak.net Wed Nov 14 19:57:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 19:57:11 +0100 (CET) Subject: [pypy-svn] r48687 - in pypy/dist/pypy/objspace/flow: . test Message-ID: <20071114185711.BFDBE81C9@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 19:57:11 2007 New Revision: 48687 Removed: pypy/dist/pypy/objspace/flow/test/test_picklegraph.py Modified: pypy/dist/pypy/objspace/flow/model.py Log: remove pickle support for flow graphs Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Wed Nov 14 19:57:11 2007 @@ -321,24 +321,6 @@ self._name = intern(name) self._nr = nr - def __reduce_ex__(self, *args): - if hasattr(self, 'concretetype'): - return _bv, (self._name, self._nr, self.concretetype) - else: - return _bv, (self._name, self._nr) - __reduce__ = __reduce_ex__ - -def _bv(_name, _nr, concretetype=None): - v = Variable.__new__(Variable, object) - v._name = _name - v._nr = _nr - if concretetype is not None: - v.concretetype = concretetype - nd = v.namesdict - if _nr >= nd.get(_name, 0): - nd[_name] = _nr + 1 - return v - class Constant(Hashable): __slots__ = ["concretetype"] @@ -347,12 +329,6 @@ Hashable.__init__(self, value) if concretetype is not None: self.concretetype = concretetype - def __reduce_ex__(self, *args): - if hasattr(self, 'concretetype'): - return Constant, (self.value, self.concretetype) - else: - return Constant, (self.value,) - __reduce__ = __reduce_ex__ class SpaceOperation(object): @@ -380,16 +356,6 @@ return "%r = %s(%s)" % (self.result, self.opname, ", ".join(map(repr, self.args))) - def __reduce_ex__(self, *args): - # avoid lots of useless list entities - return _sop, (self.opname, self.result, self.offset) + tuple(self.args) - __reduce__ = __reduce_ex__ - -# a small and efficient restorer -def _sop(opname, result, offset, *args): - return SpaceOperation(opname, args, result, offset) - - class Atom: def __init__(self, name): self.__name__ = name # make save_global happy @@ -414,56 +380,6 @@ return result -#_________________________________________________________ -# a visitor for easy traversal of the above model - -##import inspect # for getmro - -##class traverse: - -## def __init__(self, visitor, functiongraph): -## """ send the visitor over all (reachable) nodes. -## the visitor needs to have either callable attributes 'visit_typename' -## or otherwise is callable itself. -## """ -## self.visitor = visitor -## self.visitor_cache = {} -## self.seen = {} -## self.visit(functiongraph) - -## def visit(self, node): -## if id(node) in self.seen: -## return - -## # do the visit -## cls = node.__class__ -## try: -## consume = self.visitor_cache[cls] -## except KeyError: -## for subclass in inspect.getmro(cls): -## consume = getattr(self.visitor, "visit_" + subclass.__name__, None) -## if consume: -## break -## else: -## consume = getattr(self.visitor, 'visit', self.visitor) - -## assert callable(consume), "visitor not found for %r on %r" % (cls, self.visitor) - -## self.visitor_cache[cls] = consume - -## self.seen[id(node)] = consume(node) - -## # recurse -## if isinstance(node, Block): -## for obj in node.exits: -## self.visit(obj) -## elif isinstance(node, Link): -## self.visit(node.target) -## elif isinstance(node, FunctionGraph): -## self.visit(node.startblock) -## else: -## raise ValueError, "could not dispatch %r" % cls - def traverse(visit, functiongraph): block = functiongraph.startblock visit(block) From cfbolz at codespeak.net Wed Nov 14 20:28:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 20:28:12 +0100 (CET) Subject: [pypy-svn] r48688 - in pypy/dist/pypy: interpreter interpreter/test module/_codecs module/_codecs/test objspace/std objspace/std/test rlib rlib/test tool Message-ID: <20071114192812.0C13681FE@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 20:28:11 2007 New Revision: 48688 Added: pypy/dist/pypy/module/_codecs/interp_codecs.py - copied unchanged from r48687, pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py pypy/dist/pypy/rlib/runicode.py - copied unchanged from r48687, pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py pypy/dist/pypy/rlib/test/test_runicode.py - copied unchanged from r48687, pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py pypy/dist/pypy/tool/unicodefuzzer.py - copied unchanged from r48687, pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py Modified: pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/test/test_gateway.py pypy/dist/pypy/module/_codecs/__init__.py pypy/dist/pypy/module/_codecs/app_codecs.py pypy/dist/pypy/module/_codecs/test/test_codecs.py pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/test/test_stringobject.py pypy/dist/pypy/objspace/std/test/test_unicodeobject.py pypy/dist/pypy/objspace/std/unicodeobject.py pypy/dist/pypy/objspace/std/unicodetype.py Log: merge the more-unicode-improvements branch. changes: ------------------------------------------------------------------------ r48674 | cfbolz | 2007-11-14 11:05:27 +0100 (Wed, 14 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py unfail some string tests ------------------------------------------------------------------------ r48666 | cfbolz | 2007-11-13 21:22:42 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/ropeobject.py move these operations on ropes to interplevel ------------------------------------------------------------------------ r48665 | cfbolz | 2007-11-13 21:22:19 +0100 (Tue, 13 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py go through a bit less machinery when mixing strings and unicode strings in the same operation. ------------------------------------------------------------------------ r48662 | cfbolz | 2007-11-13 19:30:18 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py oops, raise an app-level, not an interplevel error ------------------------------------------------------------------------ r48652 | cfbolz | 2007-11-13 16:32:38 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py bug + fix ------------------------------------------------------------------------ r48650 | cfbolz | 2007-11-13 16:08:04 +0100 (Tue, 13 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py refactor repr__Unicode, which seems to be written with the assumption that RPython strings didn't have overalloction. also fix a bug in the process. ------------------------------------------------------------------------ r48649 | cfbolz | 2007-11-13 15:34:32 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py fix bug in utf-16-decoder ------------------------------------------------------------------------ r48644 | cfbolz | 2007-11-13 12:38:00 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py more speaking var names. don't use += for list appending ------------------------------------------------------------------------ r48639 | cfbolz | 2007-11-13 11:13:12 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py be a bit more consistent in variable names ------------------------------------------------------------------------ r48636 | cfbolz | 2007-11-13 10:51:09 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py fix the test plus a bit of niceification: call ord(ch) only once. ------------------------------------------------------------------------ r48635 | cfbolz | 2007-11-13 10:50:37 +0100 (Tue, 13 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py make the test more reliable: don't let CPython's unmarshaller renormalize the string when reloading it from a pyc file. ------------------------------------------------------------------------ r48632 | cfbolz | 2007-11-13 10:05:41 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py test + fix ------------------------------------------------------------------------ r48627 | cfbolz | 2007-11-13 00:43:34 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py more typos ------------------------------------------------------------------------ r48626 | cfbolz | 2007-11-13 00:27:50 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py the annotator doesn't like reading from the sys module directly ------------------------------------------------------------------------ r48625 | cfbolz | 2007-11-13 00:19:03 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py typo ------------------------------------------------------------------------ r48624 | cfbolz | 2007-11-13 00:10:08 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py a sanity check ------------------------------------------------------------------------ r48623 | cfbolz | 2007-11-13 00:10:00 +0100 (Tue, 13 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py change latin1 to latin_1 ------------------------------------------------------------------------ r48621 | cfbolz | 2007-11-12 23:45:55 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/interpreter/gateway.py M /pypy/branch/more-unicode-improvements/pypy/interpreter/test/test_gateway.py allow unicode in the unwrap_spec ------------------------------------------------------------------------ r48619 | cfbolz | 2007-11-12 23:12:00 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py kill kill kill some more code ------------------------------------------------------------------------ r48618 | cfbolz | 2007-11-12 23:02:18 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py this is no longer needed either ------------------------------------------------------------------------ r48617 | cfbolz | 2007-11-12 23:00:03 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py yet another decoder moved to use the runicode version ------------------------------------------------------------------------ r48616 | cfbolz | 2007-11-12 22:34:09 +0100 (Mon, 12 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py kill kill kill kill kill kill kill kill the applevel versions (they are buggy anyway) ------------------------------------------------------------------------ r48615 | cfbolz | 2007-11-12 22:23:27 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py plug the RPython decoders and encoders into the _codecs module. ------------------------------------------------------------------------ r48612 | cfbolz | 2007-11-12 21:37:24 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py change names to be more consistent with CPython ------------------------------------------------------------------------ r48611 | cfbolz | 2007-11-12 20:39:51 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py fix typo (thanks Alexander) ------------------------------------------------------------------------ r48610 | cfbolz | 2007-11-12 20:15:51 +0100 (Mon, 12 Nov 2007) | 3 lines Changed paths: A /pypy/branch/more-unicode-improvements/pypy/tool/unicodefuzzer.py small tool to encode and decode random unicode/byte strings to check encoders and decoders. Finds examples that aren't working in PyPy incredibly fast. ------------------------------------------------------------------------ r48609 | cfbolz | 2007-11-12 19:50:55 +0100 (Mon, 12 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py grr. The error handling of PyPy's utf-16 decoder is broken. Fix the RPython version at least. ------------------------------------------------------------------------ r48606 | cfbolz | 2007-11-12 18:49:47 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py tests for decoding errors, fix a small bug in utf8 ------------------------------------------------------------------------ r48604 | cfbolz | 2007-11-12 18:18:42 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py add tests for encoding error handlers, fix problems found ------------------------------------------------------------------------ r48603 | cfbolz | 2007-11-12 18:00:33 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py port encoders to RPython ------------------------------------------------------------------------ r48601 | cfbolz | 2007-11-12 16:21:22 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py add utf16 ------------------------------------------------------------------------ r48599 | cfbolz | 2007-11-12 15:36:22 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py ascii is simple too ------------------------------------------------------------------------ r48598 | cfbolz | 2007-11-12 15:29:42 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py M /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py decoding latin1 is simple ------------------------------------------------------------------------ r48597 | cfbolz | 2007-11-12 15:17:03 +0100 (Mon, 12 Nov 2007) | 3 lines Changed paths: A /pypy/branch/more-unicode-improvements/pypy/rlib/runicode.py A /pypy/branch/more-unicode-improvements/pypy/rlib/test/test_runicode.py start a new rlib library for unicode handling. so far it contains only an rpython utf-8 decoder ------------------------------------------------------------------------ r48591 | cfbolz | 2007-11-12 03:25:43 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/__init__.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/app_codecs.py A /pypy/branch/more-unicode-improvements/pypy/module/_codecs/interp_codecs.py M /pypy/branch/more-unicode-improvements/pypy/module/_codecs/test/test_codecs.py start moving some bits of the _codecs module to interplevel ------------------------------------------------------------------------ r48590 | cfbolz | 2007-11-12 02:01:08 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py tests are good: fix typos ------------------------------------------------------------------------ r48589 | cfbolz | 2007-11-12 01:40:05 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py move string.translate to interplevel ------------------------------------------------------------------------ r48588 | cfbolz | 2007-11-12 00:16:11 +0100 (Mon, 12 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/stringobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_stringobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py move string.encode and string.decode to interp-level. share code where possible. ------------------------------------------------------------------------ r48586 | cfbolz | 2007-11-11 23:51:23 +0100 (Sun, 11 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py fix error message ------------------------------------------------------------------------ r48585 | cfbolz | 2007-11-11 23:39:42 +0100 (Sun, 11 Nov 2007) | 2 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodetype.py add an XXX ------------------------------------------------------------------------ r48584 | cfbolz | 2007-11-11 23:34:03 +0100 (Sun, 11 Nov 2007) | 3 lines Changed paths: M /pypy/branch/more-unicode-improvements/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/more-unicode-improvements/pypy/objspace/std/unicodeobject.py move unicode.encode to interplevel. add several encoding tests from CPython's tests suite. ------------------------------------------------------------------------ r48581 | cfbolz | 2007-11-11 22:45:52 +0100 (Sun, 11 Nov 2007) | 2 lines Changed paths: A /pypy/branch/more-unicode-improvements (from /pypy/dist:48580) create a new branch for more unicode improvements ------------------------------------------------------------------------ Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Wed Nov 14 20:28:11 2007 @@ -164,7 +164,7 @@ app_sig.varargname = argname[2:] def visit__object(self, typ, app_sig): - if typ not in (int, str, float, r_longlong): + if typ not in (int, str, float, unicode, r_longlong): assert False, "unsupported basic type in unwrap_spec" self.checked_space_method(typ.__name__, app_sig) @@ -210,8 +210,8 @@ self.run_args.append(self.scopenext()) def visit__object(self, typ): - if typ not in (int, str, float, r_longlong): - assert False, "unsupported basic type in uwnrap_spec" + if typ not in (int, str, float, unicode, r_longlong): + assert False, "unsupported basic type in unwrap_spec" if typ is r_int is r_longlong: name = 'r_longlong' else: @@ -327,7 +327,7 @@ raise FastFuncNotSupported def visit__object(self, typ): - if typ not in (int, str, float, r_longlong): + if typ not in (int, str, float, unicode, r_longlong): assert False, "unsupported basic type in uwnrap_spec" self.unwrap.append("space.%s_w(%s)" % (typ.__name__, self.nextarg())) Modified: pypy/dist/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_gateway.py (original) +++ pypy/dist/pypy/interpreter/test/test_gateway.py Wed Nov 14 20:28:11 2007 @@ -287,6 +287,27 @@ raises(gateway.OperationError,space.call_function,w_app_g3_f,w(None)) raises(gateway.OperationError,space.call_function,w_app_g3_f,w("foo")) + def test_interp2app_unwrap_spec_unicode(self): + space = self.space + w = space.wrap + def g3_u(space, uni): + return space.wrap(len(uni)) + app_g3_u = gateway.interp2app_temp(g3_u, + unwrap_spec=[gateway.ObjSpace, + unicode]) + w_app_g3_u = space.wrap(app_g3_u) + assert self.space.eq_w( + space.call_function(w_app_g3_u, w(u"foo")), + w(3)) + assert self.space.eq_w( + space.call_function(w_app_g3_u, w("baz")), + w(3)) + raises(gateway.OperationError, space.call_function, w_app_g3_u, + w(None)) + raises(gateway.OperationError, space.call_function, w_app_g3_u, + w(42)) + + def test_interp2app_unwrap_spec_func(self): space = self.space w = space.wrap Modified: pypy/dist/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/dist/pypy/module/_codecs/__init__.py (original) +++ pypy/dist/pypy/module/_codecs/__init__.py Wed Nov 14 20:28:11 2007 @@ -4,41 +4,51 @@ appleveldefs = { '__doc__' : 'app_codecs.__doc__', '__name__' : 'app_codecs.__name__', - 'ascii_decode' : 'app_codecs.ascii_decode', - 'ascii_encode' : 'app_codecs.ascii_encode', 'charbuffer_encode' : 'app_codecs.charbuffer_encode', 'charmap_decode' : 'app_codecs.charmap_decode', 'charmap_encode' : 'app_codecs.charmap_encode', 'escape_decode' : 'app_codecs.escape_decode', 'escape_encode' : 'app_codecs.escape_encode', - 'latin_1_decode' : 'app_codecs.latin_1_decode', - 'latin_1_encode' : 'app_codecs.latin_1_encode', - 'lookup' : 'app_codecs.lookup', - 'lookup_error' : 'app_codecs.lookup_error', 'mbcs_decode' : 'app_codecs.mbcs_decode', 'mbcs_encode' : 'app_codecs.mbcs_encode', 'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode', 'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode', 'readbuffer_encode' : 'app_codecs.readbuffer_encode', - 'register' : 'app_codecs.register', - 'register_error' : 'app_codecs.register_error', 'unicode_escape_decode' : 'app_codecs.unicode_escape_decode', 'unicode_escape_encode' : 'app_codecs.unicode_escape_encode', 'unicode_internal_decode' : 'app_codecs.unicode_internal_decode', 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', - 'utf_16_be_decode' : 'app_codecs.utf_16_be_decode', - 'utf_16_be_encode' : 'app_codecs.utf_16_be_encode', - 'utf_16_decode' : 'app_codecs.utf_16_decode', - 'utf_16_encode' : 'app_codecs.utf_16_encode', - 'utf_16_ex_decode' : 'app_codecs.utf_16_ex_decode', - 'utf_16_le_decode' : 'app_codecs.utf_16_le_decode', - 'utf_16_le_encode' : 'app_codecs.utf_16_le_encode', 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', - 'utf_8_decode' : 'app_codecs.utf_8_decode', - 'utf_8_encode' : 'app_codecs.utf_8_encode', - 'encode': 'app_codecs.encode', - 'decode': 'app_codecs.decode' + '_register_existing_errors': 'app_codecs._register_existing_errors', } interpleveldefs = { + 'encode': 'interp_codecs.encode', + 'decode': 'interp_codecs.decode', + 'lookup': 'interp_codecs.lookup_codec', + 'lookup_error': 'interp_codecs.lookup_error', + 'register': 'interp_codecs.register_codec', + 'register_error': 'interp_codecs.register_error', + + # encoders and decoders + 'ascii_decode' : 'interp_codecs.ascii_decode', + 'ascii_encode' : 'interp_codecs.ascii_encode', + 'latin_1_decode' : 'interp_codecs.latin_1_decode', + 'latin_1_encode' : 'interp_codecs.latin_1_encode', + 'utf_8_decode' : 'interp_codecs.utf_8_decode', + 'utf_8_encode' : 'interp_codecs.utf_8_encode', + 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', + 'utf_16_be_encode' : 'interp_codecs.utf_16_be_encode', + 'utf_16_decode' : 'interp_codecs.utf_16_decode', + 'utf_16_encode' : 'interp_codecs.utf_16_encode', + 'utf_16_le_decode' : 'interp_codecs.utf_16_le_decode', + 'utf_16_le_encode' : 'interp_codecs.utf_16_le_encode', + 'utf_16_ex_decode' : 'interp_codecs.utf_16_ex_decode', } + + def setup_after_space_initialization(self): + "NOT_RPYTHON" + self.space.appexec([], """(): + import _codecs + _codecs._register_existing_errors() + """) Modified: pypy/dist/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/app_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/app_codecs.py Wed Nov 14 20:28:11 2007 @@ -41,107 +41,7 @@ #from unicodecodec import * import sys -#/* --- Registry ----------------------------------------------------------- */ -codec_search_path = [] -codec_search_cache = {} -codec_error_registry = {} -codec_need_encodings = [True] -def codec_register( search_function ): - """register(search_function) - - Register a codec search function. Search functions are expected to take - one argument, the encoding name in all lower case letters, and return - a tuple of functions (encoder, decoder, stream_reader, stream_writer). - """ - - if callable(search_function): - codec_search_path.append(search_function) - -register = codec_register - -def codec_lookup(encoding): - """lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) - Looks up a codec tuple in the Python codec registry and returns - a tuple of functions. - """ - if not isinstance(encoding, str): - raise TypeError("Encoding must be a string") - normalized_encoding = encoding.replace(" ", "-").lower() - result = codec_search_cache.get(normalized_encoding, None) - if not result: - if codec_need_encodings: - import encodings - if len(codec_search_path) == 0: - raise LookupError("no codec search functions registered: can't find encoding") - del codec_need_encodings[:] - for search in codec_search_path: - result = search(normalized_encoding) - if result: - if not (type(result) == tuple and len(result) == 4): - raise TypeError("codec search functions must return 4-tuples") - else: - codec_search_cache[normalized_encoding] = result - return result - if not result: - raise LookupError("unknown encoding: %s" % encoding) - return result - - -lookup = codec_lookup - -def encode(v, encoding=None, errors='strict'): - """encode(obj, [encoding[,errors]]) -> object - - Encodes obj using the codec registered for encoding. encoding defaults - to the default encoding. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a ValueError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with - codecs.register_error that can handle ValueErrors. - """ - if encoding == None: - encoding = sys.getdefaultencoding() - if isinstance(encoding, str): - encoder = lookup(encoding)[0] - if encoder and isinstance(errors, str): - res = encoder(v, errors) - return res[0] - else: - raise TypeError("Errors must be a string") - else: - raise TypeError("Encoding must be a string") - -def decode(obj, encoding=None, errors='strict'): - """decode(obj, [encoding[,errors]]) -> object - - Decodes obj using the codec registered for encoding. encoding defaults - to the default encoding. errors may be given to set a different error - handling scheme. Default is 'strict' meaning that encoding errors raise - a ValueError. Other possible values are 'ignore' and 'replace' - as well as any other name registerd with codecs.register_error that is - able to handle ValueErrors. - """ - if encoding == None: - encoding = sys.getdefaultencoding() - if isinstance(encoding, str): - decoder = lookup(encoding)[1] - if decoder and isinstance(errors, str): - res = decoder(obj, errors) - if not isinstance(res, tuple) or len(res) != 2: - raise TypeError("encoder must return a tuple (object, integer)") - return res[0] - else: - raise TypeError("Errors must be a string") - else: - raise TypeError("Encoding must be a string") - -def latin_1_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeLatin1(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) # XXX MBCS codec might involve ctypes ? def mbcs_decode(): """None @@ -161,16 +61,6 @@ v = s[1:-1] return v, len(v) -def utf_8_decode( data, errors='strict', final=False): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed = PyUnicode_DecodeUTF8Stateful(data, len(data), errors, final) - res = u''.join(res) - return res, consumed - def raw_unicode_escape_decode( data, errors='strict'): """None """ @@ -192,23 +82,6 @@ res = ''.join(res) return res, len(res) -def latin_1_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = u''.join(res) - return res, len(res) - -def utf_16_decode( data, errors='strict', final=False): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) - return res, consumed - def unicode_escape_decode( data, errors='strict'): """None """ @@ -217,13 +90,6 @@ return res, len(res) -def ascii_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeASCII(data, len(data), errors) - res = u''.join(res) - return res, len(res) - def charmap_encode(obj, errors='strict', mapping='latin-1'): """None """ @@ -284,22 +150,6 @@ res = u''.join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ - if byteorder == 0: - bm = 'native' - elif byteorder == -1: - bm = 'little' - else: - bm = 'big' - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) - return res, consumed, byteorder - # XXX needs error messages when the input is invalid def escape_decode(data, errors='strict'): """None @@ -383,20 +233,6 @@ ## len(obj)) -def ascii_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeASCII(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) - -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') - res = ''.join(res) - return res, len(res) - def raw_unicode_escape_encode( obj, errors='strict'): """None """ @@ -404,47 +240,6 @@ res = ''.join(res) return res, len(res) -def utf_8_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF8(obj, len(obj), errors) - res = ''.join(res) - return res, len(res) - -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') - res = ''.join(res) - return res, len(res) - -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') - res = ''.join(res) - return res, len(res) - -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = u''.join(res) - return res, consumed - -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = u''.join(res) - return res, consumed - def strict_errors(exc): if isinstance(exc, Exception): raise exc @@ -500,6 +295,14 @@ raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) +def _register_existing_errors(): + import _codecs + _codecs.register_error("strict", strict_errors) + _codecs.register_error("ignore", ignore_errors) + _codecs.register_error("replace", replace_errors) + _codecs.register_error("xmlcharrefreplace", xmlcharrefreplace_errors) + _codecs.register_error("backslashreplace", backslashreplace_errors) + # ---------------------------------------------------------------------- ##import sys @@ -528,41 +331,7 @@ 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, ] -unicode_latin1 = [None]*256 - -def lookup_error(errors): - """lookup_error(errors) -> handler - - Return the error handler for the specified error handling name - or raise a LookupError, if no handler exists under this name. - """ - - try: - err_handler = codec_error_registry[errors] - except KeyError: - raise LookupError("unknown error handler name %s"%errors) - return err_handler - -def register_error(errors, handler): - """register_error(errors, handler) - - Register the specified error handler under the name - errors. handler must be a callable object, that - will be called with an exception instance containing - information about the location of the encoding/decoding - error and must return a (replacement, new position) tuple. - """ - if callable(handler): - codec_error_registry[errors] = handler - else: - raise TypeError("handler must be callable") - -register_error("strict", strict_errors) -register_error("ignore", ignore_errors) -register_error("replace", replace_errors) -register_error("xmlcharrefreplace", xmlcharrefreplace_errors) -register_error("backslashreplace", backslashreplace_errors) def SPECIAL(c, encodeO, encodeWS): c = ord(c) @@ -831,197 +600,6 @@ p += p[1] return p -def PyUnicode_DecodeASCII(s, size, errors): - -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : - return [unichr(ord(s))] - if (size == 0): - return [u''] #unicode('') - p = [] - pos = 0 - while pos < len(s): - c = s[pos] - if ord(c) < 128: - p += unichr(ord(c)) - pos += 1 - else: - - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) - p += [unichr(ord(x)) for x in res[0]] - pos = res[1] - return p - -def PyUnicode_EncodeASCII(p, size, errors): - - return unicode_encode_ucs1(p, size, errors, 128) - -def PyUnicode_AsASCIIString(unistr): - - if not type(unistr) == unicode: - raise TypeError - return PyUnicode_EncodeASCII(unicode(unistr), - len(unicode), - None) - -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - - bo = 0 #/* assume native ordering by default */ - consumed = 0 - errmsg = "" - - if sys.byteorder == 'little': - ihi = 1 - ilo = 0 - else: - ihi = 0 - ilo = 1 - - - #/* Unpack UTF-16 encoded data */ - -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ - q = 0 - p = [] - if byteorder == 'native': - if (size >= 2): - bom = (ord(s[ihi]) << 8) | ord(s[ilo]) -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): - q += 2 - bo = -1 - elif bom == 0xFFFE: - q += 2 - bo = 1 - else: - if bom == 0xFEFF: - q += 2 - bo = 1 - elif bom == 0xFFFE: - q += 2 - bo = -1 - elif byteorder == 'little': - bo = -1 - else: - bo = 1 - - if (size == 0): - return [u''], 0, bo - - if (bo == -1): - #/* force LE */ - ihi = 1 - ilo = 0 - - elif (bo == 1): - #/* force BE */ - ihi = 0 - ilo = 1 - - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): - if not final: - break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): - p += unichr(ch) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) - q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += unichr(ch) - p += unichr(ch2) - else: - p += unichr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) - #endif - continue - - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - return p, q, bo - -# moved out of local scope, especially because it didn't -# have any nested variables. - -def STORECHAR(CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - return [lo, hi] - else: - return [hi, lo] - -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): - -# /* Offsets from p for storing byte pairs in the right order. */ - - - p = [] - bom = sys.byteorder - if (byteorder == 'native'): - - bom = sys.byteorder - p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' - - - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) - - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) - - return p - def PyUnicode_DecodeMBCS(s, size, errors): pass @@ -1032,7 +610,8 @@ def unicode_call_errorhandler(errors, encoding, reason, input, startinpos, endinpos, decode=True): - errorHandler = lookup_error(errors) + import _codecs + errorHandler = _codecs.lookup_error(errors) if decode: exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) else: @@ -1048,288 +627,7 @@ else: raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) -def PyUnicode_DecodeUTF8(s, size, errors): - return PyUnicode_DecodeUTF8Stateful(s, size, errors, False) - -## /* Map UTF-8 encoded prefix byte to sequence length. zero means -## illegal prefix. see RFC 2279 for details */ -utf8_code_length = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 -] - -def PyUnicode_DecodeUTF8Stateful(s, size, errors, final): - - consumed = 0 - if (size == 0): - if not final: - consumed = 0 - return u'', consumed - p = [] - pos = 0 - while pos < size: - ch = s[pos] - if ord(ch) < 0x80: - p += ch - pos += 1 - continue - - n = utf8_code_length[ord(ch)] - startinpos = pos - if (startinpos + n > size): - if not final: - break - else: - errmsg = "unexpected end of data" - endinpos = size - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - if n == 0: - errmsg = "unexpected code byte" - endinpos = startinpos+1 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - elif n == 1: - errmsg = "internal error" - endinpos = startinpos+1 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - elif n == 2: - if ((ord(s[pos+1]) & 0xc0) != 0x80): - errmsg = "invalid data" - endinpos = startinpos+2 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos]) & 0x1f) << 6) + (ord(s[pos+1]) & 0x3f) - if c < 0x80: - errmsg = "illegal encoding" - endinpos = startinpos+2 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - p += unichr(c) - pos += n - #break - elif n == 3: - if ((ord(s[pos+1]) & 0xc0) != 0x80 or - (ord(s[pos+2]) & 0xc0) != 0x80): - errmsg = "invalid data" - endinpos = startinpos+3 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos]) & 0x0f) << 12) + \ - ((ord(s[pos+1]) & 0x3f) << 6) +\ - (ord(s[pos+2]) & 0x3f) - -## /* Note: UTF-8 encodings of surrogates are considered -## legal UTF-8 sequences; -## -## XXX For wide builds (UCS-4) we should probably try -## to recombine the surrogates into a single code -## unit. -## */ - if c < 0x0800: - errmsg = "illegal encoding" - endinpos = startinpos+3 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - p += unichr(c) - pos += n - elif n == 4: -## case 4: - if ((ord(s[pos+1]) & 0xc0) != 0x80 or - (ord(s[pos+2]) & 0xc0) != 0x80 or - (ord(s[pos+3]) & 0xc0) != 0x80): - - errmsg = "invalid data" - startinpos = pos - endinpos = startinpos+4 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: - c = ((ord(s[pos+0]) & 0x7) << 18) + ((ord(s[pos+1]) & 0x3f) << 12) +\ - ((ord(s[pos+2]) & 0x3f) << 6) + (ord(s[pos+3]) & 0x3f) - #/* validate and convert to UTF-16 */ - if ((c < 0x10000) or (c > 0x10ffff)): - #/* minimum value allowed for 4 byte encoding */ - #/* maximum value allowed for UTF-16 */ - - errmsg = "illegal encoding" - startinpos = pos - endinpos = startinpos+4 - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - else: -#ifdef Py_UNICODE_WIDE - if c < sys.maxunicode: - p += unichr(c) - pos += n - else: -## /* compute and append the two surrogates: */ -## /* translate from 10000..10FFFF to 0..FFFF */ - c -= 0x10000 - #/* high surrogate = top 10 bits added to D800 */ - p += unichr(0xD800 + (c >> 10)) - #/* low surrogate = bottom 10 bits added to DC00 */ - p += unichr(0xDC00 + (c & 0x03FF)) - pos += n - else: -## default: -## /* Other sizes are only needed for UCS-4 */ - errmsg = "unsupported Unicode code range" - startinpos = pos - endinpos = startinpos+n - res = unicode_call_errorhandler( - errors, "utf8", errmsg, - s, startinpos, endinpos) - p += res[0] - pos = res[1] - - #continue - - if not final: - consumed = pos - return p, pos # consumed -def PyUnicode_EncodeUTF8(s, size, errors): - - #assert(s != None) - assert(size >= 0) - p = [] - i = 0 - while i < size: - ch = s[i] - i += 1 - if (ord(ch) < 0x80): -## /* Encode ASCII */ - p += chr(ord(ch)) - elif (ord(ch) < 0x0800) : -## /* Encode Latin-1 */ - p += chr((0xc0 | (ord(ch) >> 6))) - p += chr((0x80 | (ord(ch) & 0x3f))) - else: -## /* Encode UCS2 Unicode ordinals */ - if (ord(ch) < 0x10000): -## /* Special case: check for high surrogate */ - if (0xD800 <= ord(ch) and ord(ch) <= 0xDBFF and i != size) : - ch2 = s[i] -## /* Check for low surrogate and combine the two to -## form a UCS4 value */ - if (0xDC00 <= ord(ch2) and ord(ch2) <= 0xDFFF) : - ch3 = ((ord(ch) - 0xD800) << 10 | (ord(ch2) - 0xDC00)) + 0x10000 - i += 1 - p.extend(encodeUCS4(ch3)) - continue -## /* Fall through: handles isolated high surrogates */ - p += (chr((0xe0 | (ord(ch) >> 12)))) - p += (chr((0x80 | ((ord(ch) >> 6) & 0x3f)))) - p += (chr((0x80 | (ord(ch) & 0x3f)))) - continue - else: - p.extend(encodeUCS4(ord(ch))) - return p - -def encodeUCS4(ch): -## /* Encode UCS4 Unicode ordinals */ - p = [] - p += (chr((0xf0 | (ch >> 18)))) - p += (chr((0x80 | ((ch >> 12) & 0x3f)))) - p += (chr((0x80 | ((ch >> 6) & 0x3f)))) - p += (chr((0x80 | (ch & 0x3f)))) - return p - -#/* --- Latin-1 Codec ------------------------------------------------------ */ - -def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] - pos = 0 - p = [] - while (pos < size): - p += unichr(ord(s[pos])) - pos += 1 - return p - -def unicode_encode_ucs1(p, size, errors, limit): - - if limit == 256: - reason = "ordinal not in range(256)" - encoding = "latin-1" - else: - reason = "ordinal not in range(128)" - encoding = "ascii" - - if (size == 0): - return [''] - res = [] - pos = 0 - while pos < len(p): - #for ch in p: - ch = p[pos] - - if ord(ch) < limit: - res += chr(ord(ch)) - pos += 1 - else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 - while collend < len(p) and ord(p[collend]) >= limit: - collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += str(x[0]) - pos = x[1] - - return res - -def PyUnicode_EncodeLatin1(p, size, errors): - res = unicode_encode_ucs1(p, size, errors, 256) - return res hexdigits = [hex(i)[-1] for i in range(16)]+[hex(i)[-1].upper() for i in range(10, 16)] @@ -1523,7 +821,8 @@ # /* Default to Latin-1 */ if mapping == 'latin-1': - return PyUnicode_EncodeLatin1(p, size, errors) + import _codecs + return _codecs.latin_1_encode(p, size, errors) if (size == 0): return '' inpos = 0 @@ -1548,7 +847,8 @@ ## /* Default to Latin-1 */ if (mapping == None): - return PyUnicode_DecodeLatin1(s, size, errors) + import _codecs + return _codecs.latin_1_decode(s, size, errors) if (size == 0): return u'' Modified: pypy/dist/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/test/test_codecs.py Wed Nov 14 20:28:11 2007 @@ -6,6 +6,10 @@ space = gettestobjspace(usemodules=('unicodedata',)) cls.space = space + def test_register_noncallable(self): + import _codecs + raises(TypeError, _codecs.register, 1) + def test_bigU_codecs(self): import sys oldmaxunicode = sys.maxunicode @@ -257,6 +261,20 @@ assert '\\253'.decode('string_escape') == chr(0253) assert '\\312'.decode('string_escape') == chr(0312) + def test_decode_utf8_different_case(self): constant = u"a" assert constant.encode("utf-8") == constant.encode("UTF-8") + + def test_codec_wrong_result(self): + import _codecs + def search_function(encoding): + def f(input, errors="strict"): + return 42 + print encoding + if encoding == 'test.mytestenc': + return (f, f, None, None) + return None + _codecs.register(search_function) + raises(TypeError, "hello".decode, "test.mytestenc") + raises(TypeError, u"hello".encode, "test.mytestenc") Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Wed Nov 14 20:28:11 2007 @@ -1030,44 +1030,49 @@ return W_RopeObject(rope.rope_from_charlist(buf[:i+1])) - -app = gateway.applevel(r''' - def str_translate__Rope_ANY_ANY(s, table, deletechars=''): - """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, - which must be a string of length 256""" - - if len(table) != 256: - raise ValueError("translation table must be 256 characters long") - - L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] - return ''.join(L) - - def str_decode__Rope_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getdecoder(encoding)(str)[0] - else: - return codecs.getdecoder(encoding)(str, errors)[0] - - def str_encode__Rope_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getencoder(encoding)(str)[0] - else: - return codecs.getencoder(encoding)(str, errors)[0] -''', filename=__file__) - -str_translate__Rope_ANY_ANY = app.interphook('str_translate__Rope_ANY_ANY') -str_decode__Rope_ANY_ANY = app.interphook('str_decode__Rope_ANY_ANY') -str_encode__Rope_ANY_ANY = app.interphook('str_encode__Rope_ANY_ANY') +def str_translate__Rope_ANY_ANY(space, w_string, w_table, w_deletechars=''): + """charfilter - unicode handling is not implemented + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, + which must be a string of length 256""" + + # XXX CPython accepts buffers, too, not sure what we should do + table = space.str_w(w_table) + if len(table) != 256: + raise OperationError( + space.w_ValueError, + space.wrap("translation table must be 256 characters long")) + + node = w_string._node + chars = [] + iter = rope.CharIterator(node) + while 1: + try: + c = iter.next() + i += 1 + w_char = W_RopeObject.PREBUILT[ord(char)] + if not space.is_true(space.contains(w_deletechars, w_char)): + chars.append(table[ord(char)]) + except StopIteration: + break + return W_RopeObject(rope.rope_from_charlist(chars)) + +def str_decode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + unicode_from_string, decode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + if encoding is None and errors is None: + return unicode_from_string(space, w_string) + return decode_object(space, w_string, encoding, errors) + +def str_encode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + return encode_object(space, w_string, encoding, errors) + # methods of the iterator Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Wed Nov 14 20:28:11 2007 @@ -51,7 +51,6 @@ # XXX should this use the default encoding? return _decode_ascii(space, w_self._value) - def _is_generic(space, w_self, fun): v = w_self._value if len(v) == 0: @@ -890,44 +889,43 @@ return space.wrap("".join(buf[:i+1])) # buffer was overallocated, so slice -app = gateway.applevel(r''' - def str_translate__String_ANY_ANY(s, table, deletechars=''): - """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, - which must be a string of length 256""" - - if len(table) != 256: - raise ValueError("translation table must be 256 characters long") - - L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] - return ''.join(L) - - def str_decode__String_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getdecoder(encoding)(str)[0] - else: - return codecs.getdecoder(encoding)(str, errors)[0] - - def str_encode__String_ANY_ANY(str, encoding=None, errors=None): - import codecs - if encoding is None and errors is None: - return unicode(str) - elif errors is None: - return codecs.getencoder(encoding)(str)[0] - else: - return codecs.getencoder(encoding)(str, errors)[0] -''', filename=__file__) - +def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''): + """charfilter - unicode handling is not implemented + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, + which must be a string of length 256""" + + # XXX CPython accepts buffers, too, not sure what we should do + table = space.str_w(w_table) + if len(table) != 256: + raise OperationError( + space.w_ValueError, + space.wrap("translation table must be 256 characters long")) -str_translate__String_ANY_ANY = app.interphook('str_translate__String_ANY_ANY') -str_decode__String_ANY_ANY = app.interphook('str_decode__String_ANY_ANY') -str_encode__String_ANY_ANY = app.interphook('str_encode__String_ANY_ANY') + string = w_string._value + chars = [] + for char in string: + w_char = W_StringObject.PREBUILT[ord(char)] + if not space.is_true(space.contains(w_deletechars, w_char)): + chars.append(table[ord(char)]) + return W_StringObject(''.join(chars)) + +def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + unicode_from_string, decode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + if encoding is None and errors is None: + return unicode_from_string(space, w_string) + return decode_object(space, w_string, encoding, errors) + +def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): + #import pdb; pdb.set_trace() + from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ + encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + return encode_object(space, w_string, encoding, errors) # CPython's logic for deciding if ""%values is # an error (1 value, 0 %-formatters) or not Modified: pypy/dist/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_stringobject.py Wed Nov 14 20:28:11 2007 @@ -645,6 +645,10 @@ def test_decode(self): assert 'hello'.decode('rot-13') == 'uryyb' assert 'hello'.decode('string-escape') == 'hello' + + def test_encode(self): + assert 'hello'.encode() == 'hello' + assert type('hello'.encode()) is str def test_hash(self): # check that we have the same hash as CPython for at least 31 bits Modified: pypy/dist/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_unicodeobject.py Wed Nov 14 20:28:11 2007 @@ -1,3 +1,4 @@ +import py import sys @@ -377,3 +378,88 @@ def test_missing_cases(self): # some random cases, which are discovered to not be tested during annotation assert u'xxx'[1:1] == u'' + + # these tests test lots of encodings, so they really belong to the _codecs + # module. however, they test useful unicode methods too + # they are stolen from CPython's unit tests + + def test_codecs_utf7(self): + utfTests = [ + (u'A\u2262\u0391.', 'A+ImIDkQ.'), # RFC2152 example + (u'Hi Mom -\u263a-!', 'Hi Mom -+Jjo--!'), # RFC2152 example + (u'\u65E5\u672C\u8A9E', '+ZeVnLIqe-'), # RFC2152 example + (u'Item 3 is \u00a31.', 'Item 3 is +AKM-1.'), # RFC2152 example + (u'+', '+-'), + (u'+-', '+--'), + (u'+?', '+-?'), + (u'\?', '+AFw?'), + (u'+?', '+-?'), + (ur'\\?', '+AFwAXA?'), + (ur'\\\?', '+AFwAXABc?'), + (ur'++--', '+-+---') + ] + + for (x, y) in utfTests: + assert x.encode('utf-7') == y + + # surrogates not supported + raises(UnicodeError, unicode, '+3ADYAA-', 'utf-7') + + assert unicode('+3ADYAA-', 'utf-7', 'replace') == u'\ufffd' + + def test_codecs_utf8(self): + assert u''.encode('utf-8') == '' + assert u'\u20ac'.encode('utf-8') == '\xe2\x82\xac' + assert u'\ud800\udc02'.encode('utf-8') == '\xf0\x90\x80\x82' + assert u'\ud84d\udc56'.encode('utf-8') == '\xf0\xa3\x91\x96' + assert u'\ud800'.encode('utf-8') == '\xed\xa0\x80' + assert u'\udc00'.encode('utf-8') == '\xed\xb0\x80' + assert (u'\ud800\udc02'*1000).encode('utf-8') == '\xf0\x90\x80\x82'*1000 + assert ( + u'\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + u'\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + u'\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + u'\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + u'\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das' + u' Nunstuck git und'.encode('utf-8') == + '\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81' + '\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3' + '\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe' + '\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' + '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8' + '\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81' + '\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81' + '\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3' + '\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf' + '\xe3\x80\x8cWenn ist das Nunstuck git und' + ) + + # UTF-8 specific decoding tests + assert unicode('\xf0\xa3\x91\x96', 'utf-8') == u'\U00023456' + assert unicode('\xf0\x90\x80\x82', 'utf-8') == u'\U00010002' + assert unicode('\xe2\x82\xac', 'utf-8') == u'\u20ac' + + def test_codecs_errors(self): + # Error handling (encoding) + raises(UnicodeError, u'Andr\202 x'.encode, 'ascii') + raises(UnicodeError, u'Andr\202 x'.encode, 'ascii','strict') + assert u'Andr\202 x'.encode('ascii','ignore') == "Andr x" + assert u'Andr\202 x'.encode('ascii','replace') == "Andr? x" + + # Error handling (decoding) + raises(UnicodeError, unicode, 'Andr\202 x', 'ascii') + raises(UnicodeError, unicode, 'Andr\202 x', 'ascii','strict') + assert unicode('Andr\202 x','ascii','ignore') == u"Andr x" + assert unicode('Andr\202 x','ascii','replace') == u'Andr\uFFFD x' + + # Error handling (unknown character names) + assert "\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx" + + # Error handling (truncated escape sequence) + raises(UnicodeError, "\\".decode, "unicode-escape") + + def test_repr_bug(self): + assert (repr(u'\U00090418\u027d\U000582b9\u54c3\U000fcb6e') == + "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'") + assert (repr(u'\n') == + "u'\\n'") Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Wed Nov 14 20:28:11 2007 @@ -26,6 +26,12 @@ def unwrap(w_self, space): # for testing return w_self._value + + def create_if_subclassed(w_self): + if type(w_self) is W_UnicodeObject: + return w_self + return W_UnicodeObject(w_self._value) + W_UnicodeObject.EMPTY = W_UnicodeObject(u'') registerimplementation(W_UnicodeObject) @@ -59,7 +65,8 @@ # string-to-unicode delegation def delegate_String2Unicode(space, w_str): - w_uni = space.call_function(space.w_unicode, w_str) + from pypy.objspace.std.unicodetype import unicode_from_string + w_uni = unicode_from_string(space, w_str) assert isinstance(w_uni, W_UnicodeObject) # help the annotator! return w_uni @@ -92,17 +99,20 @@ return W_UnicodeObject(w_left._value + w_right._value) def add__String_Unicode(space, w_left, w_right): - return space.add(space.call_function(space.w_unicode, w_left) , w_right) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.add(unicode_from_string(space, w_left) , w_right) add__Rope_Unicode = add__String_Unicode def add__Unicode_String(space, w_left, w_right): - return space.add(w_left, space.call_function(space.w_unicode, w_right)) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.add(w_left, unicode_from_string(space, w_right)) add__Unicode_Rope = add__Unicode_String def contains__String_Unicode(space, w_container, w_item): - return space.contains(space.call_function(space.w_unicode, w_container), w_item ) + from pypy.objspace.std.unicodetype import unicode_from_string + return space.contains(unicode_from_string(space, w_container), w_item ) contains__Rope_Unicode = contains__String_Unicode @@ -311,8 +321,9 @@ def unicode_strip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 1, 1) def unicode_strip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'strip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_strip__Unicode_Rope = unicode_strip__Unicode_String def unicode_lstrip__Unicode_None(space, w_self, w_chars): @@ -320,8 +331,9 @@ def unicode_lstrip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 1, 0) def unicode_lstrip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'lstrip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_lstrip__Unicode_Rope = unicode_lstrip__Unicode_String @@ -330,8 +342,9 @@ def unicode_rstrip__Unicode_Unicode(space, w_self, w_chars): return _strip(space, w_self, w_chars, 0, 1) def unicode_rstrip__Unicode_String(space, w_self, w_chars): + from pypy.objspace.std.unicodetype import unicode_from_string return space.call_method(w_self, 'rstrip', - space.call_function(space.w_unicode, w_chars)) + unicode_from_string(space, w_chars)) unicode_rstrip__Unicode_Rope = unicode_rstrip__Unicode_String @@ -481,7 +494,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() leftpad = padding // 2 + (padding & width & 1) result = [fillchar] * width for i in range(len(self)): @@ -494,7 +507,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [fillchar] * width for i in range(len(self)): result[i] = self[i] @@ -506,7 +519,7 @@ fillchar = _to_unichar_w(space, w_fillchar) padding = width - len(self) if padding < 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [fillchar] * width for i in range(len(self)): result[padding + i] = self[i] @@ -519,7 +532,7 @@ return W_UnicodeObject(u'0' * width) padding = width - len(self) if padding <= 0: - return space.call_function(space.w_unicode, w_self) + return w_self.create_if_subclassed() result = [u'0'] * width for i in range(len(self)): result[padding + i] = self[i] @@ -735,28 +748,17 @@ return W_UnicodeObject(w_new._value.join(parts)) -app = gateway.applevel(r''' -import sys - -def unicode_encode__Unicode_ANY_ANY(unistr, encoding=None, errors=None): - import codecs, sys +def unicode_encode__Unicode_ANY_ANY(space, w_unistr, + w_encoding=None, + w_errors=None): + + from pypy.objspace.std.unicodetype import getdefaultencoding, \ + _get_encoding_and_errors, encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None: - encoding = sys.getdefaultencoding() - - encoder = codecs.getencoder(encoding) - if errors is None: - retval, lenght = encoder(unistr) - else: - retval, length = encoder(unistr, errors) - if not isinstance(retval,str): - raise TypeError("encoder did not return a string object (type=%s)" % - type(retval).__name__) - return retval -''') - - - -unicode_encode__Unicode_ANY_ANY = app.interphook('unicode_encode__Unicode_ANY_ANY') + encoding = getdefaultencoding(space) + w_retval = encode_object(space, w_unistr, encoding, errors) + return w_retval def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): unistr = w_unistr._value @@ -859,36 +861,23 @@ quote = '"' else: quote = '\'' - result = ['\0'] * (3 + size*6) - result[0] = 'u' - result[1] = quote - i = 2 + result = ['u', quote] j = 0 while j= 0x10000: # Resize if needed - if i + 12 > len(result): - result.extend(['\0'] * 100) - result[i] = '\\' - result[i + 1] = "U" - result[i + 2] = hexdigits[(code >> 28) & 0xf] - result[i + 3] = hexdigits[(code >> 24) & 0xf] - result[i + 4] = hexdigits[(code >> 20) & 0xf] - result[i + 5] = hexdigits[(code >> 16) & 0xf] - result[i + 6] = hexdigits[(code >> 12) & 0xf] - result[i + 7] = hexdigits[(code >> 8) & 0xf] - result[i + 8] = hexdigits[(code >> 4) & 0xf] - result[i + 9] = hexdigits[(code >> 0) & 0xf] - i += 10 + result.extend(['\\', "U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue if code >= 0xD800 and code < 0xDC00: @@ -897,70 +886,59 @@ code2 = ord(ch2) if code2 >= 0xDC00 and code2 <= 0xDFFF: code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 - if i + 12 > len(result): - result.extend(['\0'] * 100) - result[i] = '\\' - result[i + 1] = "U" - result[i + 2] = hexdigits[(code >> 28) & 0xf] - result[i + 3] = hexdigits[(code >> 24) & 0xf] - result[i + 4] = hexdigits[(code >> 20) & 0xf] - result[i + 5] = hexdigits[(code >> 16) & 0xf] - result[i + 6] = hexdigits[(code >> 12) & 0xf] - result[i + 7] = hexdigits[(code >> 8) & 0xf] - result[i + 8] = hexdigits[(code >> 4) & 0xf] - result[i + 9] = hexdigits[(code >> 0) & 0xf] - i += 10 + result.extend(["U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 2 continue if code >= 0x100: - result[i] = '\\' - result[i + 1] = "u" - result[i + 2] = hexdigits[(code >> 12) & 0xf] - result[i + 3] = hexdigits[(code >> 8) & 0xf] - result[i + 4] = hexdigits[(code >> 4) & 0xf] - result[i + 5] = hexdigits[(code >> 0) & 0xf] - i += 6 + result.extend(['\\', "u", + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue if code == ord('\\') or code == ord(quote): - result[i] = '\\' - result[i + 1] = chr(code) - i += 2 + result.append('\\') + result.append(chr(code)) j += 1 continue if code == ord('\t'): - result[i] = '\\' - result[i + 1] = "t" - i += 2 + result.append('\\') + result.append('t') j += 1 continue if code == ord('\r'): - result[i] = '\\' - result[i + 1] = "r" - i += 2 + result.append('\\') + result.append('r') j += 1 continue if code == ord('\n'): - result[i] = '\\' - result[i + 1] = "n" - i += 2 + result.append('\\') + result.append('n') j += 1 continue if code < ord(' ') or code >= 0x7f: - result[i] = '\\' - result[i + 1] = "x" - result[i + 2] = hexdigits[(code >> 4) & 0xf] - result[i + 3] = hexdigits[(code >> 0) & 0xf] - i += 4 + result.extend(['\\', "x", + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) j += 1 continue - result[i] = chr(code) - i += 1 + result.append(chr(code)) j += 1 - result[i] = quote - i += 1 - return space.wrap(''.join(result[:i])) + result.append(quote) + return space.wrap(''.join(result)) def mod__Unicode_ANY(space, w_format, w_values): @@ -983,47 +961,58 @@ from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.ropeobject import W_RopeObject def str_strip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'strip', w_chars) str_strip__Rope_Unicode = str_strip__String_Unicode def str_lstrip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'lstrip', w_chars) str_lstrip__Rope_Unicode = str_lstrip__String_Unicode def str_rstrip__String_Unicode(space, w_self, w_chars): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rstrip', w_chars) str_rstrip__Rope_Unicode = str_rstrip__String_Unicode def str_count__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'count', w_substr, w_start, w_end) str_count__Rope_Unicode_ANY_ANY = str_count__String_Unicode_ANY_ANY def str_find__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'find', w_substr, w_start, w_end) str_find__Rope_Unicode_ANY_ANY = str_find__String_Unicode_ANY_ANY def str_rfind__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rfind', w_substr, w_start, w_end) str_rfind__Rope_Unicode_ANY_ANY = str_rfind__String_Unicode_ANY_ANY def str_index__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'index', w_substr, w_start, w_end) str_index__Rope_Unicode_ANY_ANY = str_index__String_Unicode_ANY_ANY def str_rindex__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rindex', w_substr, w_start, w_end) str_rindex__Rope_Unicode_ANY_ANY = str_rindex__String_Unicode_ANY_ANY def str_replace__String_Unicode_Unicode_ANY(space, w_self, w_old, w_new, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'replace', w_old, w_new, w_maxsplit) str_replace__Rope_Unicode_Unicode_ANY = str_replace__String_Unicode_Unicode_ANY def str_split__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'split', w_delim, w_maxsplit) str_split__Rope_Unicode_ANY = str_split__String_Unicode_ANY def str_rsplit__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit): - return space.call_method(space.call_function(space.w_unicode, w_self), + from pypy.objspace.std.unicodetype import unicode_from_string + return space.call_method(unicode_from_string(space, w_self), 'rsplit', w_delim, w_maxsplit) str_rsplit__Rope_Unicode_ANY = str_rsplit__String_Unicode_ANY register_all(vars(), stringtype) Modified: pypy/dist/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodetype.py (original) +++ pypy/dist/pypy/objspace/std/unicodetype.py Wed Nov 14 20:28:11 2007 @@ -145,7 +145,36 @@ def getdefaultencoding(space): return space.sys.defaultencoding -def unicode_from_encoded_object(space, w_obj, encoding, errors): +def _get_encoding_and_errors(space, w_encoding, w_errors): + if space.is_w(w_encoding, space.w_None): + encoding = None + else: + encoding = space.str_w(w_encoding) + if space.is_w(w_errors, space.w_None): + errors = None + else: + errors = space.str_w(w_errors) + return encoding, errors + +def encode_object(space, w_object, encoding, errors): + w_codecs = space.getbuiltinmodule("_codecs") + w_encode = space.getattr(w_codecs, space.wrap("encode")) + if encoding is None: + encoding = getdefaultencoding(space) + if errors is None: + w_retval = space.call_function(w_encode, w_object, space.wrap(encoding)) + else: + w_retval = space.call_function(w_encode, w_object, space.wrap(encoding), + space.wrap(errors)) + if not space.is_true(space.isinstance(w_retval, space.w_str)): + raise OperationError( + space.w_TypeError, + space.wrap( + "encoder did not return an string object (type=%s)" % + space.type(w_retval).getname(space, '?'))) + return w_retval + +def decode_object(space, w_obj, encoding, errors): w_codecs = space.getbuiltinmodule("_codecs") if encoding is None: encoding = getdefaultencoding(space) @@ -155,6 +184,11 @@ else: w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding), space.wrap(errors)) + return w_retval + + +def unicode_from_encoded_object(space, w_obj, encoding, errors): + w_retval = decode_object(space, w_obj, encoding, errors) if not space.is_true(space.isinstance(w_retval, space.w_unicode)): raise OperationError( space.w_TypeError, @@ -163,7 +197,6 @@ space.type(w_retval).getname(space, '?'))) return w_retval - def unicode_from_object(space, w_obj): if space.is_true(space.isinstance(w_obj, space.w_str)): w_res = w_obj @@ -188,7 +221,7 @@ from pypy.objspace.std.unicodeobject import W_UnicodeObject encoding = getdefaultencoding(space) if encoding != 'ascii': - return unicode_from_object(space, w_str) + return unicode_from_encoded_object(space, w_str, encoding, "strict") s = space.str_w(w_str) try: return W_UnicodeObject(s.decode("ascii")) @@ -197,17 +230,6 @@ return unicode_from_object(space, w_str) -def _get_encoding_and_errors(space, w_encoding, w_errors): - if space.is_w(w_encoding, space.w_None): - encoding = None - else: - encoding = space.str_w(w_encoding) - if space.is_w(w_errors, space.w_None): - errors = None - else: - errors = space.str_w(w_errors) - return encoding, errors - def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work @@ -231,7 +253,7 @@ w_value = unicode_from_object(space, w_obj) else: w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) - # help the annotator! also the ._value depends on W_UnicodeObject layout + # XXX this is not true when there are different unicode implementations assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) W_UnicodeObject.__init__(w_newobj, w_value._value) From cfbolz at codespeak.net Wed Nov 14 20:36:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 20:36:28 +0100 (CET) Subject: [pypy-svn] r48689 - pypy/dist/pypy/objspace/std Message-ID: <20071114193628.D68ED828C@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 20:36:28 2007 New Revision: 48689 Modified: pypy/dist/pypy/objspace/std/ropeobject.py Log: whoops, broken tests. sorry for this Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Wed Nov 14 20:36:28 2007 @@ -1051,10 +1051,9 @@ while 1: try: c = iter.next() - i += 1 - w_char = W_RopeObject.PREBUILT[ord(char)] + w_char = W_RopeObject.PREBUILT[ord(c)] if not space.is_true(space.contains(w_deletechars, w_char)): - chars.append(table[ord(char)]) + chars.append(table[ord(c)]) except StopIteration: break return W_RopeObject(rope.rope_from_charlist(chars)) From cfbolz at codespeak.net Wed Nov 14 20:37:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 20:37:43 +0100 (CET) Subject: [pypy-svn] r48690 - pypy/branch/more-unicode-improvements Message-ID: <20071114193743.2A16582A1@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 20:37:42 2007 New Revision: 48690 Removed: pypy/branch/more-unicode-improvements/ Log: kill merged branch From cfbolz at codespeak.net Wed Nov 14 20:39:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 14 Nov 2007 20:39:36 +0100 (CET) Subject: [pypy-svn] r48691 - pypy/branch/ropes-unicode Message-ID: <20071114193936.C533C82AD@code0.codespeak.net> Author: cfbolz Date: Wed Nov 14 20:39:36 2007 New Revision: 48691 Added: pypy/branch/ropes-unicode/ - copied from r48690, pypy/dist/ Log: a new branch to try some crazy ideas about ropes and unicode From fijal at codespeak.net Wed Nov 14 20:50:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Nov 2007 20:50:16 +0100 (CET) Subject: [pypy-svn] r48692 - pypy/dist/pypy/translator/tool Message-ID: <20071114195016.D346482C8@code0.codespeak.net> Author: fijal Date: Wed Nov 14 20:50:16 2007 New Revision: 48692 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: * Fix the bug * Make src dir to be there by default Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Wed Nov 14 20:50:16 2007 @@ -2,6 +2,7 @@ import os, sys, inspect, re, imp from pypy.translator.tool import stdoutcapture +from pypy.tool.autopath import pypydir import py from pypy.tool.ansi_print import ansi_log @@ -48,6 +49,7 @@ # print "ERROR IMPORTING" # pass include_dirs = list(include_dirs) + include_dirs.append(py.path.local(pypydir).join('translator', 'c')) library_dirs = list(library_dirs) if sys.platform == 'darwin': # support Fink & Darwinports for s in ('/sw/', '/opt/local/'): @@ -141,7 +143,7 @@ lastdir.chdir() def cache_c_module(cfiles, modname, cache_dir=None, - include_dirs=None, libraries=[]): + include_dirs=[], libraries=[]): """ Same as build c module, but instead caches results. XXX currently there is no way to force a recompile, so this is pretty useless as soon as the sources (or headers they depend on) change :-/ From pypy-svn at codespeak.net Wed Nov 14 21:53:54 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Wed, 14 Nov 2007 21:53:54 +0100 (CET) Subject: [pypy-svn] October 76% OFF Message-ID: <20071114105508.18845.qmail@Z82cf.z.pppool.de> An HTML attachment was scrubbed... URL: From fijal at codespeak.net Thu Nov 15 00:03:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 00:03:52 +0100 (CET) Subject: [pypy-svn] r48696 - in pypy/dist/pypy/lib: app_test ctypes Message-ID: <20071114230352.5CA4682D2@code0.codespeak.net> Author: fijal Date: Thu Nov 15 00:03:51 2007 New Revision: 48696 Added: pypy/dist/pypy/lib/app_test/test_ctypes.py (contents, props changed) pypy/dist/pypy/lib/ctypes/ pypy/dist/pypy/lib/ctypes/__init__.py (contents, props changed) Log: Add some play around app-level ctypes Added: pypy/dist/pypy/lib/app_test/test_ctypes.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/test_ctypes.py Thu Nov 15 00:03:51 2007 @@ -0,0 +1,24 @@ + +from pypy.conftest import gettestobjspace +import sys, py + +def setup_module(mod): + if sys.platform != 'linux2': + py.test.skip("Linux only tests by now") + +class AppTestCtypes: + def setup_class(cls): + space = gettestobjspace(usemodules=('_ffi','struct')) + + def test_rand(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + rand = libc.rand + first = rand() + counter = 0 + for i in range(100): + next = rand() + if next == first: + counter += 1 + assert counter < 100 + Added: pypy/dist/pypy/lib/ctypes/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/ctypes/__init__.py Thu Nov 15 00:03:51 2007 @@ -0,0 +1,68 @@ + +import _ffi +import sys + +DEFAULT_MODE = None # XXX, mode support in _ffi + +class _CFuncPtr(object): + def __init__(self, (name, lib)): + self.name = name + self.lib = lib + + def __call__(self, *args): + if not hasattr(self, '_handle'): + self._update_handle() + # XXX eventually cast types here + return self._handle(*args) + + def _update_handle(self): + llargs = [i._lltype for i in self.argtypes] + # XXX first approximation + self._handle = self.lib._handle.ptr(self.name, llargs, + self.restype._lltype) + +class c_int(object): + _lltype = 'i' + +class CDLL(object): + """An instance of this class represents a loaded dll/shared + library, exporting functions using the standard C calling + convention (named 'cdecl' on Windows). + + The exported functions can be accessed as attributes, or by + indexing with the function name. Examples: + + .qsort -> callable object + ['qsort'] -> callable object + + Calling the functions releases the Python GIL during the call and + reaquires it afterwards. + """ + class _FuncPtr(_CFuncPtr): + #_flags_ = _FUNCFLAG_CDECL + restype = c_int # default, can be overridden in instances + argtypes = [] + + def __init__(self, name, mode=DEFAULT_MODE, handle=None): + self._name = name + if handle is None: + self._handle = _ffi.CDLL(self._name) + else: + self._handle = handle + + def __repr__(self): + return "<%s '%s', handle>" % \ + (self.__class__.__name__, self._name) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + raise AttributeError, name + func = self.__getitem__(name) + setattr(self, name, func) + return func + + def __getitem__(self, name_or_ordinal): + func = self._FuncPtr((name_or_ordinal, self)) + if not isinstance(name_or_ordinal, (int, long)): + func.__name__ = name_or_ordinal + return func From fijal at codespeak.net Thu Nov 15 01:19:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 01:19:09 +0100 (CET) Subject: [pypy-svn] r48697 - pypy/dist/pypy/rpython/module Message-ID: <20071115001909.D95E98289@code0.codespeak.net> Author: fijal Date: Thu Nov 15 01:19:08 2007 New Revision: 48697 Modified: pypy/dist/pypy/rpython/module/ll_strtod.py Log: Make ootypesystem happy Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Thu Nov 15 01:19:08 2007 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import cache_c_module from pypy.tool.autopath import pypydir +from pypy.rpython.ootypesystem import ootype class CConfig: _includes_ = ['src/ll_strtod.h'] @@ -34,8 +35,11 @@ res = ll_strtod(fmt, x) return rffi.charp2str(res) + def oofakeimpl(fmt, x): + return ootype.oostring(rarithmetic.formatd(fmt._str, x), -1) + return extdef([str, float], str, 'll_strtod.ll_strtod_formatd', - llimpl=llimpl) + llimpl=llimpl, oofakeimpl=oofakeimpl) @registering(rarithmetic.parts_to_float) def register_parts_to_float(self): @@ -50,5 +54,10 @@ raise ValueError("Wrong literal for float") return res + def oofakeimpl(sign, beforept, afterpt, exponent): + return rarithmetic.parts_to_float(sign._str, beforept._str, + afterpt._str, exponent._str) + return extdef([str, str, str, str], float, - 'll_strtod.ll_strtod_parts_to_float', llimpl=llimpl) + 'll_strtod.ll_strtod_parts_to_float', llimpl=llimpl, + oofakeimpl=oofakeimpl) From fijal at codespeak.net Thu Nov 15 01:58:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 01:58:10 +0100 (CET) Subject: [pypy-svn] r48699 - pypy/dist/pypy/module/_ffi/test Message-ID: <20071115005810.7B51282A6@code0.codespeak.net> Author: fijal Date: Thu Nov 15 01:58:10 2007 New Revision: 48699 Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Update the name Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Thu Nov 15 01:58:10 2007 @@ -9,7 +9,7 @@ if sys.platform != 'linux2': py.test.skip("Linux only tests by now") -class AppTestCTypes: +class AppTestFfi: def prepare_c_example(): from pypy.tool.udir import udir c_file = udir.join("xlib.c") From fijal at codespeak.net Thu Nov 15 01:59:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 01:59:38 +0100 (CET) Subject: [pypy-svn] r48700 - in pypy/dist/pypy/lib: app_test ctypes Message-ID: <20071115005938.04B7E82DD@code0.codespeak.net> Author: fijal Date: Thu Nov 15 01:59:38 2007 New Revision: 48700 Added: pypy/dist/pypy/lib/ctypes/util.py (contents, props changed) Modified: pypy/dist/pypy/lib/app_test/test_ctypes.py pypy/dist/pypy/lib/ctypes/__init__.py Log: A bit of cowboy-coding all over the place. The question is how far we can go into implementing ctypes without being insane Modified: pypy/dist/pypy/lib/app_test/test_ctypes.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_ctypes.py (original) +++ pypy/dist/pypy/lib/app_test/test_ctypes.py Thu Nov 15 01:59:38 2007 @@ -1,24 +1,71 @@ from pypy.conftest import gettestobjspace import sys, py +from pypy.module._ffi.test.test__ffi import AppTestFfi as BaseTestFfi def setup_module(mod): if sys.platform != 'linux2': py.test.skip("Linux only tests by now") -class AppTestCtypes: - def setup_class(cls): - space = gettestobjspace(usemodules=('_ffi','struct')) +class AppTestCtypes(BaseTestFfi): + def test_libload(self): + import ctypes + ctypes.CDLL('libc.so.6') - def test_rand(self): + def test_getattr(self): import ctypes libc = ctypes.CDLL('libc.so.6') rand = libc.rand - first = rand() - counter = 0 + rand.restype = ctypes.c_int + assert libc.rand is rand + assert libc['rand'] is not rand + assert isinstance(rand, ctypes._CFuncPtr) + raises(AttributeError, 'libc.xxxxxxxxxxxxxxxx') + + def test_short_addition(self): + import ctypes + lib = ctypes.CDLL(self.lib_name) + short_add = lib.add_shorts + short_add.argtypes = [ctypes.c_short, ctypes.c_short] + short_add.restype = ctypes.c_ushort + assert short_add(1, 2) == 3 + + def test_rand(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + func = libc.rand + first = func() + count = 0 for i in range(100): - next = rand() - if next == first: - counter += 1 - assert counter < 100 + res = func() + if res == first: + count += 1 + assert count != 100 + + def test_pow(self): + import ctypes + libm = ctypes.CDLL('libm.so') + pow = libm.pow + pow.argtypes = [ctypes.c_double, ctypes.c_double] + pow.restype = ctypes.c_double + assert pow(2.0, 2.0) == 4.0 + assert pow(3.0, 3.0) == 27.0 + assert pow(2, 2) == 4.0 + raises(ctypes.ArgumentError, "pow('x', 2.0)") + + def not_implemented(self): + skip("not implemented") + test_getchar = not_implemented + test_returning_str = not_implemented + test_strlen = not_implemented + test_time = not_implemented + test_gettimeofday = not_implemented + test_structreturn = not_implemented + test_nested_structures = not_implemented + test_array = not_implemented + test_array_of_structure = not_implemented + test_bad_parameters = not_implemented + test_implicit_structure = not_implemented + test_longs_ulongs = not_implemented + test_callback = not_implemented Modified: pypy/dist/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/dist/pypy/lib/ctypes/__init__.py (original) +++ pypy/dist/pypy/lib/ctypes/__init__.py Thu Nov 15 01:59:38 2007 @@ -4,16 +4,23 @@ DEFAULT_MODE = None # XXX, mode support in _ffi +class ArgumentError(Exception): + pass + class _CFuncPtr(object): def __init__(self, (name, lib)): self.name = name self.lib = lib - + self._update_handle() + def __call__(self, *args): - if not hasattr(self, '_handle'): - self._update_handle() - # XXX eventually cast types here - return self._handle(*args) + # XXX right now always update handle in order to keep changing + # argtypes and restype + self._update_handle() + try: + return self._handle(*args) + except TypeError, e: + raise ArgumentError("Wrong argument", e.__class__) def _update_handle(self): llargs = [i._lltype for i in self.argtypes] @@ -21,9 +28,64 @@ self._handle = self.lib._handle.ptr(self.name, llargs, self.restype._lltype) -class c_int(object): +class _SimpleCData(object): + def __init__(self, value): + self.value = value + +class c_ushort(_SimpleCData): + _lltype = 'H' + +class c_double(_SimpleCData): + _lltype = 'd' + +class c_ubyte(_SimpleCData): + _lltype = 'B' + +class c_float(_SimpleCData): + _lltype = 'f' + +class c_ulong(_SimpleCData): + _lltype = 'L' + +class c_short(_SimpleCData): + _lltype = 'h' + +class c_ubyte(_SimpleCData): + _lltype = 'b' + +class c_byte(_SimpleCData): + _lltype = 'B' + +class c_char(_SimpleCData): + _lltype = 'c' + +class c_long(_SimpleCData): + _lltype = 'l' + +class c_ulonglong(_SimpleCData): + _lltype = 'Q' + +class c_longlong(_SimpleCData): + _lltype = 'q' + +class c_int(_SimpleCData): _lltype = 'i' +class c_uint(_SimpleCData): + _lltype = 'I' + +class c_double(_SimpleCData): + _lltype = 'd' + +class c_float(_SimpleCData): + _lltype = 'f' + +c_size_t = c_ulong # XXX + +class POINTER(object): + def __init__(self, cls): + self.cls = cls + class CDLL(object): """An instance of this class represents a loaded dll/shared library, exporting functions using the standard C calling @@ -66,3 +128,24 @@ if not isinstance(name_or_ordinal, (int, long)): func.__name__ = name_or_ordinal return func + + +class LibraryLoader(object): + def __init__(self, dlltype): + self._dlltype = dlltype + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError(name) + dll = self._dlltype(name) + setattr(self, name, dll) + return dll + + def __getitem__(self, name): + return getattr(self, name) + + def LoadLibrary(self, name): + return self._dlltype(name) + +cdll = LibraryLoader(CDLL) + Added: pypy/dist/pypy/lib/ctypes/util.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/ctypes/util.py Thu Nov 15 01:59:38 2007 @@ -0,0 +1,154 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +import sys, os + +# find_library(name) returns the pathname of a library, or None. +if os.name == "nt": + def find_library(name): + # See MSDN for the REAL search order. + for directory in os.environ['PATH'].split(os.pathsep): + fname = os.path.join(directory, name) + if os.path.exists(fname): + return fname + if fname.lower().endswith(".dll"): + continue + fname = fname + ".dll" + if os.path.exists(fname): + return fname + return None + +if os.name == "ce": + # search path according to MSDN: + # - absolute path specified by filename + # - The .exe launch directory + # - the Windows directory + # - ROM dll files (where are they?) + # - OEM specified search path: HKLM\Loader\SystemPath + def find_library(name): + return name + +if os.name == "posix" and sys.platform == "darwin": + from ctypes.macholib.dyld import dyld_find as _dyld_find + def find_library(name): + possible = ['lib%s.dylib' % name, + '%s.dylib' % name, + '%s.framework/%s' % (name, name)] + for name in possible: + try: + return _dyld_find(name) + except ValueError: + continue + return None + +elif os.name == "posix": + # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump + import re, tempfile, errno + + def _findLib_gcc(name): + expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) + fdout, ccout = tempfile.mkstemp() + os.close(fdout) + cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; else CC=cc; fi;' \ + '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + try: + f = os.popen(cmd) + trace = f.read() + f.close() + finally: + try: + os.unlink(ccout) + except OSError, e: + if e.errno != errno.ENOENT: + raise + res = re.search(expr, trace) + if not res: + return None + return res.group(0) + + def _get_soname(f): + # assuming GNU binutils / ELF + if not f: + return None + cmd = "objdump -p -j .dynamic 2>/dev/null " + f + res = re.search(r'\sSONAME\s+([^\s]+)', os.popen(cmd).read()) + if not res: + return None + return res.group(1) + + if (sys.platform.startswith("freebsd") + or sys.platform.startswith("openbsd") + or sys.platform.startswith("dragonfly")): + + def _num_version(libname): + # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ] + parts = libname.split(".") + nums = [] + try: + while parts: + nums.insert(0, int(parts.pop())) + except ValueError: + pass + return nums or [ sys.maxint ] + + def find_library(name): + ename = re.escape(name) + expr = r':-l%s\.\S+ => \S*/(lib%s\.\S+)' % (ename, ename) + res = re.findall(expr, + os.popen('/sbin/ldconfig -r 2>/dev/null').read()) + if not res: + return _get_soname(_findLib_gcc(name)) + res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y))) + return res[-1] + + else: + + def _findLib_ldconfig(name): + # XXX assuming GLIBC's ldconfig (with option -p) + expr = r'/[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) + res = re.search(expr, + os.popen('/sbin/ldconfig -p 2>/dev/null').read()) + if not res: + # Hm, this works only for libs needed by the python executable. + cmd = 'ldd %s 2>/dev/null' % sys.executable + res = re.search(expr, os.popen(cmd).read()) + if not res: + return None + return res.group(0) + + def find_library(name): + return _get_soname(_findLib_ldconfig(name) or _findLib_gcc(name)) + +################################################################ +# test code + +def test(): + from ctypes import cdll + if os.name == "nt": + print cdll.msvcrt + print cdll.load("msvcrt") + print find_library("msvcrt") + + if os.name == "posix": + # find and load_version + print find_library("m") + print find_library("c") + print find_library("bz2") + + # getattr +## print cdll.m +## print cdll.bz2 + + # load + if sys.platform == "darwin": + print cdll.LoadLibrary("libm.dylib") + print cdll.LoadLibrary("libcrypto.dylib") + print cdll.LoadLibrary("libSystem.dylib") + print cdll.LoadLibrary("System.framework/System") + else: + print cdll.LoadLibrary("libm.so") + print cdll.LoadLibrary("libcrypt.so") + print find_library("crypt") + +if __name__ == "__main__": + test() From fijal at codespeak.net Thu Nov 15 02:01:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 02:01:57 +0100 (CET) Subject: [pypy-svn] r48701 - pypy/dist/pypy/translator/llvm Message-ID: <20071115010157.40E0F8263@code0.codespeak.net> Author: fijal Date: Thu Nov 15 02:01:56 2007 New Revision: 48701 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: Kill this import Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Thu Nov 15 02:01:56 2007 @@ -10,7 +10,6 @@ from pypy.tool.udir import udir from pypy.rpython.module import ll_stack -from pypy.rpython.lltypesystem.module import ll_strtod ll_stack_too_big = """ From fijal at codespeak.net Thu Nov 15 03:00:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 03:00:21 +0100 (CET) Subject: [pypy-svn] r48702 - in pypy/dist/pypy/lib: app_test ctypes Message-ID: <20071115020021.DF25C81D6@code0.codespeak.net> Author: fijal Date: Thu Nov 15 03:00:20 2007 New Revision: 48702 Modified: pypy/dist/pypy/lib/app_test/test_ctypes.py pypy/dist/pypy/lib/ctypes/__init__.py Log: Cool, bunch of passing tests Modified: pypy/dist/pypy/lib/app_test/test_ctypes.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_ctypes.py (original) +++ pypy/dist/pypy/lib/app_test/test_ctypes.py Thu Nov 15 03:00:20 2007 @@ -53,13 +53,51 @@ assert pow(2, 2) == 4.0 raises(ctypes.ArgumentError, "pow('x', 2.0)") + def test_getchar(self): + import ctypes + lib = ctypes.CDLL(self.lib_name) + get_char = lib.get_char + get_char.argtypes = [ctypes.c_char_p, ctypes.c_ushort] + get_char.restype = ctypes.c_char + assert get_char('dupa', 2) == 'p' + assert get_char('dupa', 1) == 'u' + raises(ValueError, "get_char('xxx', 2 ** 17)") + raises(ValueError, "get_char('xxx', -1)") + + def test_returning_str(self): + import ctypes + lib = ctypes.CDLL(self.lib_name) + char_check = lib.char_check + char_check.argtypes = [ctypes.c_char, ctypes.c_char] + char_check.restype = ctypes.c_char_p + assert char_check('y', 'x') == 'xxxxxx' + assert char_check('x', 'y') is None + + def test_strlen(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + strlen = libc.strlen + strlen.argtypes = [ctypes.c_char_p] + strlen.restype = ctypes.c_int + assert strlen("dupa") == 4 + assert strlen("zupa") == 4 + assert strlen("ddd\x00") == 3 + strdup = libc.strdup + strdup.argtypes = [ctypes.c_char_p] + strdup.restype = ctypes.c_char_p + assert strdup("xxx") == "xxx" + def not_implemented(self): skip("not implemented") - test_getchar = not_implemented - test_returning_str = not_implemented - test_strlen = not_implemented - test_time = not_implemented + def test_time(self): + import ctypes + libc = ctypes.CDLL('libc.so.6') + time = libc.time + time.argtypes = [ctypes.c_void_p] + time.restype = ctypes.c_long + assert time(None) != 0 + test_gettimeofday = not_implemented test_structreturn = not_implemented test_nested_structures = not_implemented Modified: pypy/dist/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/dist/pypy/lib/ctypes/__init__.py (original) +++ pypy/dist/pypy/lib/ctypes/__init__.py Thu Nov 15 03:00:20 2007 @@ -23,62 +23,62 @@ raise ArgumentError("Wrong argument", e.__class__) def _update_handle(self): - llargs = [i._lltype for i in self.argtypes] + llargs = [i._type_ for i in self.argtypes] # XXX first approximation self._handle = self.lib._handle.ptr(self.name, llargs, - self.restype._lltype) + self.restype._type_) class _SimpleCData(object): def __init__(self, value): self.value = value class c_ushort(_SimpleCData): - _lltype = 'H' + _type_ = 'H' class c_double(_SimpleCData): - _lltype = 'd' + _type_ = 'd' class c_ubyte(_SimpleCData): - _lltype = 'B' + _type_ = 'B' class c_float(_SimpleCData): - _lltype = 'f' + _type_ = 'f' class c_ulong(_SimpleCData): - _lltype = 'L' + _type_ = 'L' class c_short(_SimpleCData): - _lltype = 'h' + _type_ = 'h' class c_ubyte(_SimpleCData): - _lltype = 'b' + _type_ = 'b' class c_byte(_SimpleCData): - _lltype = 'B' + _type_ = 'B' class c_char(_SimpleCData): - _lltype = 'c' + _type_ = 'c' class c_long(_SimpleCData): - _lltype = 'l' + _type_ = 'l' class c_ulonglong(_SimpleCData): - _lltype = 'Q' + _type_ = 'Q' class c_longlong(_SimpleCData): - _lltype = 'q' + _type_ = 'q' class c_int(_SimpleCData): - _lltype = 'i' + _type_ = 'i' class c_uint(_SimpleCData): - _lltype = 'I' + _type_ = 'I' class c_double(_SimpleCData): - _lltype = 'd' + _type_ = 'd' class c_float(_SimpleCData): - _lltype = 'f' + _type_ = 'f' c_size_t = c_ulong # XXX @@ -86,6 +86,12 @@ def __init__(self, cls): self.cls = cls +class c_void_p(_SimpleCData): + _type_ = 'P' + +class c_char_p(_SimpleCData): + _type_ = 's' + class CDLL(object): """An instance of this class represents a loaded dll/shared library, exporting functions using the standard C calling From cfbolz at codespeak.net Thu Nov 15 13:22:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 13:22:16 +0100 (CET) Subject: [pypy-svn] r48706 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115122216.074E381D5@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 13:22:15 2007 New Revision: 48706 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: major refactoring of ropes to have a chance to work with unicode. one test failing. Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 13:22:15 2007 @@ -9,8 +9,8 @@ # XXX should optimize the numbers NEW_NODE_WHEN_LENGTH = 16 +CONVERT_WHEN_SMALLER = 8 MAX_DEPTH = 32 # maybe should be smaller -MIN_SLICE_LENGTH = 64 CONCATENATE_WHEN_MULTIPLYING = 128 HIGHEST_BIT_SET = intmask(1L << (NBITS - 1)) @@ -54,58 +54,84 @@ class StringNode(object): hash_cache = 0 def length(self): - return 0 + raise NotImplementedError("base class") + + def is_ascii(self): + raise NotImplementedError("base class") + + def is_bytestring(self): + raise NotImplementedError("base class") def depth(self): return 0 - def rebalance(self): - return self - def hash_part(self): raise NotImplementedError("base class") - def flatten(self): - return '' + def check_balanced(self): + return True - def __add__(self, other): - return concatenate(self, other) - - def __getitem__(self, index): - if isinstance(index, slice): - start, stop, step = index.indices(self.length()) - # XXX sucks - slicelength = len(xrange(start, stop, step)) - return getslice(self, start, stop, step, slicelength) - return self.getitem(index) + def getchar(self, index): + raise NotImplementedError("abstract base class") - def getitem(self, index): + def getunichar(self, index): raise NotImplementedError("abstract base class") - def getitem_slice(self, start, stop): - # XXX really horrible, in most cases - result = [] - for i in range(start, stop): - result.append(self.getitem(i)) - return rope_from_charlist(result) + def getint(self, index): + raise NotImplementedError("abstract base class") + + def getslice(self, start, stop): + raise NotImplementedError("abstract base class") def view(self): view([self]) - def check_balanced(self): - return True + def rebalance(self): + return self + + def flatten_string(self): + raise NotImplementedError("abstract base class") + + def flatten_unicode(self): + raise NotImplementedError("abstract base class") + + def __add__(self, other): + return concatenate(self, other) -class LiteralStringNode(StringNode): +class LiteralNode(StringNode): + def find_int(self, what, start, stop): + raise NotImplementedError("abstract base class") + + def literal_concat(self, other): + raise NotImplementedError("abstract base class") + + +class LiteralStringNode(LiteralNode): def __init__(self, s): self.s = s + is_ascii = True + for c in s: + if ord(c) >= 128: + is_ascii = False + self._is_ascii = is_ascii def length(self): return len(self.s) - def flatten(self): + def is_ascii(self): + return self._is_ascii + + def is_bytestring(self): + return True + + def flatten_string(self): return self.s + def flatten_unicode(self): + # XXX not RPython yet + return self.s.decode('latin-1') + def hash_part(self): h = self.hash_cache if not h: @@ -117,13 +143,38 @@ h = self.hash_cache = x return h - def getitem(self, index): + def getchar(self, index): return self.s[index] - def getitem_slice(self, start, stop): + def getunichar(self, index): + return unicode(self.s[index]) + + def getint(self, index): + return ord(self.s[index]) + + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralStringNode(self.s[start:stop]) + + def find_int(self, what, start, stop): + if what >= 256: + return -1 + result = self.s.find(chr(what), start, stop) + if result == -1: + return -1 + return result + + def literal_concat(self, other): + if (isinstance(other, LiteralStringNode) and + len(other.s) + len(self.s) < NEW_NODE_WHEN_LENGTH): + return LiteralStringNode(self.s + other.s) + elif (isinstance(other, LiteralUnicodeNode) and + len(other.u) + len(self.s) < NEW_NODE_WHEN_LENGTH and + len(self.s) < CONVERT_WHEN_SMALLER): + return LiteralUnicodeNode(self.s.decode("latin-1") + other.u) + return BinaryConcatNode(self, other) + def dot(self, seen, toplevel=False): if self in seen: return @@ -139,6 +190,70 @@ del i +class LiteralUnicodeNode(StringNode): + def __init__(self, u): + self.u = u + + def length(self): + return len(self.u) + + def flatten_unicode(self): + return self.u + + def is_ascii(self): + return False # usually not + + def is_bytestring(self): + return False + + def hash_part(self): + h = self.hash_cache + if not h: + x = 0 + for c in self.u: + x = (1000003*x) + ord(c) + x = intmask(x) + x |= HIGHEST_BIT_SET + h = self.hash_cache = x + return h + + def getunichar(self, index): + return self.u[index] + + def getbyte(self, index): + return ord(self.u[index]) + + def getslice(self, start, stop): + assert 0 <= start <= stop + return LiteralUnicodeNode(self.u[start:stop]) + + def find_int(self, what, start, stop): + result = node.u.find(unichr(what), start, stop) + if result == -1: + return -1 + return result + + def literal_concat(self, other): + if (isinstance(other, LiteralUnicodeNode) and + len(other.u) + len(self.u) < NEW_NODE_WHEN_LENGTH): + return LiteralStringNode(self.u + other.u) + elif (isinstance(other, LiteralStringNode) and + len(other.s) + len(self.u) < NEW_NODE_WHEN_LENGTH and + len(other.s) < CONVERT_WHEN_SMALLER): + return LiteralUnicodeNode(self.u + other.s.decode("latin-1")) + return BinaryConcatNode(self, other) + + def dot(self, seen, toplevel=False): + if self in seen: + return + seen[self] = True + addinfo = str(self.s).replace('"', "'") or "_" + if len(addinfo) > 10: + addinfo = addinfo[:3] + "..." + addinfo[-3:] + yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( + id(self), len(self.s), + repr(addinfo).replace('"', '').replace("\\", "\\\\"))) + class BinaryConcatNode(StringNode): def __init__(self, left, right): self.left = left @@ -149,6 +264,14 @@ raise self._depth = max(left.depth(), right.depth()) + 1 self.balanced = False + self._is_ascii = left.is_ascii() and right.is_ascii() + self._is_bytestring = left.is_bytestring() and right.is_bytestring() + + def is_ascii(self): + return self._is_ascii + + def is_bytestring(self): + return self._is_bytestring def check_balanced(self): if self.balanced: @@ -172,16 +295,34 @@ def depth(self): return self._depth - def getitem(self, index): + def getchar(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getchar(index - llen) + else: + return self.left.getchar(index) + + def getunichar(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getunichar(index - llen) + else: + return self.left.getunichar(index) + + def getint(self, index): llen = self.left.length() if index >= llen: - return self.right.getitem(index - llen) + return self.right.getint(index - llen) else: - return self.left.getitem(index) + return self.left.getint(index) + + def flatten_string(self): + f = fringe(self) + return "".join([node.flatten_string() for node in f]) - def flatten(self): + def flatten_unicode(self): f = fringe(self) - return "".join([node.flatten() for node in f]) + return "".join([node.flatten_string() for node in f]) def hash_part(self): h = self.hash_cache @@ -213,119 +354,37 @@ for line in child.dot(seen): yield line -class SliceNode(StringNode): - def __init__(self, start, stop, node): - assert 0 <= start <= stop - self.start = start - self.stop = stop - self.node = node - - def length(self): - return self.stop - self.start - - def getitem_slice(self, start, stop): - return self.node.getitem_slice(self.start + start, self.start + stop) - - def getitem(self, index): - return self.node.getitem(self.start + index) - - def flatten(self): - return self.node.flatten()[self.start: self.stop] - - def hash_part(self): - h = self.hash_cache - if not h: - x = 0 - for i in range(self.start, self.stop): - x = (1000003*x) + ord(self.node.getitem(i)) - x = intmask(x) - x |= HIGHEST_BIT_SET - h = self.hash_cache = x - return h - - def dot(self, seen, toplevel=False): - if self in seen: - return - seen[self] = True - yield '"%s" [shape=octagon,label="slice\\nstart=%s, stop=%s"];' % ( - id(self), self.start, self.stop) - yield '"%s" -> "%s";' % (id(self), id(self.node)) - for line in self.node.dot(seen): - yield line - -class EfficientGetitemWraper(StringNode): - def __init__(self, node): - assert isinstance(node, BinaryConcatNode) - self.node = node - self.iter = SeekableCharIterator(node) - self.nextpos = 0 - self.accesses = 0 - self.seeks = 0 - - def length(self): - return self.node.length() - - def depth(self): - return self.node.depth() - - def rebalance(self): - return EfficientGetitemWraper(self.node.rebalance()) - - def hash_part(self): - return self.node.hash_part() - - def flatten(self): - return self.node.flatten() - - def getitem(self, index): - self.accesses += 1 - nextpos = self.nextpos - self.nextpos = index + 1 - if index < nextpos: - self.iter.seekback(nextpos - index) - self.seeks += nextpos - index - elif index > nextpos: - self.iter.seekforward(index - nextpos) - self.seeks += index - nextpos - return self.iter.next() - - def view(self): - return self.node.view() - - def check_balanced(self): - return self.node.check_balanced() - - def concatenate(node1, node2): if node1.length() == 0: return node2 if node2.length() == 0: return node1 - if (isinstance(node2, LiteralStringNode) and - len(node2.s) <= NEW_NODE_WHEN_LENGTH): - if isinstance(node1, LiteralStringNode): - if len(node1.s) + len(node2.s) <= NEW_NODE_WHEN_LENGTH: - return LiteralStringNode(node1.s + node2.s) + if isinstance(node2, LiteralNode): + if isinstance(node1, LiteralNode): + return node1.literal_concat(node2) elif isinstance(node1, BinaryConcatNode): r = node1.right - if isinstance(r, LiteralStringNode): - if len(r.s) + len(node2.s) <= NEW_NODE_WHEN_LENGTH: - return BinaryConcatNode(node1.left, - LiteralStringNode(r.s + node2.s)) + if isinstance(r, LiteralNode): + return BinaryConcatNode(node1.left, + r.literal_concat(node2)) result = BinaryConcatNode(node1, node2) if result.depth() > MAX_DEPTH: #XXX better check return result.rebalance() return result -def getslice(node, start, stop, step, slicelength): +def getslice(node, start, stop, step, slicelength=-1): + if slicelength == -1: + # XXX for testing only + slicelength = len(xrange(start, stop, step)) if step != 1: start, stop, node = find_straddling(node, start, stop) - iter = SeekableCharIterator(node) + iter = SeekableItemIterator(node) iter.seekforward(start) - result = [iter.next()] + #XXX doesn't work for unicode + result = [iter.nextchar()] for i in range(slicelength - 1): iter.seekforward(step - 1) - result.append(iter.next()) + result.append(iter.nextchar()) return rope_from_charlist(result) return getslice_one(node, start, stop) @@ -342,7 +401,7 @@ getslice_right(node.left, start), getslice_left(node.right, stop - node.left.length())) else: - return getslice_primitive(node, start, stop) + return node.getslice(start, stop) def find_straddling(node, start, stop): while 1: @@ -371,7 +430,7 @@ else: return concatenate(getslice_right(node.left, start), node.right) - return getslice_primitive(node, start, node.length()) + return node.getslice(start, node.length()) def getslice_left(node, stop): while 1: @@ -385,16 +444,9 @@ else: return concatenate(node.left, getslice_left(node.right, stop - llen)) - return getslice_primitive(node, 0, stop) + return node.getslice(0, stop) -def getslice_primitive(node, start, stop): - if stop - start >= MIN_SLICE_LENGTH: - if isinstance(node, SliceNode): - return SliceNode(start + node.start, stop + node.start, - node.node) - return SliceNode(start, stop, node) - return node.getitem_slice(start, stop) def multiply(node, times): if times <= 0: @@ -510,10 +562,37 @@ size += len(chars) return rebalance(nodelist, size) +def rope_from_unicharlist(charlist): + nodelist = [] + length = len(charlist) + if length: + return LiteralStringNode.EMPTY + i = 0 + while i < length: + chunk = [] + while i < length: + c = ord(charlist[i]) + if c < 256: + break + chunk.append(unichr(c)) + i += 1 + if chunk: + nodelist.append(LiteralUnicodeNode("".join(chunk))) + chunck = [] + while i < length: + c = ord(charlist[i]) + if c >= 256: + break + chunk.append(chr(c)) + i += 1 + if chunk: + nodelist.append(LiteralStringNode("".join(chunk))) + return rebalance(nodelist, length) + # __________________________________________________________________________ # searching -def find_char(node, c, start=0, stop=-1): +def find_int(node, what, start=0, stop=-1): offset = 0 length = node.length() if stop == -1: @@ -524,14 +603,11 @@ start = newstart stop = newstop assert 0 <= start <= stop - if isinstance(node, LiteralStringNode): - result = node.s.find(c, start, stop) - if result == -1: - return -1 - return result + offset - elif isinstance(node, SliceNode): - return find_char(node.node, c, node.start + start, - node.start + stop) - node.start + offset + if isinstance(node, LiteralNode): + pos = node.find_int(what, start, stop) + if pos == -1: + return pos + return pos + offset iter = FringeIterator(node) #import pdb; pdb.set_trace() i = 0 @@ -546,20 +622,10 @@ continue searchstart = max(0, start - i) searchstop = min(stop - i, nodelength) - if isinstance(fringenode, LiteralStringNode): - st = fringenode.s - localoffset = 0 - else: - assert isinstance(fringenode, SliceNode) - n = fringenode.node - assert isinstance(n, LiteralStringNode) - st = n.s - localoffset = -fringenode.start - searchstart += fringenode.start - searchstop += fringenode.stop - pos = st.find(c, searchstart, searchstop) + assert isinstance(fringenode, LiteralNode) + pos = fringenode.find_int(what, searchstart, searchstop) if pos != -1: - return pos + i + offset + localoffset + return pos + i + offset i += nodelength return -1 @@ -570,7 +636,7 @@ if stop > len1 or stop == -1: stop = len1 if len2 == 1: - return find_char(node, subnode.getitem(0), start, stop) + return find_int(node, subnode.getint(0), start, stop) if len2 == 0: if (stop - start) < 0: return -1 @@ -579,25 +645,27 @@ return _find_node(node, subnode, start, stop, restart) def _find(node, substring, start, stop, restart): + # XXX + assert node.is_bytestring() len2 = len(substring) i = 0 m = start - iter = SeekableCharIterator(node) + iter = SeekableItemIterator(node) iter.seekforward(start) - c = iter.next() + c = iter.nextchar() while m + i < stop: if c == substring[i]: i += 1 if i == len2: return m if m + i < stop: - c = iter.next() + c = iter.nextchar() else: # mismatch, go back to the last possible starting pos if i==0: m += 1 if m + i < stop: - c = iter.next() + c = iter.nextchar() else: e = restart[i-1] new_m = m + i - e @@ -605,7 +673,7 @@ seek = m + i - new_m if seek: iter.seekback(m + i - new_m) - c = iter.next() + c = iter.nextchar() m = new_m i = e return -1 @@ -613,26 +681,26 @@ def _find_node(node, subnode, start, stop, restart): len2 = subnode.length() m = start - iter = SeekableCharIterator(node) + iter = SeekableItemIterator(node) iter.seekforward(start) - c = iter.next() + c = iter.nextint() i = 0 - subiter = SeekableCharIterator(subnode) - d = subiter.next() + subiter = SeekableItemIterator(subnode) + d = subiter.nextint() while m + i < stop: if c == d: i += 1 if i == len2: return m - d = subiter.next() + d = subiter.nextint() if m + i < stop: - c = iter.next() + c = iter.nextint() else: # mismatch, go back to the last possible starting pos if i == 0: m += 1 if m + i < stop: - c = iter.next() + c = iter.nextint() else: e = restart[i - 1] new_m = m + i - e @@ -640,20 +708,20 @@ seek = m + i - new_m if seek: iter.seekback(m + i - new_m) - c = iter.next() + c = iter.nextint() m = new_m subiter.seekback(i - e + 1) - d = subiter.next() + d = subiter.nextint() i = e return -1 def construct_restart_positions(s): - l = len(s) - restart = [0] * l + length = len(s) + restart = [0] * length restart[0] = 0 i = 1 j = 0 - while i < l: + while i < length: if s[i] == s[j]: j += 1 restart[i] = j @@ -668,43 +736,43 @@ def construct_restart_positions_node(node): # really a bit overkill - l = node.length() - restart = [0] * l + length = node.length() + restart = [0] * length restart[0] = 0 i = 1 j = 0 - iter1 = CharIterator(node) - iter1.next() - c1 = iter1.next() - iter2 = SeekableCharIterator(node) - c2 = iter2.next() - while i < l: + iter1 = ItemIterator(node) + iter1.nextint() + c1 = iter1.nextint() + iter2 = SeekableItemIterator(node) + c2 = iter2.nextint() + while 1: if c1 == c2: j += 1 - if j != l: - c2 = iter2.next() + if j < length: + c2 = iter2.nextint() restart[i] = j i += 1 - if i != l: - c1 = iter1.next() + if i < length: + c1 = iter1.nextint() else: break elif j>0: new_j = restart[j-1] assert new_j < j iter2.seekback(j - new_j) - c2 = iter2.next() + c2 = iter2.nextint() j = new_j else: restart[i] = 0 i += 1 - if i != l: - c1 = iter1.next() + if i < length: + c1 = iter1.nextint() else: break j = 0 - iter2 = SeekableCharIterator(node) - c2 = iter2.next() + iter2 = SeekableItemIterator(node) + c2 = iter2.nextint() return restart def view(objs): @@ -785,37 +853,60 @@ return result -class CharIterator(object): +class ItemIterator(object): def __init__(self, node): self.iter = FringeIterator(node) self.node = None self.nodelength = 0 self.index = 0 - def next(self): + + def getnode(self): node = self.node if node is None: while 1: node = self.node = self.iter.next() nodelength = self.nodelength = node.length() if nodelength != 0: - break - self.index = 0 + self.index = 0 + return node + return node + + def advance_index(self): index = self.index - result = node.getitem(index) if index == self.nodelength - 1: self.node = None else: self.index = index + 1 + + def nextchar(self): + node = self.getnode() + index = self.index + result = node.getchar(self.index) + self.advance_index() + return result + + def nextunichar(self): + node = self.getnode() + index = self.index + result = node.getunichar(self.index) + self.advance_index() return result -class ReverseCharIterator(object): + def nextint(self): + node = self.getnode() + index = self.index + result = node.getint(self.index) + self.advance_index() + return result + +class ReverseItemIterator(object): def __init__(self, node): self.iter = ReverseFringeIterator(node) self.node = None self.index = 0 - def next(self): + def getnode(self): node = self.node index = self.index if node is None: @@ -823,16 +914,36 @@ node = self.node = self.iter.next() index = self.index = node.length() - 1 if index != -1: - break - result = node.getitem(index) - if index == 0: + return node + return node + + + def advance_index(self): + if self.index == 0: self.node = None else: - self.index = index - 1 + self.index -= 1 + + def nextchar(self): + node = self.getnode() + result = node.getchar(self.index) + self.advance_index() + return result + + def nextint(self): + node = self.getnode() + result = node.getint(self.index) + self.advance_index() + return result + + def nextunichar(self): + node = self.getnode() + result = node.getunichar(self.index) + self.advance_index() return result -class SeekableCharIterator(object): +class SeekableItemIterator(object): def __init__(self, node): self.iter = SeekableFringeIterator(node) self.node = self.nextnode() @@ -848,15 +959,34 @@ self.index = 0 return node - def next(self): + + def advance_index(self): + if self.index == self.nodelength - 1: + self.node = None + self.index += 1 + + def nextchar(self): node = self.node if node is None: node = self.nextnode() - index = self.index - result = self.node.getitem(index) - if self.index == self.nodelength - 1: - self.node = None - self.index = index + 1 + result = self.node.getchar(self.index) + self.advance_index() + return result + + def nextunichar(self): + node = self.node + if node is None: + node = self.nextnode() + result = self.node.getunichar(self.index) + self.advance_index() + return result + + def nextint(self): + node = self.node + if node is None: + node = self.nextnode() + result = self.node.getint(self.index) + self.advance_index() return result def seekforward(self, numchars): @@ -898,16 +1028,18 @@ class FindIterator(object): def __init__(self, node, sub, start=0, stop=-1): self.node = node + self.sub = sub len1 = self.length = node.length() - substring = self.substring = sub.flatten() # XXX for now - len2 = len(substring) + len2 = sub.length() self.search_length = len2 if len2 == 0: self.restart_positions = None elif len2 == 1: self.restart_positions = None else: - self.restart_positions = construct_restart_positions(substring) + self.restart_positions = construct_restart_positions_node(sub) + # XXX + assert self.restart_positions == construct_restart_positions(sub.flatten_string()) self.start = start if stop == -1 or stop > len1: stop = len1 @@ -921,8 +1053,8 @@ self.start += 1 return start elif self.search_length == 1: - result = find_char(self.node, self.substring[0], - self.start, self.stop) + result = find_int(self.node, self.sub.getint(0), + self.start, self.stop) if result == -1: self.start = self.length raise StopIteration @@ -930,8 +1062,8 @@ return result if self.start >= self.stop: raise StopIteration - result = _find(self.node, self.substring, self.start, - self.stop, self.restart_positions) + result = _find_node(self.node, self.sub, self.start, + self.stop, self.restart_positions) if result == -1: self.start = self.length raise StopIteration @@ -952,15 +1084,18 @@ if (isinstance(node1, LiteralStringNode) and isinstance(node2, LiteralStringNode)): return node1.s == node2.s - iter1 = CharIterator(node1) - iter2 = CharIterator(node2) + if (isinstance(node1, LiteralUnicodeNode) and + isinstance(node2, LiteralUnicodeNode)): + return node1.u == node2.u + iter1 = ItemIterator(node1) + iter2 = ItemIterator(node2) # XXX could be cleverer and detect partial equalities while 1: try: - c = iter1.next() + c = iter1.nextint() except StopIteration: return True - if c != iter2.next(): + if c != iter2.nextint(): return False def compare(node1, node2): @@ -975,10 +1110,10 @@ cmplen = min(len1, len2) i = 0 - iter1 = CharIterator(node1) - iter2 = CharIterator(node2) + iter1 = ItemIterator(node1) + iter2 = ItemIterator(node2) while i < cmplen: - diff = ord(iter1.next()) - ord(iter2.next()) + diff = iter1.nextint() - iter2.nextint() if diff != 0: return diff i += 1 @@ -994,6 +1129,6 @@ return -1 x = rope.hash_part() x <<= 1 # get rid of the bit that is always set - x ^= ord(rope.getitem(0)) + x ^= rope.getint(0) x ^= rope.length() return intmask(x) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 13:22:15 2007 @@ -28,7 +28,7 @@ continue start = random.randrange(len(st) // 3) stop = random.randrange(len(st) // 3 * 2, len(st)) - curr = curr[start: stop] + curr = getslice_one(curr, start, stop) st = st[start: stop] return curr, st @@ -38,20 +38,20 @@ LiteralStringNode("d" * 32) + LiteralStringNode("ef" * 32) + LiteralStringNode("")) assert s.depth() == 3 - assert s.flatten() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) + assert s.flatten_string() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) s = s.rebalance() - assert s.flatten() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) + assert s.flatten_string() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) def test_dont_rebalance_again(): s = (LiteralStringNode("a" * 32) + LiteralStringNode("b" * 32) + LiteralStringNode("d" * 32) + LiteralStringNode("e" * 32) + LiteralStringNode("")) assert s.depth() == 3 - assert s.flatten() == "".join([c * 32 for c in "abde"]) + assert s.flatten_string() == "".join([c * 32 for c in "abde"]) s = s.rebalance() assert s.check_balanced() assert s.balanced - assert s.flatten() == "".join([c * 32 for c in "abde"]) + assert s.flatten_string() == "".join([c * 32 for c in "abde"]) def test_random_addition_test(): seed = random.randrange(10000) @@ -67,9 +67,9 @@ else: curr = LiteralStringNode(a) + curr st = a + st - assert curr.flatten() == st + assert curr.flatten_string() == st curr = curr.rebalance() - assert curr.flatten() == st + assert curr.flatten_string() == st def test_getitem(): result = "".join([c * 32 for c in "a", "bc", "d", "ef"]) @@ -79,7 +79,8 @@ s2 = s1.rebalance() for i in range(len(result)): for s in [s1, s2]: - assert s[i] == result[i] + assert s.getchar(i) == result[i] + assert s.getint(i) == ord(result[i]) def test_getslice(): result = "".join([c * 32 for c in "a", "bc", "d", "ef"]) @@ -90,14 +91,14 @@ for s in [s1, s2]: for start in range(0, len(result)): for stop in range(start, len(result)): - assert s[start:stop].flatten() == result[start:stop] + assert getslice_one(s, start, stop).flatten_string() == result[start:stop] def test_getslice_bug(): s1 = LiteralStringNode("/home/arigo/svn/pypy/branch/rope-branch/pypy/bin") s2 = LiteralStringNode("/pypy") s = s1 + s2 r = getslice_one(s, 1, 5) - assert r.flatten() == "home" + assert r.flatten_string() == "home" def test_getslice_step(): @@ -105,13 +106,13 @@ LiteralStringNode("nopqrstu") + LiteralStringNode("vwxyz") + LiteralStringNode("zyxwvut") + LiteralStringNode("srqpomnlk")) s2 = s1.rebalance() - result = s1.flatten() - assert s2.flatten() == result + result = s1.flatten_string() + assert s2.flatten_string() == result for s in [s1, s2]: for start in range(0, len(result)): for stop in range(start, len(result)): for step in range(1, stop - start): - assert s[start:stop:step].flatten() == result[start:stop:step] + assert getslice(s, start, stop, step).flatten_string() == result[start:stop:step] def test_random_addition_and_slicing(): @@ -141,27 +142,27 @@ #import pdb; pdb.set_trace() start = random.randrange(len(st) // 3) stop = random.randrange(len(st) // 3 * 2, len(st)) - curr = curr[start: stop] + curr = getslice_one(curr, start, stop) st = st[start: stop] - assert curr.flatten() == st + assert curr.flatten_string() == st curr = curr.rebalance() - assert curr.flatten() == st + assert curr.flatten_string() == st def test_iteration(): rope, real_st = make_random_string(200) - iter = CharIterator(rope) + iter = ItemIterator(rope) for c in real_st: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) def test_reverse_iteration(): rope, real_st = make_random_string(200) - iter = ReverseCharIterator(rope) + iter = ReverseItemIterator(rope) for c in py.builtin.reversed(real_st): - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) def test_multiply(): strs = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), @@ -175,7 +176,7 @@ for r, st in strs: for i in times: r2 = multiply(r, i) - assert r2.flatten() == st * i + assert r2.flatten_string() == st * i def test_join(): seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), @@ -186,7 +187,7 @@ l = list(l) for s, st in seps: node = join(s, l) - result1 = node.flatten() + result1 = node.flatten_string() result2 = st.join(strs) for i in range(node.length()): assert result1[i] == result2[i] @@ -196,7 +197,7 @@ ':', '213', '>'] l = [LiteralStringNode(s) for s in strings] node = join(LiteralStringNode(""), l) - assert node.flatten() == ''.join(strings) + assert node.flatten_string() == ''.join(strings) def test_join_random(): l, strs = zip(*[make_random_string(10 * i) for i in range(1, 5)]) @@ -205,7 +206,7 @@ make_random_string(500)] for s, st in seps: node = join(s, l) - result1 = node.flatten() + result1 = node.flatten_string() result2 = st.join(strs) for i in range(node.length()): assert result1[i] == result2[i] @@ -214,18 +215,18 @@ rope = BinaryConcatNode(BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")), LiteralStringNode("ghi")) - iter = SeekableCharIterator(rope) + iter = SeekableItemIterator(rope) for c in "abcdefgh": - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c for i in range(7): iter.seekback(i) for c in "abcdefghi"[-1-i:-1]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - c2 = iter.next() + c2 = iter.nextchar() assert c2 == "i" - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) def test_fringe_iterator(): ABC = LiteralStringNode("abc") @@ -277,41 +278,37 @@ LiteralStringNode("def")), LiteralStringNode("ghi")) rope = rope + rope - result = rope.flatten() + result = rope.flatten_string() for j in range(len(result) - 1): for i in range(len(result) - 1 - j): - iter = SeekableCharIterator(rope) + iter = SeekableItemIterator(rope) # if (j, i) == (3, 1): # import pdb; pdb.set_trace() for c in result[:j]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c iter.seekforward(i) for c in result[i + j:]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) -def test_find_char(): +def test_find_int(): rope, st = make_random_string() - rope = rope[10:100] + rope = getslice_one(rope, 10, 100) st = st[10:100] for i in range(len(st)): print i for j in range(i + 1, len(st)): c = st[i:j][(j - i) // 2] - pos = find_char(rope, c, i, j) + pos = find_int(rope, ord(c), i, j) assert pos == st.find(c, i, j) -def test_find_char_bugs(): - r = find_char(LiteralStringNode("ascii"), " ", 0, 5) +def test_find_int_bugs(): + r = find_int(LiteralStringNode("ascii"), ord(" "), 0, 5) assert r == -1 - r = find_char(LiteralStringNode("a"), "a") + r = find_int(LiteralStringNode("a"), ord("a")) assert r == 0 - r = find_char(BinaryConcatNode( - LiteralStringNode("x"), - SliceNode(1, 9, LiteralStringNode("a" * 10))), "a") - assert r == 1 def test_restart_positions(): @@ -335,16 +332,16 @@ for i in range(0, 100, 10): chars = ["a"] * 50 + ["b"] * i node = rope_from_charlist(chars) - assert node.flatten() == "a" * 50 + "b" * i - assert rope_from_charlist([]).flatten() == "" + assert node.flatten_string() == "a" * 50 + "b" * i + assert rope_from_charlist([]).flatten_string() == "" def test_find_iterator(): - for searchstring in ["abc", "a", "", "x", "xyz"]: + for searchstring in ["abc", "a", "", "x", "xyz", "abababcabcabb"]: node = join(LiteralStringNode(searchstring), [LiteralStringNode("cde" * i) for i in range(1, 10)]) - # node.view() + #node.view() iter = FindIterator(node, LiteralStringNode(searchstring)) - s = node.flatten() + s = node.flatten_string() assert s == searchstring.join(["cde" * i for i in range(1, 10)]) start = 0 while 1: @@ -365,10 +362,10 @@ assert hash_rope(rope) == -1 continue h = hash_rope(rope) - x = LiteralStringNode(rope.flatten()).hash_part() + x = LiteralStringNode(rope.flatten_string()).hash_part() assert x == rope.hash_part() x <<= 1 - x ^= ord(rope.getitem(0)) + x ^= rope.getint(0) x ^= rope.length() assert intmask(x) == h # hash again to check for cache effects @@ -496,24 +493,25 @@ for j in range(1, 10000, 7): assert intmask(i ** j) == masked_power(i, j) -def test_EfficientGetitemWraper(): - node1, _ = make_random_string(slicing=False) - node2 = EfficientGetitemWraper(node1) - for i in range(node2.length()): - assert node1.getitem(i) == node2.getitem(i) - for j in range(1000): - i = random.randrange(node1.length()) - assert node1.getitem(i) == node2.getitem(i) def test_seekable_bug(): node = BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")) - iter = SeekableCharIterator(node) - c = iter.next(); assert c == "a" - c = iter.next(); assert c == "b" - c = iter.next(); assert c == "c" + iter = SeekableItemIterator(node) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" iter.seekback(1) - c = iter.next(); assert c == "c" - c = iter.next(); assert c == "d" - c = iter.next(); assert c == "e" - c = iter.next(); assert c == "f" - py.test.raises(StopIteration, iter.next) + c = iter.nextchar(); assert c == "c" + c = iter.nextchar(); assert c == "d" + c = iter.nextchar(); assert c == "e" + c = iter.nextchar(); assert c == "f" + py.test.raises(StopIteration, iter.nextchar) + node = LiteralStringNode("abcdef") + iter = SeekableItemIterator(node) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" + iter.seekback(3) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" From fijal at codespeak.net Thu Nov 15 14:22:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 14:22:54 +0100 (CET) Subject: [pypy-svn] r48707 - pypy/extradoc/talk/pycon2008 Message-ID: <20071115132254.AB041820F@code0.codespeak.net> Author: fijal Date: Thu Nov 15 14:22:53 2007 New Revision: 48707 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: tutorial update. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Thu Nov 15 14:22:53 2007 @@ -36,12 +36,16 @@ Requirements ============ -XXX prepare +Attendees are welcome to bring they're own laptops, while one for a group +should suffice. We would like to know create codespeak.net accounts for people +in advance, in order to provide smooth working on branches. Software +requirements: -* pypy checkout, codespeak account +* subversion -* pygame, ctypes or python2.5, libgc (boehm) - all not necessary, - but very useful +* pypy checkout, from svn + +* python2.4 or 2.5 Note for Reviewers ================== @@ -88,8 +92,8 @@ * Hands-on pypy, 1.5h -Example -+++++++ +Examples +++++++++ We would like to present one or two examples during the talk and offer some tasks for people to experiment with. The exact details about @@ -106,6 +110,19 @@ * Implement dict.keys() as a lazily computed list. +Hands-on session details +++++++++++++++++++++++++ + +The hands-on session would be organized in groups of 2-5 persons, hacking +one particular task, preferable on one computer with assistance of someone +familiar with the pypy codebase. If there would be too many people, people +familiar with pypy would need to move between groups. We'll prepare some tasks +(like mentioned above), people are encouraged to suggest they're own tasks, +preferably in advance so we can tell whether it's feasible to try that +in 1.5h. Thanks to pypy design, we don't expect anyone to be familiar +with coding in RPython, hence it would be only prototyping, with eventual +followup during the sprint time. + * Tutorial outline for website publication, if different from the above. From fijal at codespeak.net Thu Nov 15 14:44:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 14:44:55 +0100 (CET) Subject: [pypy-svn] r48708 - pypy/dist/pypy/translator/c Message-ID: <20071115134455.A80F181F9@code0.codespeak.net> Author: fijal Date: Thu Nov 15 14:44:54 2007 New Revision: 48708 Modified: pypy/dist/pypy/translator/c/node.py Log: Squash some hacks Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Nov 15 14:44:54 2007 @@ -821,11 +821,7 @@ # 'fnobj' is one of the ll_xyz() functions with the suggested_primitive # flag in pypy.rpython.module.*. The corresponding C wrappers are # written by hand in src/ll_*.h, and declared in extfunc.EXTERNALS. - if sandbox and not ( - fnobj._name.startswith('ll_strtod_') # XXX!! TEMPORARY! - or fnobj._name.startswith('ll_time_') # XXX!! TEMPORARY! - or fnobj._name.startswith('ll_stack_') # XXX!! TEMPORARY! - ): + if sandbox and not fnobj._name.startswith('ll_stack_'): return sandbox_stub(fnobj, db) db.externalfuncs[fnobj._callable] = fnobj return [] From fijal at codespeak.net Thu Nov 15 14:46:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 14:46:11 +0100 (CET) Subject: [pypy-svn] r48709 - pypy/dist/pypy/translator/c Message-ID: <20071115134611.D4B8D82A5@code0.codespeak.net> Author: fijal Date: Thu Nov 15 14:46:11 2007 New Revision: 48709 Modified: pypy/dist/pypy/translator/c/node.py Log: Leave comment that this is really a hack Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Nov 15 14:46:11 2007 @@ -821,7 +821,7 @@ # 'fnobj' is one of the ll_xyz() functions with the suggested_primitive # flag in pypy.rpython.module.*. The corresponding C wrappers are # written by hand in src/ll_*.h, and declared in extfunc.EXTERNALS. - if sandbox and not fnobj._name.startswith('ll_stack_'): + if sandbox and not fnobj._name.startswith('ll_stack_'): # XXX!!! Temporary return sandbox_stub(fnobj, db) db.externalfuncs[fnobj._callable] = fnobj return [] From fijal at codespeak.net Thu Nov 15 15:02:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 15:02:26 +0100 (CET) Subject: [pypy-svn] r48710 - pypy/dist/pypy/rpython/module Message-ID: <20071115140226.ADD95820D@code0.codespeak.net> Author: fijal Date: Thu Nov 15 15:02:25 2007 New Revision: 48710 Modified: pypy/dist/pypy/rpython/module/ll_strtod.py Log: Put sandboxsafe flag here as well. Took me quite a bit to figure out :-/ Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Thu Nov 15 15:02:25 2007 @@ -39,7 +39,8 @@ return ootype.oostring(rarithmetic.formatd(fmt._str, x), -1) return extdef([str, float], str, 'll_strtod.ll_strtod_formatd', - llimpl=llimpl, oofakeimpl=oofakeimpl) + llimpl=llimpl, oofakeimpl=oofakeimpl, + sandboxsafe=True) @registering(rarithmetic.parts_to_float) def register_parts_to_float(self): @@ -60,4 +61,4 @@ return extdef([str, str, str, str], float, 'll_strtod.ll_strtod_parts_to_float', llimpl=llimpl, - oofakeimpl=oofakeimpl) + oofakeimpl=oofakeimpl, sandboxsafe=True) From lac at codespeak.net Thu Nov 15 15:12:57 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Thu, 15 Nov 2007 15:12:57 +0100 (CET) Subject: [pypy-svn] r48711 - pypy/extradoc/talk/pycon2008 Message-ID: <20071115141257.7FF2A81F7@code0.codespeak.net> Author: lac Date: Thu Nov 15 15:12:56 2007 New Revision: 48711 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: rewrite openoing paragraph, and stick this in before fijal makes more major restructuring Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Thu Nov 15 15:12:56 2007 @@ -50,12 +50,16 @@ Note for Reviewers ================== -As this is a very advanced topic, we don't expect too many people to show -up. In case the tutorial would be accepted and there would be less than 10 -people showing up, we can do the tutorial for free. - -Also, in case there would be CPython developement tutorial, it would be -nice if those two will not collide, if possible. +Note to the organisers: This is a very advanced topic, and therefore +it is hard to predict how many people will show up. We realise that +you reserve the right to cancel tutorials if fewer than 10 people +indicate an interest, and we think that this is a very real possibility +in our case. Should that occur, we would be willing to give the +tutorial for free. + +If there is a python-dev tutorial planned for python developers, then +it would be a good idea if they did not conflict, because this +tutorial exactly targets the same potential audience. Promotional summary =================== @@ -69,7 +73,7 @@ session. Afterwards will be welcomed to experiment in groups on they're own on implementing prepared interpreter changes. -Attendees are welcome to bring their own laptops. +Attendees are welcome to bring their own laptops. Outline ======= From pypy-svn at codespeak.net Thu Nov 15 15:29:09 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Thu, 15 Nov 2007 15:29:09 +0100 (CET) Subject: [pypy-svn] October 72% OFF Message-ID: <20071115082904.5272.qmail@ppp91-76-79-234.pppoe.mtu-net.ru> An HTML attachment was scrubbed... URL: From fijal at codespeak.net Thu Nov 15 15:33:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Nov 2007 15:33:56 +0100 (CET) Subject: [pypy-svn] r48712 - pypy/dist/pypy/interpreter Message-ID: <20071115143356.AA91C8252@code0.codespeak.net> Author: fijal Date: Thu Nov 15 15:33:55 2007 New Revision: 48712 Modified: pypy/dist/pypy/interpreter/executioncontext.py Log: Add a comment about minor optimization possible Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Thu Nov 15 15:33:55 2007 @@ -99,6 +99,9 @@ def bytecode_trace(self, frame): "Trace function called before each bytecode." + # XXX there should be some flag here which checks whether + # this should be really invoked. We spend roughly 0.5% time + # here when not doing anything # First, call yield_thread() before each Nth bytecode, # as selected by sys.setcheckinterval() ticker = self.ticker From cfbolz at codespeak.net Thu Nov 15 18:05:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 18:05:33 +0100 (CET) Subject: [pypy-svn] r48713 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115170533.1112381BE@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 18:05:33 2007 New Revision: 48713 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: off by one error ? all tests pass Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 18:05:33 2007 @@ -760,7 +760,7 @@ elif j>0: new_j = restart[j-1] assert new_j < j - iter2.seekback(j - new_j) + iter2.seekback(j - new_j + 1) c2 = iter2.nextint() j = new_j else: Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 18:05:33 2007 @@ -317,6 +317,17 @@ assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] restart = construct_restart_positions("ababcabab") assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] + restart = construct_restart_positions("ababcababb") + assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4, 0] + restart = construct_restart_positions_node( + BinaryConcatNode(LiteralStringNode("aba"), LiteralStringNode("bcababb"))) + assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4, 0] + restart = construct_restart_positions("ababb") + assert restart == [0, 0, 1, 2, 0] + restart = construct_restart_positions_node(LiteralStringNode("ababb")) + assert restart == [0, 0, 1, 2, 0] + #abababcabcabb + def test_find(): node = BinaryConcatNode(LiteralStringNode("aba"), From cfbolz at codespeak.net Thu Nov 15 18:22:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 18:22:27 +0100 (CET) Subject: [pypy-svn] r48714 - pypy/branch/ropes-unicode/pypy/objspace/std Message-ID: <20071115172227.4624D8306@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 18:22:25 2007 New Revision: 48714 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Log: make the ropes of the std obj space work again Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Thu Nov 15 18:22:25 2007 @@ -24,7 +24,7 @@ return "%s(%r)" % (w_self.__class__.__name__, w_self._node) def unwrap(w_self, space): - return w_self._node.flatten() + return w_self._node.flatten_string() def create_if_subclassed(w_self): if type(w_self) is W_RopeObject: @@ -49,7 +49,7 @@ def __init__(w_self, w_rope, index=0): w_self.node = node = w_rope._node - w_self.char_iter = rope.CharIterator(node) + w_self.item_iter = rope.ItemIterator(node) w_self.index = index registerimplementation(W_RopeIterObject) @@ -58,9 +58,9 @@ l = w_self._node.length() if l == 0: return space.w_False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for i in range(l): - if not fun(iter.next()): + if not fun(iter.nextchar()): return space.w_False return space.w_True _is_generic._annspecialcase_ = "specialize:arg(2)" @@ -104,9 +104,9 @@ if l == 0: return space.w_False cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for idx in range(l): - c = iter.next() + c = iter.nextchar() if c.islower(): return space.w_False elif not cased and c.isupper(): @@ -121,9 +121,9 @@ if l == 0: return space.w_False cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for idx in range(l): - c = iter.next() + c = iter.nextchar() if c.isupper(): return space.w_False elif not cased and c.islower(): @@ -138,9 +138,9 @@ cased = False previous_is_cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for pos in range(0, w_self._node.length()): - ch = iter.next() + ch = iter.nextchar() if ch.isupper(): if previous_is_cased: return space.w_False @@ -158,9 +158,9 @@ def str_upper__Rope(space, w_self): l = w_self._node.length() res = [' '] * l - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for i in range(l): - ch = iter.next() + ch = iter.nextchar() res[i] = _upper(ch) return W_RopeObject(rope.rope_from_charlist(res)) @@ -168,9 +168,9 @@ def str_lower__Rope(space, w_self): l = w_self._node.length() res = [' '] * l - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for i in range(l): - ch = iter.next() + ch = iter.nextchar() res[i] = _lower(ch) return W_RopeObject(rope.rope_from_charlist(res)) @@ -178,9 +178,9 @@ def str_swapcase__Rope(space, w_self): l = w_self._node.length() res = [' '] * l - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for i in range(l): - ch = iter.next() + ch = iter.nextchar() if ch.isupper(): o = ord(ch) + 32 res[i] = chr(o) @@ -198,8 +198,8 @@ length = node.length() buffer = [' '] * length if length > 0: - iter = rope.CharIterator(node) - ch = iter.next() + iter = rope.ItemIterator(node) + ch = iter.nextchar() if ch.islower(): o = ord(ch) - 32 buffer[0] = chr(o) @@ -207,7 +207,7 @@ buffer[0] = ch for i in range(1, length): - ch = iter.next() + ch = iter.nextchar() if ch.isupper(): o = ord(ch) + 32 buffer[i] = chr(o) @@ -224,9 +224,9 @@ buffer = [' '] * length prev_letter = ' ' - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) for pos in range(0, length): - ch = iter.next() + ch = iter.nextchar() if not prev_letter.isalpha(): buffer[pos] = _upper(ch) else: @@ -242,11 +242,11 @@ node = w_self._node length = node.length() i = 0 - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) while True: # find the beginning of the next word while i < length: - if not iter.next().isspace(): + if not iter.nextchar().isspace(): break # found i += 1 else: @@ -257,7 +257,7 @@ j = length # take all the rest of the string else: j = i + 1 - while j < length and not iter.next().isspace(): + while j < length and not iter.nextchar().isspace(): j += 1 maxsplit -= 1 # NB. if it's already < 0, it stays < 0 @@ -298,7 +298,7 @@ # XXX works but flattens maxsplit = space.int_w(w_maxsplit) res_w = [] - value = w_self._node.flatten() + value = w_self._node.flatten_string() i = len(value)-1 while True: # starting from the end, find the end of the next word @@ -334,9 +334,9 @@ # XXX works but flattens maxsplit = space.int_w(w_maxsplit) res_w = [] - value = w_self._node.flatten() + value = w_self._node.flatten_string() end = len(value) - by = w_by._node.flatten() + by = w_by._node.flatten_string() bylen = len(by) if bylen == 0: raise OperationError(space.w_ValueError, space.wrap("empty separator")) @@ -446,8 +446,8 @@ def str_rfind__Rope_Rope_ANY_ANY(space, w_self, w_sub, w_start, w_end): # XXX works but flattens (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - self = self.flatten() - sub = sub.flatten() + self = self.flatten_string() + sub = sub.flatten_string() res = self.rfind(sub, start, end) return wrapint(space, res) @@ -475,8 +475,8 @@ if not sub.length(): raise OperationError(space.w_ValueError, space.wrap("empty separator")) - flattened_self = self.flatten() - flattened_sub = sub.flatten() + flattened_self = self.flatten_string() + flattened_sub = sub.flatten_string() pos = flattened_self.rfind(flattened_sub) if pos == -1: return space.newtuple([W_RopeObject.EMPTY, W_RopeObject.EMPTY, w_self]) @@ -501,8 +501,8 @@ def str_rindex__Rope_Rope_ANY_ANY(space, w_self, w_sub, w_start, w_end): (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) # XXX works but flattens - self = self.flatten() - sub = sub.flatten() + self = self.flatten_string() + sub = sub.flatten_string() res = self.rfind(sub, start, end) if res < 0: raise OperationError(space.w_ValueError, @@ -527,9 +527,9 @@ upper = maxsplit - 1 assert upper >= 0 substrings = [by] - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) for i in range(upper): - substrings.append(rope.LiteralStringNode.PREBUILT[ord(iter.next())]) + substrings.append(rope.LiteralStringNode.PREBUILT[ord(iter.nextchar())]) substrings.append(by) substrings.append(rope.getslice_one(node, upper, length)) try: @@ -570,13 +570,13 @@ if left: #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.CharIterator(node) - while lpos < rpos and iter.next() in u_chars: + iter = rope.ItemIterator(node) + while lpos < rpos and iter.nextchar() in u_chars: lpos += 1 if right: - iter = rope.ReverseCharIterator(node) - while rpos > lpos and iter.next() in u_chars: + iter = rope.ReverseItemIterator(node) + while rpos > lpos and iter.nextchar() in u_chars: rpos -= 1 return W_RopeObject(rope.getslice_one(node, lpos, rpos)) @@ -591,13 +591,13 @@ if left: #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.CharIterator(node) - while lpos < rpos and iter.next().isspace(): + iter = rope.ItemIterator(node) + while lpos < rpos and iter.nextchar().isspace(): lpos += 1 if right: - iter = rope.ReverseCharIterator(node) - while rpos > lpos and iter.next().isspace(): + iter = rope.ReverseItemIterator(node) + while rpos > lpos and iter.nextchar().isspace(): rpos -= 1 assert rpos >= lpos # annotator hint, don't remove @@ -671,11 +671,11 @@ begin = end - suffix.length() if begin < start: return False - iter1 = rope.SeekableCharIterator(self) + iter1 = rope.SeekableItemIterator(self) iter1.seekforward(begin) - iter2 = rope.CharIterator(suffix) + iter2 = rope.ItemIterator(suffix) for i in range(suffix.length()): - if iter1.next() != iter2.next(): + if iter1.nextchar() != iter2.nextchar(): return False return True @@ -702,11 +702,11 @@ stop = start + prefix.length() if stop > end: return False - iter1 = rope.SeekableCharIterator(self) + iter1 = rope.SeekableItemIterator(self) iter1.seekforward(start) - iter2 = rope.CharIterator(prefix) + iter2 = rope.ItemIterator(prefix) for i in range(prefix.length()): - if iter1.next() != iter2.next(): + if iter1.nextchar() != iter2.nextchar(): return False return True @@ -732,13 +732,13 @@ distance = tabsize if length: distance = 0 - iter = rope.ReverseCharIterator(node) + iter = rope.ReverseItemIterator(node) while 1: # no sophisticated linebreak support now # '\r' just for passing adapted CPython test try: - char = iter.next() + char = iter.nextchar() except StopIteration: break if char == "\n" or char == "\r": @@ -800,17 +800,17 @@ return space.newlist([]) strs_w = [] - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) i = j = 0 last = " " - char = iter.next() + char = iter.nextchar() while i < length: # Find a line and append it while char != '\n' and char != '\r': try: i += 1 last = char - char = iter.next() + char = iter.nextchar() except StopIteration: break # Skip the line break reading CRLF as one line break @@ -818,7 +818,7 @@ i += 1 last = char try: - char = iter.next() + char = iter.nextchar() except StopIteration: pass else: @@ -826,7 +826,7 @@ i += 1 try: last = char - char = iter.next() + char = iter.nextchar() except StopIteration: pass if keepends: @@ -853,7 +853,7 @@ return W_RopeObject(rope.multiply(zero, width)) middle = width - length - firstchar = node.getitem(0) + firstchar = node.getchar(0) if length > 0 and (firstchar == '+' or firstchar == '-'): return W_RopeObject(rope.rebalance( [rope.LiteralStringNode.PREBUILT[ord(firstchar)], @@ -865,7 +865,7 @@ rope.multiply(zero, middle), node)) def str_w__Rope(space, w_str): - return w_str._node.flatten() + return w_str._node.flatten_string() def hash__Rope(space, w_str): return wrapint(space, rope.hash_rope(w_str._node)) @@ -910,7 +910,7 @@ exc = space.call_function(space.w_IndexError, space.wrap("string index out of range")) raise OperationError(space.w_IndexError, exc) - return wrapchar(space, node.getitem(ival)) + return wrapchar(space, node.getchar(ival)) def getitem__Rope_Slice(space, w_str, w_slice): node = w_str._node @@ -976,7 +976,7 @@ space.w_TypeError, space.wrap("ord() expected a character, but string " "of length %d found"% (w_str._node.length(),))) - return space.wrap(ord(node.flatten()[0])) + return space.wrap(ord(node.flatten_string()[0])) def getnewargs__Rope(space, w_str): return space.newtuple([W_RopeObject(w_str._node)]) @@ -989,15 +989,16 @@ buf = [' '] * (length * 4 + 2) # safely overallocate quote = "'" - if rope.find_char(node, quote) != -1 and rope.find_char(node, '"') == -1: + if (rope.find_int(node, ord(quote)) != -1 and + rope.find_int(node, ord('"')) == -1): quote = '"' buf[0] = quote - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) while 1: try: - c = iter.next() + c = iter.nextchar() i += 1 except StopIteration: break @@ -1047,10 +1048,10 @@ node = w_string._node chars = [] - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) while 1: try: - c = iter.next() + c = iter.nextchar() w_char = W_RopeObject.PREBUILT[ord(c)] if not space.is_true(space.contains(w_deletechars, w_char)): chars.append(table[ord(c)]) @@ -1082,7 +1083,7 @@ if w_ropeiter.node is None: raise OperationError(space.w_StopIteration, space.w_None) try: - char = w_ropeiter.char_iter.next() + char = w_ropeiter.item_iter.nextchar() w_item = space.wrap(char) except StopIteration: w_ropeiter.node = None From cfbolz at codespeak.net Thu Nov 15 18:39:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 18:39:04 +0100 (CET) Subject: [pypy-svn] r48715 - pypy/branch/ropes-unicode/pypy/objspace/std Message-ID: <20071115173904.4D6158314@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 18:39:02 2007 New Revision: 48715 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Log: move start/endswith helpers to rope.py Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 18:39:02 2007 @@ -1120,6 +1120,39 @@ return len1 - len2 +def startswith(self, prefix, start, end): + if prefix.length() == 0: + return True + if self.length() == 0: + return False + stop = start + prefix.length() + if stop > end: + return False + iter1 = SeekableItemIterator(self) + iter1.seekforward(start) + iter2 = ItemIterator(prefix) + for i in range(prefix.length()): + if iter1.nextint() != iter2.nextint(): + return False + return True + +def endswith(self, suffix, start, end): + if suffix.length() == 0: + return True + if self.length() == 0: + return False + begin = end - suffix.length() + if begin < start: + return False + iter1 = SeekableItemIterator(self) + iter1.seekforward(begin) + iter2 = ItemIterator(suffix) + for i in range(suffix.length()): + if iter1.nextint() != iter2.nextint(): + return False + return True + + # __________________________________________________________________________ # misc Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Thu Nov 15 18:39:02 2007 @@ -663,65 +663,34 @@ i += 1 return wrapint(space, i) -def ropeendswith(self, suffix, start, end): - if suffix.length() == 0: - return True - if self.length() == 0: - return False - begin = end - suffix.length() - if begin < start: - return False - iter1 = rope.SeekableItemIterator(self) - iter1.seekforward(begin) - iter2 = rope.ItemIterator(suffix) - for i in range(suffix.length()): - if iter1.nextchar() != iter2.nextchar(): - return False - return True - def str_endswith__Rope_Rope_ANY_ANY(space, w_self, w_suffix, w_start, w_end): (self, suffix, start, end) = _convert_idx_params(space, w_self, w_suffix, w_start, w_end) - return space.newbool(ropeendswith(self, suffix, start, end)) + return space.newbool(rope.endswith(self, suffix, start, end)) def str_endswith__Rope_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): suffix = rope_w(space, w_suffix) - if ropeendswith(self, suffix, start, end): + if rope.endswith(self, suffix, start, end): return space.w_True return space.w_False -def ropestartswith(self, prefix, start, end): - if prefix.length() == 0: - return True - if self.length() == 0: - return False - stop = start + prefix.length() - if stop > end: - return False - iter1 = rope.SeekableItemIterator(self) - iter1.seekforward(start) - iter2 = rope.ItemIterator(prefix) - for i in range(prefix.length()): - if iter1.nextchar() != iter2.nextchar(): - return False - return True def str_startswith__Rope_Rope_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (self, prefix, start, end) = _convert_idx_params(space, w_self, w_prefix, w_start, w_end) - return space.newbool(ropestartswith(self, prefix, start, end)) + return space.newbool(rope.startswith(self, prefix, start, end)) def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): prefix = rope_w(space, w_prefix) - if ropestartswith(self, prefix, start, end): + if rope.startswith(self, prefix, start, end): return space.w_True return space.w_False From cfbolz at codespeak.net Thu Nov 15 18:44:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 18:44:19 +0100 (CET) Subject: [pypy-svn] r48716 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115174419.57021824D@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 18:44:18 2007 New Revision: 48716 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: some tests for unicode support in ropes Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 18:44:18 2007 @@ -220,7 +220,7 @@ def getunichar(self, index): return self.u[index] - def getbyte(self, index): + def getint(self, index): return ord(self.u[index]) def getslice(self, start, stop): Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 18:44:18 2007 @@ -1,8 +1,9 @@ import py -import random +import random, sys from pypy.objspace.std.rope import * -def make_random_string(operations=10, slicing=True, print_seed=True): +def make_random_string(operations=10, slicing=True, print_seed=True, + unicode=False): seed = random.randrange(10000) if print_seed: print seed @@ -14,14 +15,20 @@ else: choice = [0, 1] for i in range(operations): - a = (chr(random.randrange(ord('a'), ord('z') + 1)) * - random.randrange(500)) + if not unicode: + a = (chr(random.randrange(ord('a'), ord('z') + 1)) * + random.randrange(500)) + node = LiteralStringNode(a) + else: + a = (unichr(random.randrange(sys.maxunicode)) * + random.randrange(500)) + node = LiteralUnicodeNode(a) c = random.choice(choice) if c == 0: - curr = curr + LiteralStringNode(a) + curr = curr + node st = st + a elif c == 1: - curr = LiteralStringNode(a) + curr + curr = node + curr st = a + st else: if len(st) < 10: @@ -82,6 +89,14 @@ assert s.getchar(i) == result[i] assert s.getint(i) == ord(result[i]) +def test_getitem_unicode(): + s1, result = make_random_string(200, unicode=True) + s2 = s1.rebalance() + for i in range(len(result)): + for s in [s1, s2]: + assert s.getunichar(i) == result[i] + assert s.getint(i) == ord(result[i]) + def test_getslice(): result = "".join([c * 32 for c in "a", "bc", "d", "ef"]) s1 = (LiteralStringNode("a" * 32) + LiteralStringNode("bc" * 32) + From cfbolz at codespeak.net Thu Nov 15 19:01:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 19:01:12 +0100 (CET) Subject: [pypy-svn] r48717 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115180112.F065082D5@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 19:01:12 2007 New Revision: 48717 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: tests and more fixes for unicode ops with ropes Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 19:01:12 2007 @@ -109,6 +109,7 @@ class LiteralStringNode(LiteralNode): def __init__(self, s): + assert isinstance(s, str) self.s = s is_ascii = True for c in s: @@ -190,8 +191,9 @@ del i -class LiteralUnicodeNode(StringNode): +class LiteralUnicodeNode(LiteralNode): def __init__(self, u): + assert isinstance(u, unicode) self.u = u def length(self): @@ -228,7 +230,7 @@ return LiteralUnicodeNode(self.u[start:stop]) def find_int(self, what, start, stop): - result = node.u.find(unichr(what), start, stop) + result = self.u.find(unichr(what), start, stop) if result == -1: return -1 return result @@ -236,7 +238,7 @@ def literal_concat(self, other): if (isinstance(other, LiteralUnicodeNode) and len(other.u) + len(self.u) < NEW_NODE_WHEN_LENGTH): - return LiteralStringNode(self.u + other.u) + return LiteralUnicodeNode(self.u + other.u) elif (isinstance(other, LiteralStringNode) and len(other.s) + len(self.u) < NEW_NODE_WHEN_LENGTH and len(other.s) < CONVERT_WHEN_SMALLER): @@ -247,11 +249,11 @@ if self in seen: return seen[self] = True - addinfo = str(self.s).replace('"', "'") or "_" + addinfo = repr(self.u).replace('"', "'") or "_" if len(addinfo) > 10: addinfo = addinfo[:3] + "..." + addinfo[-3:] yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( - id(self), len(self.s), + id(self), len(self.u), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) class BinaryConcatNode(StringNode): @@ -322,7 +324,7 @@ def flatten_unicode(self): f = fringe(self) - return "".join([node.flatten_string() for node in f]) + return "".join([node.flatten_unicode() for node in f]) def hash_part(self): h = self.hash_cache @@ -1038,8 +1040,6 @@ self.restart_positions = None else: self.restart_positions = construct_restart_positions_node(sub) - # XXX - assert self.restart_positions == construct_restart_positions(sub.flatten_string()) self.start = start if stop == -1 or stop > len1: stop = len1 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 19:01:12 2007 @@ -380,8 +380,31 @@ assert r1 == r2 start = r2 + max(len(searchstring), 1) + +def test_find_iterator_unicode(): + for searchstring in [ + u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", + u"\u6666\u7777\u8888", + u"\uAAAA\uBBBB\uAAAA\uBBBB\uAAAA\uBBBB\uCCCC\uAAAA\uBBBB\uCCCC\uAAAA\uBBBB\uBBBB"]: + node = join(LiteralUnicodeNode(searchstring), + [LiteralUnicodeNode(u"\ucccc\udddd" * i) + for i in range(1, 10)]) + iter = FindIterator(node, LiteralUnicodeNode(searchstring)) + s = node.flatten_unicode() + assert s == searchstring.join([u"\ucccc\udddd" * i for i in range(1, 10)]) + start = 0 + while 1: + r2 = s.find(searchstring, start) + try: + r1 = iter.next() + except StopIteration: + assert r2 == -1 + break + assert r1 == r2 + start = r2 + max(len(searchstring), 1) + def test_hash(): - from pypy.rlib.rarithmetic import _hash_string, intmask + from pypy.rlib.rarithmetic import intmask for i in range(10): rope, _ = make_random_string() if rope.length() == 0: @@ -489,13 +512,26 @@ assert s.hash_part() == h assert s.hash_part() == h +def test_hash_part_unicode(): + a, st = make_random_string(unicode=True) + h = a.hash_part() + for split in range(1, len(st) - 1): + s1 = LiteralUnicodeNode(st[:split]) + s2 = LiteralUnicodeNode(st[split:]) + s = BinaryConcatNode(s1, s2) + if h is None: + h = s.hash_part() + else: + # try twice due to caching reasons + assert s.hash_part() == h + assert s.hash_part() == h + def test_hash_part_more(): for i in range(100): rope, st = make_random_string() h = rope.hash_part() assert LiteralStringNode(st).hash_part() == h - def test_equality(): l = [make_random_string() for i in range(3)] l.append((LiteralStringNode(""), "")) @@ -503,6 +539,13 @@ for rope2, st2 in l: assert eq(rope1, rope2) == (st1 == st2) +def test_equality_unicode(): + l = [make_random_string() for i in range(3)] + l.append((LiteralStringNode(""), "")) + for rope1, st1 in l: + for rope2, st2 in l: + assert eq(rope1, rope2) == (st1 == st2) + def test_compare_random(): l = [make_random_string() for i in range(3)] l.append((LiteralStringNode(""), "")) @@ -513,6 +556,17 @@ c = c // abs(c) assert c == cmp(st1, st2) +def test_compare_random_unicode(): + l = [make_random_string() for i in range(3)] + l.append((LiteralStringNode(""), "")) + for rope1, st1 in l: + for rope2, st2 in l: + c = compare(rope1, rope2) + if c: + c = c // abs(c) + assert c == cmp(st1, st2) + + def test_power(): for i in range(0, 60, 13): print i From cfbolz at codespeak.net Thu Nov 15 19:10:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 19:10:53 +0100 (CET) Subject: [pypy-svn] r48718 - pypy/branch/ropes-unicode/pypy/objspace/std/test Message-ID: <20071115181053.04E108326@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 19:10:53 2007 New Revision: 48718 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: more passing tests Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 19:10:53 2007 @@ -170,6 +170,29 @@ c2 = iter.nextchar() assert c2 == c py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) + +def test_iteration_unicode(): + rope, real_st = make_random_string(200, unicode=True) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) def test_reverse_iteration(): rope, real_st = make_random_string(200) @@ -178,6 +201,24 @@ c2 = iter.nextchar() assert c2 == c py.test.raises(StopIteration, iter.nextchar) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) + +def test_reverse_iteration_unicode(): + rope, real_st = make_random_string(200, unicode=True) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) def test_multiply(): strs = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), @@ -193,6 +234,21 @@ r2 = multiply(r, i) assert r2.flatten_string() == st * i +def test_multiply_unicode(): + strs = [(LiteralUnicodeNode(u"\uaaaa"), u"\uaaaa"), + (LiteralUnicodeNode(u"\uaaaa\ubbbb\ucccc"), u"\uaaaa\ubbbb\ucccc"), + make_random_string(500)] + times = range(100) + for i in range(9, 30): + times.append(i ** 2 - 1) + times.append(i ** 2) + times.append(i ** 2 + 1) + times.append(i ** 2 + 2) + for r, st in strs: + for i in times: + r2 = multiply(r, i) + assert r2.flatten_unicode() == st * i + def test_join(): seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), (LiteralStringNode("d"), "d"), (LiteralStringNode(""), "")] @@ -226,6 +282,18 @@ for i in range(node.length()): assert result1[i] == result2[i] +def test_join_random_unicode(): + l, strs = zip(*[make_random_string(10 * i, unicode=True) for i in range(1, 5)]) + l = list(l) + seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), + make_random_string(500)] + for s, st in seps: + node = join(s, l) + result1 = node.flatten_unicode() + result2 = st.join(strs) + for i in range(node.length()): + assert result1[i] == result2[i] + def test_seekbackward(): rope = BinaryConcatNode(BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")), From cfbolz at codespeak.net Thu Nov 15 22:19:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 22:19:10 +0100 (CET) Subject: [pypy-svn] r48719 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115211910.6AB5180BC@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 22:19:08 2007 New Revision: 48719 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: some more functionality for ropes Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 22:19:08 2007 @@ -80,6 +80,9 @@ def getint(self, index): raise NotImplementedError("abstract base class") + def getrope(self, index): + raise NotImplementedError("abstract base class") + def getslice(self, start, stop): raise NotImplementedError("abstract base class") @@ -153,6 +156,9 @@ def getint(self, index): return ord(self.s[index]) + def getrope(self, index): + return LiteralStringNode.PREBUILT[ord(self.s[index])] + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralStringNode(self.s[start:stop]) @@ -225,6 +231,14 @@ def getint(self, index): return ord(self.u[index]) + def getrope(self, index): + ch = ord(self.u[index]) + if ch < 256: + return LiteralStringNode.PREBUILT[ord(self.s[index])] + if len(self.u) == 1: + return self + return LiteralUnicodeNode(unichr(ch)) + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralUnicodeNode(self.u[start:stop]) @@ -256,6 +270,7 @@ id(self), len(self.u), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) + class BinaryConcatNode(StringNode): def __init__(self, left, right): self.left = left @@ -318,6 +333,13 @@ else: return self.left.getint(index) + def getrope(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getrope(index - llen) + else: + return self.left.getrope(index) + def flatten_string(self): f = fringe(self) return "".join([node.flatten_string() for node in f]) @@ -590,6 +612,9 @@ if chunk: nodelist.append(LiteralStringNode("".join(chunk))) return rebalance(nodelist, length) +rope_from_unicharlist._annspecialcase_ = "specialize:argtype(0)" + +rope_from_unicode = rope_from_unicharlist # __________________________________________________________________________ # searching @@ -1152,6 +1177,25 @@ return False return True +def strip(node, left=True, right=True, predicate=lambda i: chr(i).isspace()): + length = node.length() + + lpos = 0 + rpos = length + + if left: + iter = ItemIterator(node) + while lpos < rpos and predicate(iter.nextint()): + lpos += 1 + + if right: + iter = ReverseItemIterator(node) + while rpos > lpos and predicate(iter.nextint()): + rpos -= 1 + + assert rpos >= lpos + return getslice_one(node, lpos, rpos) +strip._annspecialcase_ = "specialize:arg(3)" # __________________________________________________________________________ # misc @@ -1165,3 +1209,4 @@ x ^= rope.getint(0) x ^= rope.length() return intmask(x) + Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 22:19:08 2007 @@ -663,3 +663,35 @@ c = iter.nextchar(); assert c == "a" c = iter.nextchar(); assert c == "b" c = iter.nextchar(); assert c == "c" + +def test_strip(): + node = BinaryConcatNode(LiteralStringNode(" \t\n abc "), + LiteralStringNode("def ")) + r = strip(node) + assert r.flatten_string() == "abc def" + r = strip(node, left=False) + assert r.flatten_string() == " \t\n abc def" + r = strip(node, right=False) + assert r.flatten_string() == "abc def " + + node = BinaryConcatNode(LiteralStringNode("aaaYYYYa"), + LiteralStringNode("abab")) + predicate = lambda i: chr(i) in "ab" + r = strip(node, predicate=predicate) + assert r.flatten_string() == "YYYY" + r = strip(node, left=False, predicate=predicate) + assert r.flatten_string() == "aaaYYYY" + r = strip(node, right=False, predicate=predicate) + assert r.flatten_string() == "YYYYaabab" + +def test_getrope(): + s1, result = make_random_string(200, unicode=True) + s2 = s1.rebalance() + for i in range(len(result)): + for s in [s1, s2]: + r = s.getrope(i) + assert r.length() == 1 + assert r.getint(0) == s.getint(i) + assert isinstance(r, LiteralNode) + assert r.getint(0) >= 128 or isinstance(r, LiteralStringNode) + assert r.getrope(0) is r From cfbolz at codespeak.net Thu Nov 15 23:55:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 15 Nov 2007 23:55:34 +0100 (CET) Subject: [pypy-svn] r48720 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071115225534.2A93482BF@code0.codespeak.net> Author: cfbolz Date: Thu Nov 15 23:55:32 2007 New Revision: 48720 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: add a split function to the rope module Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Thu Nov 15 23:55:32 2007 @@ -1197,6 +1197,21 @@ return getslice_one(node, lpos, rpos) strip._annspecialcase_ = "specialize:arg(3)" +def split(node, sub, maxsplit=-1): + startidx = 0 + substrings = [] + iter = FindIterator(node, sub) + while maxsplit != 0: + try: + foundidx = iter.next() + except StopIteration: + break + substrings.append(getslice_one(node, startidx, foundidx)) + startidx = foundidx + sub.length() + maxsplit = maxsplit - 1 + substrings.append(getslice_one(node, startidx, node.length())) + return substrings + # __________________________________________________________________________ # misc Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Thu Nov 15 23:55:32 2007 @@ -17,6 +17,7 @@ from pypy.objspace.std.stringtype import str_typedef as typedef def __init__(w_self, node): + assert node.is_bytestring() w_self._node = node def __repr__(w_self): @@ -155,42 +156,35 @@ return space.newbool(cased) -def str_upper__Rope(space, w_self): - l = w_self._node.length() +def _local_transform(node, transform): + l = node.length() res = [' '] * l - iter = rope.ItemIterator(w_self._node) + iter = rope.ItemIterator(node) for i in range(l): ch = iter.nextchar() - res[i] = _upper(ch) + res[i] = transform(ch) return W_RopeObject(rope.rope_from_charlist(res)) +_local_transform._annspecialcase_ = "specialize:arg(1)" + +def str_upper__Rope(space, w_self): + return _local_transform(w_self._node, _upper) def str_lower__Rope(space, w_self): - l = w_self._node.length() - res = [' '] * l - iter = rope.ItemIterator(w_self._node) - for i in range(l): - ch = iter.nextchar() - res[i] = _lower(ch) + return _local_transform(w_self._node, _lower) - return W_RopeObject(rope.rope_from_charlist(res)) +def _swapcase(ch): + if ch.isupper(): + o = ord(ch) + 32 + return chr(o) + elif ch.islower(): + o = ord(ch) - 32 + return chr(o) + else: + return ch def str_swapcase__Rope(space, w_self): - l = w_self._node.length() - res = [' '] * l - iter = rope.ItemIterator(w_self._node) - for i in range(l): - ch = iter.nextchar() - if ch.isupper(): - o = ord(ch) + 32 - res[i] = chr(o) - elif ch.islower(): - o = ord(ch) - 32 - res[i] = chr(o) - else: - res[i] = ch - - return W_RopeObject(rope.rope_from_charlist(res)) + return _local_transform(w_self._node, _swapcase) def str_capitalize__Rope(space, w_self): @@ -218,6 +212,7 @@ return W_RopeObject(rope.rope_from_charlist(buffer)) + def str_title__Rope(space, w_self): node = w_self._node length = node.length() @@ -269,29 +264,16 @@ return space.newlist(res_w) - def str_split__Rope_Rope_ANY(space, w_self, w_by, w_maxsplit=-1): maxsplit = space.int_w(w_maxsplit) - res_w = [] start = 0 selfnode = w_self._node bynode = w_by._node bylen = bynode.length() if bylen == 0: raise OperationError(space.w_ValueError, space.wrap("empty separator")) - - iter = rope.FindIterator(selfnode, bynode) - while maxsplit != 0: - try: - next = iter.next() - except StopIteration: - break - res_w.append(W_RopeObject(rope.getslice_one(selfnode, start, next))) - start = next + bylen - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - res_w.append(W_RopeObject(rope.getslice_one( - selfnode, start, selfnode.length()))) + res_w = [W_RopeObject(node) + for node in rope.split(selfnode, bynode, maxsplit)] return space.newlist(res_w) def str_rsplit__Rope_None_ANY(space, w_self, w_none, w_maxsplit=-1): @@ -537,22 +519,9 @@ except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) - startidx = 0 - substrings = [] - iter = rope.FindIterator(node, sub) - try: - foundidx = iter.next() - except StopIteration: + substrings = rope.split(node, sub, maxsplit) + if substrings is None: return w_self.create_if_subclassed() - while maxsplit != 0: - substrings.append(rope.getslice_one(node, startidx, foundidx)) - startidx = foundidx + sub.length() - try: - foundidx = iter.next() - except StopIteration: - break - maxsplit = maxsplit - 1 - substrings.append(rope.getslice_one(node, startidx, length)) try: return W_RopeObject(rope.join(by, substrings)) except OverflowError: @@ -581,47 +550,24 @@ return W_RopeObject(rope.getslice_one(node, lpos, rpos)) -def _strip_none(space, w_self, left, right): - "internal function called by str_xstrip methods" - node = w_self._node - length = node.length() - - lpos = 0 - rpos = length - - if left: - #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.ItemIterator(node) - while lpos < rpos and iter.nextchar().isspace(): - lpos += 1 - - if right: - iter = rope.ReverseItemIterator(node) - while rpos > lpos and iter.nextchar().isspace(): - rpos -= 1 - - assert rpos >= lpos # annotator hint, don't remove - return W_RopeObject(rope.getslice_one(node, lpos, rpos)) def str_strip__Rope_Rope(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=1, right=1) def str_strip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=1) + return W_RopeObject(rope.strip(w_self._node, left=True, right=True)) def str_rstrip__Rope_Rope(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=0, right=1) def str_rstrip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=0, right=1) + return W_RopeObject(rope.strip(w_self._node, left=False, right=True)) - def str_lstrip__Rope_Rope(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=1, right=0) def str_lstrip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=0) - + return W_RopeObject(rope.strip(w_self._node, left=True, right=False)) def str_center__Rope_ANY_ANY(space, w_self, w_arg, w_fillchar): @@ -899,13 +845,6 @@ if mul <= 0: return W_RopeObject.EMPTY node = w_str._node - length = node.length() -# try: -# buflen = ovfcheck(mul * length) -# except OverflowError: -# raise OperationError( -# space.w_OverflowError, -# space.wrap("repeated string is too long: %d %d" % (length, mul))) try: return W_RopeObject(rope.multiply(node, mul)) except OverflowError: Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Thu Nov 15 23:55:32 2007 @@ -695,3 +695,22 @@ assert isinstance(r, LiteralNode) assert r.getint(0) >= 128 or isinstance(r, LiteralStringNode) assert r.getrope(0) is r + +def test_split(): + seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), + (LiteralStringNode("d"), "d"), (LiteralStringNode(""), "")] + l, strs = zip(*[(LiteralStringNode("x"), "x"), + (LiteralStringNode("xyz"), "xyz"), + (LiteralStringNode("w"), "w")]) + l = list(l) + for s, st in seps: + node = join(s, l) + l2 = split(node, s) + for n1, n2 in zip(l, l2): + assert n1.flatten_string() == n2.flatten_string() + for i in range(4): + l1 = split(LiteralStringNode("ababababa"), LiteralStringNode("b"), i) + l2 = "ababababa".split("b", i) + assert len(l1) == len(l2) + for n, s in zip(l1, l2): + assert n.flatten_string() == s From cfbolz at codespeak.net Fri Nov 16 00:13:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 16 Nov 2007 00:13:20 +0100 (CET) Subject: [pypy-svn] r48722 - pypy/branch/ropes-unicode/pypy/objspace/std Message-ID: <20071115231320.9FE2482FB@code0.codespeak.net> Author: cfbolz Date: Fri Nov 16 00:13:20 2007 New Revision: 48722 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Log: use split for expandtabs too Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Fri Nov 16 00:13:20 2007 @@ -532,7 +532,7 @@ "internal function called by str_xstrip methods" node = w_self._node length = node.length() - u_chars = space.str_w(w_chars) + u_chars = w_chars._node.flatten_string() lpos = 0 rpos = length @@ -625,7 +625,6 @@ return space.w_False - def str_startswith__Rope_Rope_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (self, prefix, start, end) = _convert_idx_params(space, w_self, w_prefix, w_start, w_end) @@ -675,39 +674,18 @@ return W_RopeObject.EMPTY tabsize = space.int_w(w_tabsize) - expanded = [] - iter = rope.FindIterator(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) - #split = u_self.split("\t") - #u_expanded = oldtoken = split.pop(0) - - #for token in split: - # u_expanded += " " * _tabindent(oldtoken,u_tabsize) + token - # oldtoken = token - start = 0 - try: - start = iter.next() - last = rope.getslice_one(node, 0, start) - start += 1 - except StopIteration: - return w_self.create_if_subclassed() - expanded.append(last) - while 1: + splitted = rope.split(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) + last = splitted[0] + expanded = [last] + for i in range(1, len(splitted)): expanded.append(rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], _tabindent(last, tabsize))) - try: - next = iter.next() - except StopIteration: - break - last = rope.getslice_one(node, start, next) + last = splitted[i] expanded.append(last) - start = next + 1 - - expanded.append(rope.getslice_one(node, start, length)) return W_RopeObject(rope.rebalance(expanded)) def str_splitlines__Rope_ANY(space, w_self, w_keepends): - #import pdb; pdb.set_trace() keepends = bool(space.int_w(w_keepends)) # truth value, but type checked node = w_self._node length = node.length() From cfbolz at codespeak.net Fri Nov 16 00:28:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 16 Nov 2007 00:28:41 +0100 (CET) Subject: [pypy-svn] r48723 - pypy/branch/ropes-unicode/pypy/objspace/std/test Message-ID: <20071115232841.7A84382E1@code0.codespeak.net> Author: cfbolz Date: Fri Nov 16 00:28:38 2007 New Revision: 48723 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: a failing unicode test Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Fri Nov 16 00:28:38 2007 @@ -305,6 +305,16 @@ assert not u'hello'.startswith((u'he', u'hel'), 0, 1) assert u'hello'.startswith((u'he', u'hel'), 0, 2) raises(TypeError, u'hello'.startswith, (42,)) + + def test_startswith_endswith_convert(self): + py.test.skip("fix me") + assert 'hello'.startswith((u'he\u1111', u'he')) + assert not 'hello'.startswith((u'lo\u1111', u'llo')) + assert 'hello'.startswith((u'hellox\u1111', u'hello')) + assert 'hello'.startswith((u'lo', u'he\u1111'), 0, -1) + assert not 'hello'.endswith((u'he\u1111', u'he')) + assert 'hello'.endswith((u'\u1111lo', u'llo')) + assert 'hello'.endswith((u'\u1111hellox', u'hello')) def test_endswith(self): assert u'ab'.endswith(u'ab') is True From cfbolz at codespeak.net Fri Nov 16 00:38:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 16 Nov 2007 00:38:15 +0100 (CET) Subject: [pypy-svn] r48724 - pypy/branch/ropes-unicode/pypy/objspace/std/test Message-ID: <20071115233815.866438303@code0.codespeak.net> Author: cfbolz Date: Fri Nov 16 00:38:15 2007 New Revision: 48724 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: hm, applevel skips are different Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Fri Nov 16 00:38:15 2007 @@ -307,7 +307,7 @@ raises(TypeError, u'hello'.startswith, (42,)) def test_startswith_endswith_convert(self): - py.test.skip("fix me") + skip("fix me") assert 'hello'.startswith((u'he\u1111', u'he')) assert not 'hello'.startswith((u'lo\u1111', u'llo')) assert 'hello'.startswith((u'hellox\u1111', u'hello')) From cfbolz at codespeak.net Fri Nov 16 00:38:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 16 Nov 2007 00:38:34 +0100 (CET) Subject: [pypy-svn] r48725 - pypy/branch/ropes-unicode/pypy/objspace/std Message-ID: <20071115233834.2CBC68317@code0.codespeak.net> Author: cfbolz Date: Fri Nov 16 00:38:33 2007 New Revision: 48725 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Log: share some code with normal strings Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Fri Nov 16 00:38:33 2007 @@ -11,7 +11,8 @@ from pypy.objspace.std.stringtype import wrapchar from pypy.objspace.std import rope -from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY +from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY,\ + _upper, _lower class W_RopeObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef @@ -66,20 +67,6 @@ return space.w_True _is_generic._annspecialcase_ = "specialize:arg(2)" -def _upper(ch): - if ch.islower(): - o = ord(ch) - 32 - return chr(o) - else: - return ch - -def _lower(ch): - if ch.isupper(): - o = ord(ch) + 32 - return chr(o) - else: - return ch - _isspace = lambda c: c.isspace() _isdigit = lambda c: c.isdigit() _isalpha = lambda c: c.isalpha() From antocuni at codespeak.net Fri Nov 16 13:34:49 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 16 Nov 2007 13:34:49 +0100 (CET) Subject: [pypy-svn] r48726 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071116123449.8C90F8236@code0.codespeak.net> Author: antocuni Date: Fri Nov 16 13:34:47 2007 New Revision: 48726 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: I finally managed to organize my travel :-) Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Fri Nov 16 13:34:47 2007 @@ -18,6 +18,7 @@ Armin Rigo lives thereish Alexander Schremmer 19-26 Laura and Jacob's Richard Emslie 19-26 Laura and Jacob's +Antonio Cuni 19-22 Laura and Jacob's ==================== ============== ===================== @@ -27,7 +28,6 @@ Name Arrive/Depart Accomodation ==================== ============== ===================== Michael Hudson ? ? -Antonio Cuni ? ? Andrew Thompson ? ? Stephan Diehl ? ? Eric van Riet paap ? ? From fijal at codespeak.net Fri Nov 16 14:14:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Nov 2007 14:14:26 +0100 (CET) Subject: [pypy-svn] r48727 - pypy/extradoc/talk/pycon2008 Message-ID: <20071116131426.146A38244@code0.codespeak.net> Author: fijal Date: Fri Nov 16 14:14:25 2007 New Revision: 48727 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Some update to tutorial Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Fri Nov 16 14:14:25 2007 @@ -3,7 +3,11 @@ XXX We need a sexier title. -* Presenter name(s) & contact information (email, telephone). +Presenter name(s) & contact information +======================================= + +Laura Creighton +Maciej Fijalkowski Intended audience ================= @@ -37,7 +41,7 @@ ============ Attendees are welcome to bring they're own laptops, while one for a group -should suffice. We would like to know create codespeak.net accounts for people +should suffice. We would like to create codespeak.net accounts for people in advance, in order to provide smooth working on branches. Software requirements: @@ -99,17 +103,19 @@ Examples ++++++++ -We would like to present one or two examples during the talk and +We would like to present one or two examples of live-coding and changing +the virtual machine during the talk and offer some tasks for people to experiment with. The exact details about -the examples will be prepared later, especially because given current -pypy developement pace, a lot of things might be implemented till march. +the examples will be prepared later, especially that given current +pypy developement pace, a lot of things might be implemented before March. -One of the following might be especially good for the purpose of talk: +One of the following might be especially good for the purpose of the tutorial: * (Re)implement the thunk objspace, the way to "freeze" computation as long as they're not needed. -* Implement specialized sort of tuples, like pairs, indistinguishable from +* Implement specialized sort of tuples, optimized for small number + of elements (for example two), indistinguishable from normal tuples from the application level code. * Implement dict.keys() as a lazily computed list. @@ -118,19 +124,27 @@ ++++++++++++++++++++++++ The hands-on session would be organized in groups of 2-5 persons, hacking -one particular task, preferable on one computer with assistance of someone +one particular task, preferable on one computer with the assistance of someone familiar with the pypy codebase. If there would be too many people, people familiar with pypy would need to move between groups. We'll prepare some tasks -(like mentioned above), people are encouraged to suggest they're own tasks, +(similiar to mentioned above), although people are encouraged to suggest +their own tasks, preferably in advance so we can tell whether it's feasible to try that -in 1.5h. Thanks to pypy design, we don't expect anyone to be familiar +in 1.5h. Thanks to the pypy design, we don't expect anyone to be familiar with coding in RPython, hence it would be only prototyping, with eventual followup during the sprint time. +Tutorial outline for website publication +======================================== + +XXX fill in +Presenter bio +============= -* Tutorial outline for website publication, if different from the above. +XXX fill in -* Presenter bio (one paragraph), written in the third person ("Presenter Name is..."). Include where you live, your job, your projects, books you've written, volunteer activities, family, etc. +Previous experience +=================== -* Previous experience: references of presentation/teaching engagements (sessions taught at OSCON, EuroPython, local user groups, at work, etc.). Extra consideration will be given to presenters with prior experience. +Europython 2007, Ruby 2007, sprints XXX laura, you insert yours here From fijal at codespeak.net Fri Nov 16 19:21:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Nov 2007 19:21:12 +0100 (CET) Subject: [pypy-svn] r48729 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071116182112.864C78258@code0.codespeak.net> Author: fijal Date: Fri Nov 16 19:21:11 2007 New Revision: 48729 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: Add os.setuid/setgid Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Fri Nov 16 19:21:11 2007 @@ -80,6 +80,18 @@ return extdef([], int, llimpl=c_func_llimpl, export_name='ll_os.ll_os_' + name) + def extdef_for_function_int_to_int(self, name, **kwds): + c_func = self.llexternal(name, [rffi.INT], rffi.INT, **kwds) + def c_func_llimpl(arg): + res = rffi.cast(rffi.LONG, c_func(arg)) + if res == -1: + raise OSError(rffi.get_errno(), "%s failed" % name) + + c_func_llimpl.func_name = name + '_llimpl' + + return extdef([int], None, llimpl=c_func_llimpl, + export_name='ll_os.ll_os_' + name) + @registering_if(os, 'execv') def register_os_execv(self): os_execv = self.llexternal('execv', [rffi.CCHARP, rffi.CCHARPP], @@ -344,6 +356,14 @@ def register_os_geteuid(self): return self.extdef_for_os_function_returning_int('geteuid') + @registering_if(os, 'setuid') + def register_os_setuid(self): + return self.extdef_for_function_int_to_int('setuid') + + @registering_if(os, 'setgid') + def register_os_setgid(self): + return self.extdef_for_function_int_to_int('setgid') + @registering_if(os, 'getpid') def register_os_getpid(self): return self.extdef_for_os_function_returning_int('getpid') Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Fri Nov 16 19:21:11 2007 @@ -134,6 +134,13 @@ res = self.interpret(fun, [value]) assert res == fun(value) + if hasattr(os, 'setuid'): + def test_os_setuid(self): + def f(): + os.setuid(os.getuid()) + return os.getuid() + assert self.interpret(f, []) == f() + class TestLLtype(BaseTestPosix, LLRtypeMixin): pass From fijal at codespeak.net Fri Nov 16 20:04:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Nov 2007 20:04:28 +0100 (CET) Subject: [pypy-svn] r48730 - pypy/dist/pypy/module/posix Message-ID: <20071116190428.CECB5816B@code0.codespeak.net> Author: fijal Date: Fri Nov 16 20:04:27 2007 New Revision: 48730 Modified: pypy/dist/pypy/module/posix/interp_posix.py Log: Add os.setuid/setgid. I've got completely no idea how to test it though. Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Fri Nov 16 20:04:27 2007 @@ -552,6 +552,24 @@ return space.wrap(os.getuid()) getuid.unwrap_spec = [ObjSpace] +def setuid(space, arg): + """ setuid(uid) + + Set the current process's user id. + """ + os.setuid(arg) + return space.w_None +setuid.unwrap_spec = [ObjSpace, int] + +def setgid(space, arg): + """ setgid(gid) + + Set the current process's group id. + """ + os.setgid(arg) + return space.w_None +setgid.unwrap_spec = [ObjSpace, int] + def getgid(space): """ getgid() -> gid From rxe at codespeak.net Fri Nov 16 22:31:33 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 16 Nov 2007 22:31:33 +0100 (CET) Subject: [pypy-svn] r48733 - pypy/dist/pypy/rpython/module Message-ID: <20071116213133.D44C5819B@code0.codespeak.net> Author: rxe Date: Fri Nov 16 22:31:33 2007 New Revision: 48733 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: eventually figured oot memory corruption issues with os.stat and friends on llvm backend. STRUCT_STAT is completely wrong since CConfig was not being called, and llvm backend actually uses STRUCT_STAT in malloc and call (c backend doesnt). not sure what this would break by doing it by default so is conditional for someone else to fix :-) Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Fri Nov 16 22:31:33 2007 @@ -142,6 +142,19 @@ else: _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] + +# XXX need someone to look this over... +hack_to_run_cconfig = False +if hack_to_run_cconfig: + from pypy.rpython.module.ll_os import CConfig + from pypy.rpython.tool import rffi_platform as platform + class CConfig: + _includes_ = INCLUDES + LL_STRUCT = platform.Struct('struct stat', LL_STAT_FIELDS) + config = platform.configure(CConfig) + LL_STRUCT = config['LL_STRUCT'] + LL_STAT_FIELDS = [(xname[2:], LL_STRUCT._flds[xname]) for xname in LL_STRUCT._names_without_voids()] + STRUCT_STAT = rffi.CStructPtr(_name_struct_stat, *LL_STAT_FIELDS) From rxe at codespeak.net Sat Nov 17 02:34:32 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 02:34:32 +0100 (CET) Subject: [pypy-svn] r48737 - pypy/dist/pypy/translator/c/test Message-ID: <20071117013432.423168159@code0.codespeak.net> Author: rxe Date: Sat Nov 17 02:34:30 2007 New Revision: 48737 Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py Log: clean up dead code Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/c/test/test_extfunc.py Sat Nov 17 02:34:30 2007 @@ -4,19 +4,8 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong from pypy.translator.c.test.test_genc import compile -from pypy.translator.c.extfunc import EXTERNALS posix = __import__(os.name) -def test_all_suggested_primitives(): - for modulename in ['ll_os', 'll_os_path', 'll_time']: - mod = __import__('pypy.rpython.module.%s' % modulename, - None, None, ['__doc__']) - for func in mod.__dict__.values(): - if getattr(func, 'suggested_primitive', False): - yield suggested_primitive_implemented, func -def suggested_primitive_implemented(func): - assert func in EXTERNALS, "missing C implementation for %r" % (func,) - # note: clock synchronizes itself! def test_time_clock(): def does_stuff(): From rxe at codespeak.net Sat Nov 17 02:44:58 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 02:44:58 +0100 (CET) Subject: [pypy-svn] r48738 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071117014458.6A6CC8186@code0.codespeak.net> Author: rxe Date: Sat Nov 17 02:44:58 2007 New Revision: 48738 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/module/genexterns.c Log: cleanup from more externs to rffi Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sat Nov 17 02:44:58 2007 @@ -30,16 +30,6 @@ ll_stack.ll_stack_too_big: 'LL_stack_too_big_', } - -math_functions = [ - 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', - 'floor', 'log', 'log10', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', - 'pow', 'atan2', 'fmod', 'ldexp', 'hypot' - ] -import math -for name in math_functions: - EXTERNALS['ll_math.ll_math_%s' % name] = 'LL_math_%s' % name - def predeclare_stuff(c_db): modules = {} def module_name(c_name): @@ -266,15 +256,15 @@ # ask gcpolicy for any code needed ccode.append('%s\n' % db.gcpolicy.genextern_code()) - - ccode.append('\n') + for c_include in c_includes: ccode.append('#include <%s>\n' % c_include) - + for c_source in c_sources: - for l in c_source: - ccode.append(l) ccode.append('\n') + for l in c_source: + ccode.append(l + '\n') + ccode.append('\n') # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Sat Nov 17 02:44:58 2007 @@ -14,17 +14,6 @@ abort(); \ } -#ifdef LL_NEED_MATH - FAKE_ERROR(OverflowError); - FAKE_ERROR(ValueError); - #include "c/src/ll_math.h" - -#endif - -#ifdef LL_NEED_STRTOD - #include "c/src/ll_strtod.h" -#endif - #ifdef LL_NEED_STACK FAKE_ERROR(RuntimeError); #include "c/src/thread.h" From rxe at codespeak.net Sat Nov 17 02:45:34 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 02:45:34 +0100 (CET) Subject: [pypy-svn] r48739 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071117014534.93EA88192@code0.codespeak.net> Author: rxe Date: Sat Nov 17 02:45:34 2007 New Revision: 48739 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py pypy/dist/pypy/translator/llvm/test/test_rffi.py Log: all rffi externs are passing now Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Sat Nov 17 02:45:34 2007 @@ -13,10 +13,10 @@ FLOAT_PRECISION = 8 # prevents resource leaking -use_isolate = True +use_isolate = False # if test can't be run using isolate, skip the test (useful for buildbots) -run_isolated_only = True +run_isolated_only = False from pypy import conftest @@ -40,7 +40,32 @@ else: del _ext_modules[:] +def setup_module(mod): + from pypy.rpython.lltypesystem import lltype + from pypy.rpython.lltypesystem.rffi import llexternal + + c_source = py.code.Source(""" + int get_errno() { + return errno; + } + """) + get_errno = llexternal('get_errno', [], lltype.Signed, sources=[c_source]) + + c_source = py.code.Source(""" + void set_errno(int _errno) { + errno = _errno; + } + """) + set_errno = llexternal('set_errno', [lltype.Signed], lltype.Void, sources=[c_source]) + global _get_errno, _set_errno + import pypy.rpython.lltypesystem.rffi + pypy.rpython.lltypesystem.rffi.get_errno, _get_errno = get_errno, pypy.rpython.lltypesystem.rffi.get_errno + pypy.rpython.lltypesystem.rffi.set_errno, _set_errno= set_errno, pypy.rpython.lltypesystem.rffi.set_errno + def teardown_module(mod): + import pypy.rpython.lltypesystem.rffi + pypy.rpython.lltypesystem.rffi.get_errno = _get_errno + pypy.rpython.lltypesystem.rffi.set_errno = _set_errno _cleanup() def llvm_test(): @@ -189,7 +214,7 @@ py.test.skip('PowerPC --> %s' % reason) def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): - pass + py.test.skip("skip_llinterpreter") def interpret(self, fn, args, annotation=None): fn = self._compile(fn, args, annotation) @@ -204,6 +229,7 @@ self.interpret(fn, args) except ExceptionWrapper, ex: assert issubclass(eval(ex.class_name), exception) + return True else: assert False, 'function did raise no exception at all' Modified: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Sat Nov 17 02:45:34 2007 @@ -1,7 +1,9 @@ import py from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin from pypy.translator.llvm.test.runtest import * -py.test.skip("skip these until we have resolved most external functions issues") class TestLLVMBuiltin(LLVMTest, BaseTestRbuiltin): - pass + def _skip_llinterpreter(self): + LLVMTest._skip_llinterpreter(self) + test_os_open = _skip_llinterpreter + test_debug_llinterpcall = _skip_llinterpreter Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rffi.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Sat Nov 17 02:45:34 2007 @@ -9,47 +9,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.test.runtest import * - -monkey_patch = False -if monkey_patch: - from pypy.rpython.lltypesystem.rffi import llexternal - c_source = py.code.Source(""" - int get_errno() { - return errno; - } - """) - get_errno = llexternal('get_errno', [], lltype.Signed, sources=[c_source]) - - c_source = py.code.Source(""" - void set_errno(int _errno) { - errno = _errno; - } - """) - set_errno = llexternal('set_errno', [lltype.Signed], lltype.Void, sources=[c_source]) - import pypy.rpython.lltypesystem.rffi - pypy.rpython.lltypesystem.rffi.get_errno = get_errno - pypy.rpython.lltypesystem.rffi.set_errno = set_errno -else: - py.test.skip("rffi not there yet for llvm") - -from pypy.rpython.lltypesystem.rffi import * - -def test_basic(): - c_source = py.code.Source(""" - int z(int x) - { - return (x + 3); - } - """) - z = llexternal('z', [Signed], Signed, sources=[c_source]) - - def f(): - return z(8) - - xf = compile_function(f, []) - assert xf() == 8+3 - -#py.test.skip("Extfunc support in llvm needs refactoring (!!??!)" +from pypy.rpython.lltypesystem import rffi def test_external_function_ll_os_dup(): def fn(): From rxe at codespeak.net Sat Nov 17 02:46:49 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 02:46:49 +0100 (CET) Subject: [pypy-svn] r48740 - pypy/dist/pypy/rpython/lltypesystem/module/test Message-ID: <20071117014649.0260681A1@code0.codespeak.net> Author: rxe Date: Sat Nov 17 02:46:49 2007 New Revision: 48740 Modified: pypy/dist/pypy/rpython/lltypesystem/module/test/ (props changed) Log: fixeol From rxe at codespeak.net Sat Nov 17 02:48:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 02:48:03 +0100 (CET) Subject: [pypy-svn] r48741 - pypy/dist/pypy/rpython/module Message-ID: <20071117014803.1ECDA819F@code0.codespeak.net> Author: rxe Date: Sat Nov 17 02:48:02 2007 New Revision: 48741 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: enable CConfig always - no tests seem to be failing Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Sat Nov 17 02:48:02 2007 @@ -143,20 +143,18 @@ _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] -# XXX need someone to look this over... -hack_to_run_cconfig = False -if hack_to_run_cconfig: - from pypy.rpython.module.ll_os import CConfig - from pypy.rpython.tool import rffi_platform as platform - class CConfig: - _includes_ = INCLUDES - LL_STRUCT = platform.Struct('struct stat', LL_STAT_FIELDS) - config = platform.configure(CConfig) - LL_STRUCT = config['LL_STRUCT'] - LL_STAT_FIELDS = [(xname[2:], LL_STRUCT._flds[xname]) for xname in LL_STRUCT._names_without_voids()] +# XXX need someone to look this over please... -STRUCT_STAT = rffi.CStructPtr(_name_struct_stat, *LL_STAT_FIELDS) +from pypy.rpython.module.ll_os import CConfig +from pypy.rpython.tool import rffi_platform as platform +class CConfig: + _includes_ = INCLUDES + STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) +config = platform.configure(CConfig) +STAT_STRUCT = config['STAT_STRUCT'] +_LL_STAT_FIELDS = [(n[2:], STAT_STRUCT._flds[n]) for n in STAT_STRUCT._names] +STRUCT_STAT = rffi.CStructPtr(_name_struct_stat, *_LL_STAT_FIELDS) def build_stat_result(st): # only for LL backends From cfbolz at codespeak.net Sat Nov 17 10:57:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Nov 2007 10:57:12 +0100 (CET) Subject: [pypy-svn] r48742 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071117095712.18D998150@code0.codespeak.net> Author: cfbolz Date: Sat Nov 17 10:57:11 2007 New Revision: 48742 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Log: specify my dates a bit more Modified: pypy/extradoc/sprintinfo/gothenburg-2007/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/people.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/people.txt Sat Nov 17 10:57:11 2007 @@ -8,7 +8,7 @@ ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== -Carl Friedrich Bolz ? ? +Carl Friedrich Bolz 19-25 Samuele's and Armin's Samuele Pedroni lives there Anders Chrigstrom lives there Jacob Hallen lives there From cfbolz at codespeak.net Sat Nov 17 11:11:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Nov 2007 11:11:15 +0100 (CET) Subject: [pypy-svn] r48743 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071117101115.E04B98161@code0.codespeak.net> Author: cfbolz Date: Sat Nov 17 11:11:15 2007 New Revision: 48743 Added: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (contents, props changed) Log: add a planning file, to not change the announcement all the time when adding/removing tasks Added: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sat Nov 17 11:11:15 2007 @@ -0,0 +1,59 @@ +Topics and goals +---------------- + +Meta-Tasks +++++++++++ + + - Tracker-gardening + - Start slowly thinking about 1.1 plans + +Translation toolchain ++++++++++++++++++++++ + + + - finish rctypes removal + + - move away from implicit keepalives + - raw_malloc should be split, rffi one should not use obmalloc (it's + not thread-safe) + - delegate finding type stuff like vtables etc to GC, cleaner interface + for rtti, simplify translator/c/gc.py + + - clean up the tangle of including headers in the C backend + + - review pdbplus, especially the graph commands, also in the light of + https://codespeak.net/issue/pypy-dev/issue303 and the fact that we + can have more than one translator/annotator around (with the + timeshifter) + + - kill half concrete wrapper + + - kill opaque hacks on the C backend + + - decide how to implement constants in rffi + + - think about/remove orphaned parts + + - implement callback mechanism for rffi/lltype/ll2ctypes + +Interpreter ++++++++++++ + + - there's a shadow tracking bug it seems + + - fix the taint space + + - review the things implemented at applevel whether they are + performance-critical + + - app-level ctypes + + - remove prebuilt costate vars (might happen before) + +JIT ++++ + + - repair the JIT tests (if it doesn't happen before), the keep alive killing merge broke them + - start writing real unit tests for the merging logic + + From cfbolz at codespeak.net Sat Nov 17 11:12:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Nov 2007 11:12:03 +0100 (CET) Subject: [pypy-svn] r48744 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071117101203.448D28161@code0.codespeak.net> Author: cfbolz Date: Sat Nov 17 11:12:02 2007 New Revision: 48744 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: a small task for Maciek Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sat Nov 17 11:12:02 2007 @@ -36,6 +36,8 @@ - implement callback mechanism for rffi/lltype/ll2ctypes + - find out why our memcpy behaves so erratically + Interpreter +++++++++++ From antocuni at codespeak.net Sat Nov 17 11:15:06 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 17 Nov 2007 11:15:06 +0100 (CET) Subject: [pypy-svn] r48745 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071117101506.0D512815E@code0.codespeak.net> Author: antocuni Date: Sat Nov 17 11:15:03 2007 New Revision: 48745 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: a new task for the sprint Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sat Nov 17 11:15:03 2007 @@ -38,6 +38,8 @@ - find out why our memcpy behaves so erratically + - refactor OO external objects + Interpreter +++++++++++ From cfbolz at codespeak.net Sat Nov 17 11:25:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Nov 2007 11:25:34 +0100 (CET) Subject: [pypy-svn] r48746 - pypy/dist/pypy/rlib/test Message-ID: <20071117102534.C9CAB8162@code0.codespeak.net> Author: cfbolz Date: Sat Nov 17 11:25:34 2007 New Revision: 48746 Modified: pypy/dist/pypy/rlib/test/test_runicode.py Log: fix wrong test Modified: pypy/dist/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_runicode.py (original) +++ pypy/dist/pypy/rlib/test/test_runicode.py Sat Nov 17 11:25:34 2007 @@ -187,8 +187,9 @@ def test_utf8(self): from pypy.rpython.test.test_llinterp import interpret def f(x): - s1 = "\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93" * x - u = runicode.str_decode_utf_8(s1, len(s1), True) + + s1 = "".join(["\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93"] * x) + u, consumed = runicode.str_decode_utf_8(s1, len(s1), True) s2 = runicode.unicode_encode_utf_8(u, len(u), True) return s1 == s2 res = interpret(f, [2]) From antocuni at codespeak.net Sat Nov 17 11:51:10 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 17 Nov 2007 11:51:10 +0100 (CET) Subject: [pypy-svn] r48747 - pypy/dist/pypy/translator/goal Message-ID: <20071117105110.29E898178@code0.codespeak.net> Author: antocuni Date: Sat Nov 17 11:51:09 2007 New Revision: 48747 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: rename also the .jar file when using the jvm backend Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Sat Nov 17 11:51:09 2007 @@ -141,6 +141,10 @@ basename_dir = basename + '-data' realname_dir = realname + '-data' run("mv %s %s" % (basename_dir, realname_dir)) + elif backend == 'jvm': + basename_jar = basename + '.jar' + realname_jar = realname + '.jar' + run("mv %s %s" % (basename_jar, realname_jar)) #pypy = open(basename, 'rb').read() #if len(pypy) > 0: # open(realname, 'wb').write(pypy) From fijal at codespeak.net Sat Nov 17 12:00:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Nov 2007 12:00:21 +0100 (CET) Subject: [pypy-svn] r48748 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071117110021.A47918117@code0.codespeak.net> Author: fijal Date: Sat Nov 17 12:00:21 2007 New Revision: 48748 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: Shuffle tasks a bit Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sat Nov 17 12:00:21 2007 @@ -5,17 +5,19 @@ ++++++++++ - Tracker-gardening + - Review the cpython regression test suite (skips/failures) - Start slowly thinking about 1.1 plans Translation toolchain +++++++++++++++++++++ - - finish rctypes removal - move away from implicit keepalives + - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) + - delegate finding type stuff like vtables etc to GC, cleaner interface for rtti, simplify translator/c/gc.py @@ -50,14 +52,19 @@ - review the things implemented at applevel whether they are performance-critical - - app-level ctypes - - remove prebuilt costate vars (might happen before) JIT +++ - repair the JIT tests (if it doesn't happen before), the keep alive killing merge broke them + - start writing real unit tests for the merging logic +Discussion issues ++++++++++++++++++ + + - app-level ctypes + + - separate compilation From lac at codespeak.net Sat Nov 17 17:39:30 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sat, 17 Nov 2007 17:39:30 +0100 (CET) Subject: [pypy-svn] r48761 - pypy/extradoc/talk/pycon2008 Message-ID: <20071117163930.9C69D813B@code0.codespeak.net> Author: lac Date: Sat Nov 17 17:39:28 2007 New Revision: 48761 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: try again, link went down Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Sat Nov 17 17:39:28 2007 @@ -73,9 +73,9 @@ This tutorial will give you detailed description how the PyPy python interpreter is constructed and how to change it to meet your needs. -It'll also contain simple changes to the interpreter codebase as live coding -session. Afterwards will be welcomed to experiment in groups on they're own -on implementing prepared interpreter changes. +It'll also contain simple changes to the interpreter codebase in a live coding +session. Afterwards, we will work in small groups to experiment on +attendee-selected changes to the interpreter. Attendees are welcome to bring their own laptops. @@ -88,7 +88,7 @@ * RPython - the restricted python - * Objectspace concept + * The Objectspace concept * Backends - how we can compile to all platforms @@ -106,8 +106,8 @@ We would like to present one or two examples of live-coding and changing the virtual machine during the talk and offer some tasks for people to experiment with. The exact details about -the examples will be prepared later, especially that given current -pypy developement pace, a lot of things might be implemented before March. +the examples will be prepared later, when we get a sense of the interests +of the attendees. One of the following might be especially good for the purpose of the tutorial: @@ -123,16 +123,24 @@ Hands-on session details ++++++++++++++++++++++++ -The hands-on session would be organized in groups of 2-5 persons, hacking +The hands-on session would be organized in groups of 2-4 persons, hacking one particular task, preferable on one computer with the assistance of someone -familiar with the pypy codebase. If there would be too many people, people -familiar with pypy would need to move between groups. We'll prepare some tasks -(similiar to mentioned above), although people are encouraged to suggest -their own tasks, -preferably in advance so we can tell whether it's feasible to try that -in 1.5h. Thanks to the pypy design, we don't expect anyone to be familiar -with coding in RPython, hence it would be only prototyping, with eventual -followup during the sprint time. +familiar with the pypy codebase. If the tutorial attendees greatly +outnumber the pypy core team, then the people +familiar with pypy would need to move between groups. + +We'd like tutorial attendees to suggest their own improvements, so that +they get to scratch an itch that personally matters to them. We will +also provide a selected list of things we think would be interesting, +in the off-chance that people either find this sort of imagining +difficult, or can only think of tasks that would take more than +1.5 hours to complete. The idea is to give such people enough familiarity +with pypy that they could attempt larger projects on their own. + +Given that tutorial members are not expected to be proficient with RPython, +what is likely is that in the tutorial their solutions will only be +prototyped (in Python). Converting to RPython may have to wait for the +post-conference Sprint. Tutorial outline for website publication ======================================== @@ -147,4 +155,10 @@ Previous experience =================== -Europython 2007, Ruby 2007, sprints XXX laura, you insert yours here +Europython 2007, Ruby 2007, sprints + +Laura Creighton was a partner with Jim Joyce in Jim Joyce's Unix +bookstore/The Gawain Group. She has designed hundreds of hands-on +tutorials -- though she nearly always left them for Jim Joyce, or +one of our hired presenters to deliver. In the 15 years since +Jim's death, she has learned to make better presentations. From rxe at codespeak.net Sat Nov 17 23:53:21 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 23:53:21 +0100 (CET) Subject: [pypy-svn] r48763 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071117225321.EEA7B816B@code0.codespeak.net> Author: rxe Date: Sat Nov 17 23:53:21 2007 New Revision: 48763 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/opaquenode.py pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Log: sync up local changes before sprint. * clean ups from llvm2.0 transistion * add a debug string node, and a function for debug in codewriter * fix up some test mess Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Sat Nov 17 23:53:21 2007 @@ -1,6 +1,34 @@ from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.node import ConstantNode +class DebugStrNode(ConstantNode): + prefix = "@dg_isnt" + def __init__(self, value): + self.value = value + self.make_name() + + def get_length(self): + return len(self.value) + 1 + + def get_typerepr(self): + return '[%s x i8]' % self.get_length() + + def constantvalue(self): + return '%s c"%s\\00"' % (self.get_typerepr(), self.value) + + def get_childref(self, index): + x = "getelementptr(%s* %s, i32 0, i32 %s)" % ( + self.get_typerepr(), + self.name, + index) + #XXX probably why we are failing anyways + return 'bitcast(i8* %s to [0 x i8]*)' % x + + def writeglobalconstants(self, codewriter): + codewriter.globalinstance(self.ref, self.constantvalue()) + codewriter.newline() + codewriter.newline() + class ArrayNode(ConstantNode): __slots__ = "db value arraytype".split() prefix = '@a_inst' Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Sat Nov 17 23:53:21 2007 @@ -9,9 +9,9 @@ linkage = 'internal ' #/internal (disabled for now because of the JIT) def __init__(self, file, db, tail=None, cconv=None, linkage=None): + self.db = db self.file = file self.word_repr = db.get_machine_word() - self.uword_repr = db.get_machine_uword() if tail is not None: self.tail = tail if cconv is not None: @@ -48,7 +48,6 @@ def write_lines(self, lines, patch=False): for l in lines.split("\n"): if patch: - l = l.replace('UWORD', self.uword_repr) l = l.replace('WORD', self.word_repr) l = l.replace('POSTFIX', postfix()) l = l.replace('CC', self.cconv) @@ -88,11 +87,6 @@ cconv = self.cconv self._append("declare %s %s" %(cconv, decl,)) - def startimpl(self): - self.newline() - #self._append("implementation") - self.newline() - def br_uncond(self, blockname): self._indent("br label %%%s" %(blockname,)) @@ -210,3 +204,10 @@ def free(self, vartype, varref): self._indent("free %s %s" % (vartype, varref)) + + def debug_print(self, s): + var = self.db.repr_tmpvar() + node = self.db.create_debug_string(s) + self.call(var, "i32", "@write", + ['i32', '[0 x i8]*', 'i32'], + ['2', node.get_childref(0), '%d' % node.get_length()]) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat Nov 17 23:53:21 2007 @@ -11,7 +11,7 @@ from pypy.translator.llvm.structnode import StructNode, StructVarsizeNode, \ getindexhelper, FixedSizeArrayNode from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ - VoidArrayNode, ArrayNoLengthNode + VoidArrayNode, ArrayNoLengthNode, DebugStrNode from pypy.rpython.lltypesystem import lltype, llmemory from pypy.objspace.flow.model import Constant, Variable @@ -29,7 +29,6 @@ self.obj2node = {} self._pendingsetup = [] self._tmpcount = 1 - self.helper2ptr = {} self.primitives = Primitives(self) @@ -38,25 +37,34 @@ self.typedefnodes = [] self.containernodes = [] + self.debugstringnodes = [] + + #_______debuggging______________________________________ def dump_pbcs(self): r = "" for k, v in self.obj2node.iteritems(): + + if isinstance(v, FuncImplNode): + continue if isinstance(k, lltype.LowLevelType): continue assert isinstance(lltype.typeOf(k), lltype.ContainerType) + # Only dump top levels p, _ = lltype.parentlink(k) - if p is None: - ref = v.get_ref() - pbc_ref = v.get_ref() + ref = v.get_ref() + type_ = self.repr_type(lltype.Ptr(lltype.typeOf(k))) + pbc_ref = v.get_pbcref(type_) - r += "\ndump_pbcs %s (%s)\n" \ - "getref -> %s \n" \ - "pbcref -> %s \n" % (v, k, ref, pbc_ref) + r += "\ndump_pbcs %s (%s)\n" \ + "parent %s\n" \ + "type %s\n" \ + "getref -> %s \n" \ + "pbcref -> %s \n" % (v, k, p, type_, ref, pbc_ref) return r #_______setting up and preparation______________________________ @@ -244,11 +252,7 @@ return self.primitives.repr(arg.concretetype, arg.value) else: assert isinstance(arg.value, lltype._ptr) - if isinstance(arg.value._obj, int): - rt = self.repr_type(arg.concretetype) - v = repr(arg.value._obj) - return 'cast (int %s to %s)' % (v, rt) - elif not arg.value: + if not arg.value: return 'null' else: node = self.obj2node[arg.value._obj] @@ -297,9 +301,6 @@ if value is None: return None, "%s null" % toptr - if isinstance(value, int): - return None, '%s cast (int %s to %s)' % (toptr, value, toptr) - node = self.obj2node[value] ref = node.get_pbcref(toptr) return node, "%s %s" % (toptr, ref) @@ -326,9 +327,6 @@ def get_machine_word(self): return self.primitives[lltype.Signed] - def get_machine_uword(self): - return self.primitives[lltype.Unsigned] - def is_function_ptr(self, arg): if isinstance(arg, (Constant, Variable)): arg = arg.concretetype @@ -341,6 +339,10 @@ node = self.obj2node[parent] return node.get_childref(child) + def create_debug_string(self, s): + r = DebugStrNode(s) + self.debugstringnodes.append(r) + return r class Primitives(object): def __init__(self, database): @@ -355,7 +357,7 @@ lltype.UnsignedLongLong: "i64", lltype.SignedLongLong: "i64", llmemory.Address: "i8*", - #llmemory.WeakGcAddress: "sbyte*", + #XXX llmemory.WeakGcAddress: "i8*", } # 32 bit platform @@ -462,7 +464,6 @@ def repr_singlefloat(self, type_, value): from pypy.rlib.rarithmetic import isinf, isnan - # XXX this doesnt work on 1.9 -> only hex repr supported for single floats f = float(value) if isinf(f) or isnan(f): import struct @@ -557,7 +558,6 @@ def get_offset(self, value, initialindices=None): " return (from_type, (indices, ...), to_type) " word = self.database.get_machine_word() - uword = self.database.get_machine_uword() indices = initialindices or [(word, 0)] if isinstance(value, llmemory.ItemOffset): @@ -572,7 +572,7 @@ # jumps to a field position in a struct from_ = value.TYPE pos = getindexhelper(value.fldname, value.TYPE) - indices.append((uword, pos)) + indices.append((word, pos)) to = getattr(value.TYPE, value.fldname) elif isinstance(value, llmemory.ArrayLengthOffset): @@ -580,14 +580,14 @@ from_ = value.TYPE # or assert isinstance(value.TYPE, lltype.Array) if not value.TYPE._hints.get("nolength", False): - indices.append((uword, 0)) + indices.append((word, 0)) to = lltype.Signed elif isinstance(value, llmemory.ArrayItemsOffset): # jumps to the beginning of array area from_ = value.TYPE if not isinstance(value.TYPE, lltype.FixedSizeArray) and not value.TYPE._hints.get("nolength", False): - indices.append((uword, 1)) + indices.append((word, 1)) indices.append((word, 0)) # go to the 1st item to = value.TYPE.OF Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Sat Nov 17 23:53:21 2007 @@ -76,14 +76,14 @@ exc_flag=False): """ assumes malloc of word size """ XXX - uword = self.db.get_machine_uword() + word = self.db.get_machine_word() boundary_size = 0 # malloc_size is unsigned right now - codewriter.malloc(targetvar, "sbyte", size) + codewriter.malloc(targetvar, "i8", size) # XXX uses own cconv codewriter.call(None, 'void', '@llvm.memset' + postfix(), - ['sbyte*', 'ubyte', uword, uword], + ['i8*', 'i8', word, word], [targetvar, 0, size, boundary_size], cconv='ccc') @@ -107,10 +107,7 @@ exc_flag=False): """ assumes malloc of word size """ boundary_size = 0 - word = self.db.get_machine_word() - uword = self.db.get_machine_uword() - fnname = '@pypy_malloc' + (atomic and '_atomic' or '') ## XXX (arigo) disabled the ring buffer for comparison purposes @@ -121,9 +118,6 @@ ## # dont clear the ringbuffer data ## atomic = False - # malloc_size is unsigned right now - #sizeu = '%malloc_sizeu' + self.get_count() - #codewriter.cast(sizeu, word, size, uword) codewriter.call(targetvar, 'i8*', fnname, [word], [size]) if atomic: Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sat Nov 17 23:53:21 2007 @@ -17,6 +17,8 @@ from pypy.translator.llvm.log import log class GenLLVM(object): + debug = False + # see create_codewriter() below function_count = {} @@ -66,6 +68,8 @@ if hasattr(node, 'writeimpl'): node.writeimpl(codewriter) + self._debug() + codewriter.comment("End of file") codewriter.close() self._checkpoint('done') @@ -228,3 +232,13 @@ for s in stats: log('STATS %s' % str(s)) + def _debug(self): + if self.debug: + if self.db.debugstringnodes: + codewriter.header_comment("Debug string") + for node in self.db.debugstringnodes: + node.writeglobalconstants(codewriter) + + #print "Start" + #print self.db.dump_pbcs() + #print "End" Modified: pypy/dist/pypy/translator/llvm/opaquenode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opaquenode.py (original) +++ pypy/dist/pypy/translator/llvm/opaquenode.py Sat Nov 17 23:53:21 2007 @@ -43,23 +43,3 @@ def constantvalue(self): return "%s zeroinitializer" % self.db.repr_type(self.value._TYPE) - def writesetupcode(self, codewriter): - T = self.value._TYPE - # XXX similar non generic hacks to genc for now - if T.tag == 'ThreadLock': - XXX - argrefs = [self.get_ref()] - argtypes = [self.db.repr_type(T) + "*"] - lock = self.value.externalobj - argtypes.append("int") - if lock.locked(): - argrefs.append('1') - else: - argrefs.append('0') - - # XXX Check result - codewriter.call(self.db.repr_tmpvar(), - "sbyte*", - "%RPyOpaque_LLVM_SETUP_ThreadLock", - argtypes, argrefs) - # XXX Check result Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Sat Nov 17 23:53:21 2007 @@ -214,7 +214,8 @@ py.test.skip('PowerPC --> %s' % reason) def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): - py.test.skip("skip_llinterpreter") + if skipLL: + py.test.skip("skip_llinterpreter - skipLL=True") def interpret(self, fn, args, annotation=None): fn = self._compile(fn, args, annotation) Modified: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Sat Nov 17 23:53:21 2007 @@ -1,9 +1,12 @@ import py from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin + from pypy.translator.llvm.test.runtest import * class TestLLVMBuiltin(LLVMTest, BaseTestRbuiltin): - def _skip_llinterpreter(self): - LLVMTest._skip_llinterpreter(self) - test_os_open = _skip_llinterpreter - test_debug_llinterpcall = _skip_llinterpreter + def _skip(self): + py.test.skip("XXX specialize this") + + test_os_dup = _skip + test_os_open = _skip + test_debug_llinterpcall = _skip From rxe at codespeak.net Sat Nov 17 23:54:17 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 23:54:17 +0100 (CET) Subject: [pypy-svn] r48764 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071117225417.D2AB38190@code0.codespeak.net> Author: rxe Date: Sat Nov 17 23:54:17 2007 New Revision: 48764 Added: pypy/dist/pypy/translator/llvm/test/test_bigtest.py Log: add richards and pystoned as tests Added: pypy/dist/pypy/translator/llvm/test/test_bigtest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_bigtest.py Sat Nov 17 23:54:17 2007 @@ -0,0 +1,25 @@ +import py +from pypy.translator.llvm.test.runtest import * + +def test_richards(): + from pypy.translator.goal.richards import entry_point + entry_point = compile_function(entry_point, [int]) + + result, start, end = entry_point(1000) + assert result + assert end - start > 0 and end - start < 5.0 + +def test_rpystone(): + #py.test.skip("clock doesnt work") + + # XXX monkey patch hack XXX + from time import time + import pypy.translator.test.rpystone + pypy.translator.test.rpystone.clock = time + # XXX monkey patch hack XXX + + from pypy.translator.test.rpystone import pystones as entry_point + entry_point = compile_function(entry_point, [int]) + t, pystones = entry_point(50000) + assert 0 < t < 5 + assert 100000 < pystones < 100000000 From rxe at codespeak.net Sat Nov 17 23:57:52 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 17 Nov 2007 23:57:52 +0100 (CET) Subject: [pypy-svn] r48765 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071117225752.0BFCD8195@code0.codespeak.net> Author: rxe Date: Sat Nov 17 23:57:51 2007 New Revision: 48765 Modified: pypy/dist/pypy/translator/llvm/test/test_bigtest.py Log: remove rubbish just checked in Modified: pypy/dist/pypy/translator/llvm/test/test_bigtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_bigtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_bigtest.py Sat Nov 17 23:57:51 2007 @@ -10,14 +10,6 @@ assert end - start > 0 and end - start < 5.0 def test_rpystone(): - #py.test.skip("clock doesnt work") - - # XXX monkey patch hack XXX - from time import time - import pypy.translator.test.rpystone - pypy.translator.test.rpystone.clock = time - # XXX monkey patch hack XXX - from pypy.translator.test.rpystone import pystones as entry_point entry_point = compile_function(entry_point, [int]) t, pystones = entry_point(50000) From cfbolz at codespeak.net Sun Nov 18 00:04:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 18 Nov 2007 00:04:32 +0100 (CET) Subject: [pypy-svn] r48768 - in pypy/branch/ropes-unicode/pypy: config objspace/std Message-ID: <20071117230432.52AED8138@code0.codespeak.net> Author: cfbolz Date: Sun Nov 18 00:04:31 2007 New Revision: 48768 Added: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (contents, props changed) Modified: pypy/branch/ropes-unicode/pypy/config/pypyoption.py pypy/branch/ropes-unicode/pypy/objspace/std/objspace.py pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py Log: checking in some very much in-progress work before I leave for the sprint: the start of an implementation of unicode strings as ropes. Far from completion, but I don't want to risk my laptop being stolen. Modified: pypy/branch/ropes-unicode/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/config/pypyoption.py (original) +++ pypy/branch/ropes-unicode/pypy/config/pypyoption.py Sun Nov 18 00:04:31 2007 @@ -153,6 +153,10 @@ suggests=[("objspace.std.withprebuiltchar", True), ("objspace.std.sharesmallstr", True)]), + BoolOption("withropeunicod", "use ropes for the unicode implementation", + default=False, + requires=[("objspace.std.withrope", True)]), + BoolOption("withmultidict", "use dictionaries optimized for flexibility", default=False), Modified: pypy/branch/ropes-unicode/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/objspace.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/objspace.py Sun Nov 18 00:04:31 2007 @@ -400,7 +400,8 @@ from pypy.objspace.std.stringtype import wrapstr return wrapstr(self, x) if isinstance(x, unicode): - return W_UnicodeObject(x) + from pypy.objspace.std.unicodetype import wrapunicode + return wrapunicode(self, x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, Wrappable): Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Sun Nov 18 00:04:31 2007 @@ -920,6 +920,13 @@ self.advance_index() return result + def nextrope(self): + node = self.getnode() + index = self.index + result = node.getrope(self.index) + self.advance_index() + return result + def nextint(self): node = self.getnode() index = self.index @@ -1212,6 +1219,18 @@ substrings.append(getslice_one(node, startidx, node.length())) return substrings +def split_completely(node, maxsplit=-1): + upper = node.length() + if maxsplit > 0 and maxsplit < upper + 2: + upper = maxsplit - 1 + assert upper >= 0 + substrings = [by] + iter = ItemIterator(node) + for i in range(upper): + substrings.append(iter.nextrope()) + substrings.append(rope.getslice_one(node, upper, length)) + + # __________________________________________________________________________ # misc @@ -1225,3 +1244,69 @@ x ^= rope.length() return intmask(x) +# ____________________________________________________________ +# to and from unicode conversion + +def str_decode(rope, encoding): + assert rope.is_bytestring() + if encoding == "ascii": + if rope.is_ascii(): + return rope + elif encoding == "latin-1": + return rope + elif encoding == "utf-8": + from pypy.rlib.runicode import str_decode_utf_8 + if rope.is_ascii(): + return rope + elif isinstance(rope, BinaryConcatNode): + lresult = str_decode(rope.left, "utf-8") + if result is not None: + return BinaryConcatNode(lresult, + str_decode(rope.right, "utf-8")) + elif isinstance(rope, LiteralStringNode): + result, consumed = str_decode_utf_8(rope.s, len(rope.s), False, + "strict") + if consumed < len(rope.s): + return None + return rope_from_unicode(result) + s = rope.flatten_string() + return str_decode_utf_8(s, len(s), True) + else: + raise NotImplementedError("unknown encoding") + +def unicode_encode(rope, encoding): + if encoding == "ascii": + if rope.is_ascii(): + return rope + elif encoding == "latin-1": + return rope + elif encoding == "utf-8": + from pypy.rlib.runicode import unicode_encode_utf_8 + if rope.is_ascii(): + return rope + elif isinstance(rope, BinaryConcatNode): + return BinaryConcatNode(unicode_encode(rope.left, "utf-8"), + unicode_encode(rope.right, "utf-8")) + elif isinstance(rope, LiteralUnicodeNode): + return unicode_encode_utf_8(rope.u, len(rope.u), "strict") + elif isinstance(rope, LiteralStringNode): + return LiteralStringNode(_str_encode_utf_8(rope.s)) + s = rope.flatten_string() + return str_decode_utf_8(s, len(s), True) + else: + raise NotImplementedError("unknown encoding") + +def _str_encode_utf_8(s): + size = len(s) + result = [] + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + if (ch < 0x80): + # Encode ASCII + result.append(chr(ch)) + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + return LiteralStringNode("".join(s)) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Sun Nov 18 00:04:31 2007 @@ -498,7 +498,7 @@ substrings = [by] iter = rope.ItemIterator(node) for i in range(upper): - substrings.append(rope.LiteralStringNode.PREBUILT[ord(iter.nextchar())]) + substrings.append(iter.nextrope()]) substrings.append(by) substrings.append(rope.getslice_one(node, upper, length)) try: @@ -507,7 +507,7 @@ raise OperationError(space.w_OverflowError, space.wrap("string too long")) substrings = rope.split(node, sub, maxsplit) - if substrings is None: + if not substrings: return w_self.create_if_subclassed() try: return W_RopeObject(rope.join(by, substrings)) @@ -639,10 +639,10 @@ # no sophisticated linebreak support now # '\r' just for passing adapted CPython test try: - char = iter.nextchar() + char = iter.nextint() except StopIteration: break - if char == "\n" or char == "\r": + if char == ord("\n") or char == ord("\r"): break distance += 1 Added: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- (empty file) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Sun Nov 18 00:04:31 2007 @@ -0,0 +1,1025 @@ +from pypy.objspace.std.objspace import * +from pypy.interpreter import gateway +from pypy.objspace.std.stringobject import W_StringObject +from pypy.objspace.std.unicodeobject import _normalize_index +from pypy.objspace.std.ropeobject import W_RopeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std import rope +from pypy.objspace.std.sliceobject import W_SliceObject +from pypy.objspace.std import slicetype +from pypy.objspace.std.tupleobject import W_TupleObject +from pypy.rlib.rarithmetic import intmask, ovfcheck +from pypy.module.unicodedata import unicodedb_3_2_0 as unicodedb +from pypy.tool.sourcetools import func_with_new_name + +from pypy.objspace.std.formatting import mod_format + + +def wrapunicode(space, uni): + return W_RopeUnicodeObject(rope.rope_from_unicode(uni)) + +def unicode_from_string(space, w_str): + from pypy.objspace.std.unicodetype import getdefaultencoding, \ + unicode_from_encoded_object + assert isinstance(w_str, W_RopeObject) + node = w_str._node + encoding = getdefaultencoding(space) + if encoding == 'ascii': + result = rope.str_decode_ascii(node) + if result is not None: + return W_RopeUnicodeObject(result) + elif encoding == 'latin-1': + assert node.is_bytestring() + return W_RopeUnicodeObject(node) + elif encoding == "utf-8": + result = rope.str_decode_utf8(node) + if result is not None: + return W_RopeUnicodeObject(result) + return unicode_from_encoded_object(space, w_str, encoding, "strict") + + +class W_RopeUnicodeObject(W_Object): + from pypy.objspace.std.unicodetype import unicode_typedef as typedef + + def __init__(w_self, node): + w_self._node = node + + def __repr__(w_self): + """ representation for debugging purposes """ + return "%s(%r)" % (w_self.__class__.__name__, w_self._node) + + def unwrap(w_self, space): + # for testing + return w_self._node.flatten_unicode() + + def create_if_subclassed(w_self): + if type(w_self) is W_RopeUnicodeObject: + return w_self + return W_RopeUnicodeObject(w_self._value) + +W_RopeUnicodeObject.EMPTY = W_RopeUnicodeObject(rope.LiteralStringNode.EMPTY) + +registerimplementation(W_RopeUnicodeObject) + + +def ropeunicode_w(space, w_str): + if isinstance(w_str, W_RopeUnicodeObject): + return w_str._node + # XXX do the right thing for W_RopeObject + return rope.LiteralUnicodeNode(space.unicode_w(w_str)) + + +# Helper for converting int/long +def unicode_to_decimal_w(space, w_unistr): + if not isinstance(w_unistr, W_RopeUnicodeObject): + raise OperationError(space.w_TypeError, + space.wrap("expected unicode")) + unistr = w_unistr._node + length = unistr.length() + result = ['\0'] * length + digits = [ '0', '1', '2', '3', '4', + '5', '6', '7', '8', '9'] + iter = rope.ItemIterator(unistr) + for i in range(length): + uchr = iter.nextint() + if unicodedb.isspace(uchr): + result[i] = ' ' + continue + try: + result[i] = digits[unicodedb.decimal(uchr)] + except KeyError: + if 0 < uchr < 256: + result[i] = chr(uchr) + else: + w_encoding = space.wrap('decimal') + w_start = space.wrap(i) + w_end = space.wrap(i+1) + w_reason = space.wrap('invalid decimal Unicode string') + raise OperationError(space.w_UnicodeEncodeError,space.newtuple ([w_encoding, w_unistr, w_start, w_end, w_reason])) + return ''.join(result) + +# string-to-unicode delegation +def delegate_Rope2RopeUnicode(space, w_rope): + w_uni = unicode_from_string(space, w_rope) + assert isinstance(w_uni, W_RopeUnicodeObject) # help the annotator! + return w_uni + +def str_w__RopeUnicode(space, w_uni): + return space.str_w(space.str(w_uni)) + +def unicode_w__RopeUnicode(space, w_uni): + return w_uni._node.flatten_unicode() + +def str__RopeUnicode(space, w_uni): + return space.call_method(w_uni, 'encode') + +def lt__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + n1 = w_str1._node + n2 = w_str2._node + return space.newbool(rope.compare(n1, n2) < 0) + +def le__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + n1 = w_str1._node + n2 = w_str2._node + return space.newbool(rope.compare(n1, n2) <= 0) + +def _eq(w_str1, w_str2): + result = rope.eq(w_str1._node, w_str2._node) + return result + +def eq__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + return space.newbool(_eq(w_str1, w_str2)) + +def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + return space.newbool(not _eq(w_str1, w_str2)) + +def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + n1 = w_str1._node + n2 = w_str2._node + return space.newbool(rope.compare(n1, n2) > 0) + +def ge__RopeUnicode_RopeUnicode(space, w_str1, w_str2): + n1 = w_str1._node + n2 = w_str2._node + return space.newbool(rope.compare(n1, n2) >= 0) + + +def ord__RopeUnicode(space, w_uni): + if w_uni.length() != 1: + raise OperationError(space.w_TypeError, space.wrap('ord() expected a character')) + return space.wrap(w_uni._node.getint(0)) + +def getnewargs__RopeUnicode(space, w_uni): + return space.newtuple([W_RopeUnicodeObject(w_uni._node)]) + +def add__RopeUnicode_RopeUnicode(space, w_left, w_right): + right = w_right._node + left = w_left._node + try: + return W_RopeUnicodeObject(rope.concatenate(left, right)) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("string too long")) + +def add__Rope_RopeUnicode(space, w_left, w_right): + return space.add(unicode_from_string(space, w_left) , w_right) + +def add__RopeUnicode_Rope(space, w_left, w_right): + return space.add(w_left, unicode_from_string(space, w_right)) + +def contains__RopeUnicode_RopeUnicode(space, w_container, w_item): + item = w_item._node + container = w_container._node + return space.newbool(rope.find(container, item) != -1) + +def contains__Rope_RopeUnicode(space, w_container, w_item): + return space.contains(unicode_from_string(space, w_container), w_item ) + +def unicode_join__RopeUnicode_ANY(space, w_self, w_list): + l_w = space.unpackiterable(w_list) + delim = w_self._value + totlen = 0 + if len(l_w) == 0: + return W_RopeUnicodeObject.EMPTY + if (len(l_w) == 1 and + space.is_w(space.type(l_w[0]), space.w_unicode)): + return l_w[0] + + values_list = [] + for i in range(len(l_w)): + w_item = l_w[i] + if isinstance(w_item, W_RopeUnicodeObject): + # shortcut for performane + item = w_item._node + elif space.is_true(space.isinstance(w_item, space.w_str)): + item = unicode_from_string(space, w_item)._node + else: + w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), + space.wrap(i)) + raise OperationError(space.w_TypeError, w_msg) + values_list.append(item) + try: + return W_RopeUnicodeObject(rope.join(self, values_list)) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("string too long")) + +def hash__RopeUnicode(space, w_uni): + return space.wrap(rope.hash_rope(w_uni._node)) + +def len__RopeUnicode(space, w_uni): + return space.wrap(w_uni._node.length()) + +def getitem__RopeUnicode_ANY(space, w_uni, w_index): + ival = space.getindex_w(w_index, space.w_IndexError, "string index") + uni = w_uni._node + ulen = uni.length() + if ival < 0: + ival += ulen + if ival < 0 or ival >= ulen: + exc = space.call_function(space.w_IndexError, + space.wrap("unicode index out of range")) + raise OperationError(space.w_IndexError, exc) + return W_RopeUnicodeObject(uni.getrope(ival)) + +def getitem__RopeUnicode_Slice(space, w_uni, w_slice): + node = w_uni._noed + length = node.length() + start, stop, step, sl = w_slice.indices4(space, length) + if sl == 0: + return W_RopeUnicodeObject.EMPTY + return W_RopeUnicodeObject(rope.getslice(node, start, stop, step, sl)) + +def mul__RopeUnicode_ANY(space, w_uni, w_times): + try: + times = space.getindex_w(w_times, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise + node = w_uni._node + try: + return W_RopeUnicodeObject(rope.multiply(node, mul)) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("string too long")) + +def mul__ANY_RopeUnicode(space, w_times, w_uni): + return mul__RopeUnicode_ANY(space, w_uni, w_times) + + +def make_generic(funcname): + def func(space, w_self): + node = w_self._node + if node.length() == 0: + return space.w_False + iter = rope.ItemIterator(node) + for idx in range(node.length()): + if not getattr(unicodedb, funcname)(iter.nextint()): + return space.w_False + return space.w_True + return func_with_new_name(func, "unicode_%s__RopeUnicode" % (funcname, )) + +unicode_isspace__RopeUnicode = make_generic("isspace") +unicode_isalpha__RopeUnicode = make_generic("isalpha") +unicode_isalnum__RopeUnicode = make_generic("isalnum") +unicode_isdecimal__RopeUnicode = make_generic("isdecimal") +unicode_isdigit__RopeUnicode = make_generic("isdigit") +unicode_isnumeric__RopeUnicode = make_generic("isnumeric") + +def unicode_islower__RopeUnicode(space, w_unicode): + cased = False + iter = rope.ItemIterator(w_unicode._node) + while 1: + try: + ch = iter.nextint() + except StopIteration: + return space.newbool(cased) + if (unicodedb.isupper(ch) or + unicodedb.istitle(ch)): + return space.w_False + if not cased and unicodedb.islower(ch): + cased = True + +def unicode_isupper__RopeUnicode(space, w_unicode): + cased = False + iter = rope.ItemIterator(w_unicode._node) + while 1: + try: + ch = iter.nextint() + except StopIteration: + return space.newbool(cased) + if (unicodedb.islower(ch) or + unicodedb.istitle(ch)): + return space.w_False + if not cased and unicodedb.isupper(ch): + cased = True + +def unicode_istitle__RopeUnicode(space, w_unicode): + cased = False + previous_is_cased = False + iter = rope.ItemIterator(w_unicode._node) + while 1: + try: + ch = iter.nextint() + except StopIteration: + return space.newbool(cased) + if (unicodedb.isupper(ch) or + unicodedb.istitle(ch)): + if previous_is_cased: + return space.w_False + previous_is_cased = cased = True + elif unicodedb.islower(ch): + if not previous_is_cased: + return space.w_False + previous_is_cased = cased = True + else: + previous_is_cased = False + + +def _strip(space, w_self, w_chars, left, right): + "internal function called by str_xstrip methods" + XXX + u_self = w_self._value + u_chars = w_chars._value + + lpos = 0 + rpos = len(u_self) + + if left: + while lpos < rpos and u_self[lpos] in u_chars: + lpos += 1 + + if right: + while rpos > lpos and u_self[rpos - 1] in u_chars: + rpos -= 1 + + assert rpos >= 0 + result = u_self[lpos: rpos] + return W_UnicodeObject(result) + +def unicode_strip__RopeUnicode_None(space, w_self, w_chars): + return W_RopeUnicodeObject(rope.strip(w_self._none, True, True, _isspace)) +def unicode_strip__RopeUnicode_RopeUnicode(space, w_self, w_chars): + return _strip(space, w_self, w_chars, 1, 1) +def unicode_strip__RopeUnicode_Rope(space, w_self, w_chars): + return space.call_method(w_self, 'strip', + unicode_from_string(space, w_chars)) + +def unicode_lstrip__RopeUnicode_None(space, w_self, w_chars): + return W_RopeUnicodeObject(rope.strip(w_self._none, True, False, _isspace)) +def unicode_lstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars): + return _strip(space, w_self, w_chars, 1, 0) +def unicode_lstrip__RopeUnicode_Rope(space, w_self, w_chars): + return space.call_method(w_self, 'lstrip', + unicode_from_string(space, w_chars)) + +def unicode_rstrip__RopeUnicode_None(space, w_self, w_chars): + return W_RopeUnicodeObject(rope.strip(w_self._none, False, True, _isspace)) +def unicode_rstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars): + return _strip(space, w_self, w_chars, 0, 1) +def unicode_rstrip__RopeUnicode_Rope(space, w_self, w_chars): + return space.call_method(w_self, 'rstrip', + unicode_from_string(space, w_chars)) + +def unicode_capitalize__RopeUnicode(space, w_self): + input = w_self._node + length = input.length() + if length == 0: + return w_self + result = [u'\0'] * length + iter = rope.ItemIterator(input) + result[0] = unichr(unicodedb.toupper(iter.nextint())) + for i in range(1, length): + result[i] = unichr(unicodedb.tolower(iter.nextint())) + return W_RopeUnicodeObject(rope.rope_from_unicharlist(result)) + +def unicode_title__RopeUnicode(space, w_self): + input = w_self._node + length = input.length() + if length == 0: + return w_self + result = [u'\0'] * length + iter = rope.ItemIterator(input) + + previous_is_cased = False + for i in range(len(input)): + unichar = iter.nextint() + if previous_is_cased: + result[i] = unichr(unicodedb.tolower(unichar)) + else: + result[i] = unichr(unicodedb.totitle(unichar)) + previous_is_cased = unicodedb.iscased(unichar) + return W_RopeUnicodeObject(rope.rope_from_unicharlist(result)) + + +def _local_transform(node, transform): + l = node.length() + res = [u' '] * l + iter = rope.ItemIterator(node) + for i in range(l): + ch = iter.nextint() + res[i] = transform(ch) + + return W_RopeUnicodeObject(rope.rope_from_unicharlist(res)) +_local_transform._annspecialcase_ = "specialize:arg(1)" + +def _tolower(ordch): + return unichr(unicodedb.tolower(ordch)) +def unicode_lower__RopeUnicode(space, w_self): + return _local_transform(w_self._node, _tolower) + +def _toupper(ordch): + return unichr(unicodedb.toupper(ordch)) +def unicode_upper__RopeUnicode(space, w_self): + return _local_transform(w_self._node, _toupper) + +def _swapcase(ordch): + if unicodedb.islower(unichar): + return unichr(unicodedb.toupper(unichar)) + elif unicodedb.isupper(unichar): + return unichr(unicodedb.tolower(unichar)) + else: + return unichr(ordch) + +def unicode_swapcase__RopeUnicode(space, w_self): + return _local_transform(w_self._node, _swapcase) + +def _convert_idx_params(space, w_self, w_start, w_end): + self = w_self._node + length = w_self._node.length() + start = slicetype.adapt_bound(space, length, w_start) + end = slicetype.adapt_bound(space, length, w_end) + + assert start >= 0 + assert end >= 0 + + return (self, start, end) + +def _check_startswith_substring(str, substr, start, end): + XXX + substr_len = len(substr) + + if end - start < substr_len: + return False # substring is too long + + for i in range(substr_len): + if str[start + i] != substr[i]: + return False + return True + +def _check_endswith_substring(str, substr, start, end): + XXX + substr_len = len(substr) + + if end - start < substr_len: + return False # substring is too long + start = end - substr_len + for i in range(substr_len): + if str[start + i] != substr[i]: + return False + return True + +def unicode_endswith__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + return space.newbool(rope.endswith(self, w_substr._node, start, end)) + +def unicode_startswith__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + # XXX this stuff can be waaay better for ootypebased backends if + # we re-use more of our rpython machinery (ie implement startswith + # with additional parameters as rpython) + + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + return space.newbool(rope.startswith(self, w_substr._node, start, end)) + +def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for w_prefix in space.unpacktuple(w_prefixes): + prefix = unicoderope_w(space, w_prefix) + if rope.startswith(self, prefxix, start, end) + return space.w_True + return space.w_False + +def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, + w_start, w_end): + unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) + for w_suffix in space.unpacktuple(w_suffixes): + suffix = unicoderope_w(space, w_suffix) + if _check_endswith_substring(unistr, suffix, start, end): + return space.w_True + return space.w_False + + +def _to_unichar_w(space, w_char): + try: + unistr = ropeunicode_w(space, w_char) + except OperationError, e: + if e.match(space, space.w_TypeError): + msg = 'The fill character cannot be converted to Unicode' + raise OperationError(space.w_TypeError, space.wrap(msg)) + else: + raise + + if unistr.length() != 1: + raise OperationError(space.w_TypeError, space.wrap('The fill character must be exactly one character long')) + return unistr + +def unicode_center__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): + self = w_self._node + length = self.length + width = space.int_w(w_width) + fillchar = _to_unichar_w(space, w_fillchar) + padding = width - length + if padding < 0: + return w_self.create_if_subclassed() + offset = padding // 2 + pre = rope.multiply(fillchar, offset) + post = rope.multiply(fillchar, (padding - offset)) + centered = rope.rebalance([pre, node, post]) + return W_RopeUnicodeObject(centered) + +def unicode_ljust__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): + self = w_self._node + length = self.length + width = space.int_w(w_width) + fillchar = _to_unichar_w(space, w_fillchar) + padding = width - length + if padding < 0: + return w_self.create_if_subclassed() + resultnode = rope.concatenate(self, rope.multiply(fillchar, d)) + return W_RopeUnicodeObject(resultnode) + +def unicode_rjust__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): + self = w_self._node + length = self.length + width = space.int_w(w_width) + fillchar = _to_unichar_w(space, w_fillchar) + padding = width - length + if padding < 0: + return w_self.create_if_subclassed() + resultnode = rope.concatenate(rope.multiply(fillchar, d), self) + return W_RopeUnicodeObject(resultnode) + +def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width): + self = w_self._value + length = self.length + width = space.int_w(w_width) + if self.length() == 0: + return W_RopeUnicodeObject( + rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], width)) + padding = width - self.length() + if padding <= 0: + return w_self.create_if_subclassed() + firstchar = self.getunichar(0) + if firstchar in (u'+', u'-'): + return W_RopeUnicodeObject(rope.rebalance( + [rope.LiteralStringNode.PREBUILT[ord(firstchar)], + rope.multiply(zero, middle), + rope.getslice_one(node, 1, length)])) + else: + middle = width - length + return W_RopeUnicodeObject(rope.concatenate( + rope.multiply(zero, middle), node)) + +def unicode_splitlines__RopeUnicode_ANY(space, w_self, w_keepends): + XXX + self = w_self._value + keepends = 0 + if space.int_w(w_keepends): + keepends = 1 + if len(self) == 0: + return space.newlist([]) + + start = 0 + end = len(self) + pos = 0 + lines = [] + while pos < end: + if unicodedb.islinebreak(ord(self[pos])): + if (self[pos] == u'\r' and pos + 1 < end and + self[pos + 1] == u'\n'): + # Count CRLF as one linebreak + lines.append(W_UnicodeObject(self[start:pos + keepends * 2])) + pos += 1 + else: + lines.append(W_UnicodeObject(self[start:pos + keepends])) + pos += 1 + start = pos + else: + pos += 1 + if not unicodedb.islinebreak(ord(self[end - 1])): + lines.append(W_UnicodeObject(self[start:])) + return space.newlist(lines) + + +def unicode_find__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + sub = w_substr._node + return space.wrap(rope.find(self, sub, start, end)) + +def unicode_rfind__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + self = self.flatten_unicode() + sub = sub.flatten_unicode() + res = self.rfind(sub, start, end) + return wrapint(space, res) + +def unicode_index__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + sub = w_substr._node + res = rope.find(self, sub, start, end) + if res < 0: + raise OperationError(space.w_ValueError, + space.wrap("substring not found in string.index")) + return space.wrap(res) + +def unicode_rindex__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + # XXX works but flattens string + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + self = self.flatten_unicode() + sub = sub.flatten_unicode() + res = self.rfind(sub, start, end) + if res < 0: + raise OperationError(space.w_ValueError, + space.wrap("substring not found in string.rindex")) + + return wrapint(space, res) + +def unicode_count__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + self, start, end = _convert_idx_params(space, w_self, w_start, w_end) + substr = w_substr._value + return space.wrap(self.count(substr, start, end)) + +def unicode_split__RopeUnicode_None_ANY(space, w_self, w_none, w_maxsplit): + self = w_self._value + maxsplit = space.int_w(w_maxsplit) + parts = [] + if len(self) == 0: + return space.newlist([]) + start = 0 + end = len(self) + inword = 0 + + while maxsplit != 0 and start < end: + index = start + for index in range(start, end): + if _isspace(self[index]): + break + else: + inword = 1 + else: + break + if inword == 1: + parts.append(W_UnicodeObject(self[start:index])) + maxsplit -= 1 + # Eat whitespace + for start in range(index + 1, end): + if not _isspace(self[start]): + break + else: + return space.newlist(parts) + + parts.append(W_UnicodeObject(self[start:])) + return space.newlist(parts) + +def unicode_split__RopeUnicode_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): + maxsplit = space.int_w(w_maxsplit) + start = 0 + selfnode = w_self._node + bynode = w_by._node + bylen = bynode.length() + if bylen == 0: + raise OperationError(space.w_ValueError, space.wrap("empty separator")) + res_w = [W_RopeUnicodeObject(node) + for node in rope.split(selfnode, bynode, maxsplit)] + return space.newlist(res_w) + +def unicode_rsplit__RopeUnicode_None_ANY(space, w_self, w_none, w_maxsplit): + XXX + self = w_self._value + maxsplit = space.int_w(w_maxsplit) + parts = [] + if len(self) == 0: + return space.newlist([]) + start = 0 + end = len(self) + inword = 0 + + while maxsplit != 0 and start < end: + index = end + for index in range(end-1, start-1, -1): + if _isspace(self[index]): + break + else: + inword = 1 + else: + break + if inword == 1: + parts.append(W_UnicodeObject(self[index+1:end])) + maxsplit -= 1 + # Eat whitespace + for end in range(index, start-1, -1): + if not _isspace(self[end-1]): + break + else: + return space.newlist(parts) + + parts.append(W_UnicodeObject(self[:end])) + parts.reverse() + return space.newlist(parts) + +def unicode_rsplit__RopeUnicode_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): + # XXX works but flattens + self = w_self._node.flatten_unicode() + delim = w_delim._node.flatten_unicode() + maxsplit = space.int_w(w_maxsplit) + delim_len = len(delim) + if delim_len == 0: + raise OperationError(space.w_ValueError, + space.wrap('empty separator')) + parts = [] + if len(self) == 0: + return space.newlist([]) + start = 0 + end = len(self) + while maxsplit != 0: + index = self.rfind(delim, 0, end) + if index < 0: + break + parts.append(W_RopeUnicodeObject( + rope.getslice_one(w_self._node, index+delim_len, end))) + end = index + maxsplit -= 1 + parts.append(W_RopeUnicodeObject( + rope.getslice_one(w_self._node, 0, :end))) + parts.reverse() + return space.newlist(parts) + +def _split_into_chars(self, maxsplit): + if maxsplit == 0: + return [self] + index = 0 + end = len(self) + parts = [rope.LiteralStringNode.EMPTY] + maxsplit -= 1 + while maxsplit != 0: + if index >= end: + break + parts.append(self.getrope(index)) + index += 1 + maxsplit -= 1 + parts.append(rope.getslice_one(self, index, self.length()) + return parts + +def unicode_replace__RopeUnicode_RopeUnicode_RopeUnicode_ANY( + space, w_self, w_old, w_new, w_maxsplit): + old = w_old._node + oldlength = old.length + if not oldlength: + self = w_self._value + maxsplit = space.int_w(w_maxsplit) + parts = _split_into_chars(self, maxsplit) + return W_RopeUnicodeObject(rope.join(w_new._node, parts)) + substrings = rope.split(node, old, maxsplit) + if not substrings: + return w_self.create_if_subclassed() + substrings = rope.split(node, sub, maxsplit) + if substrings is None: + return w_self.create_if_subclassed() + try: + return W_RopeObject(rope.join(by, substrings)) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("string too long")) + try: + return W_RopeObject(rope.join(by, substrings)) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("string too long")) + return W_UnicodeObject(w_new._value.join(parts)) + + +def unicode_encode__Unicode_ANY_ANY(space, w_unistr, + w_encoding=None, + w_errors=None): + + from pypy.objspace.std.unicodetype import getdefaultencoding, \ + _get_encoding_and_errors, encode_object + encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) + if encoding is None: + encoding = getdefaultencoding(space) + w_retval = encode_object(space, w_unistr, encoding, errors) + return w_retval + +def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): + self = w_unistr._node + sub = w_unisub._node + if not sub.length(): + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) + pos = rope.find(self, sub) + if pos == -1: + return space.newtuple([w_self, W_RopeUnicodeObject.EMPTY, + W_RopeUnicodeObject.EMPTY]) + else: + return space.newtuple( + [W_RopeUnicodeObject(rope.getslice_one(self, 0, pos)), + w_sub, + W_RopeUnicodeObject(rope.getslice_one(self, pos + sub.length(), + self.length()))]) + +def unicode_rpartition__Unicode_Unicode(space, w_unistr, w_unisub): + # XXX works but flattens + unistr = w_unistr._node.flatten_unicode() + unisub = w_unisub._node.flatten_unicode() + if not unisub: + raise OperationError(space.w_ValueError, + space.wrap("empty separator")) + pos = unistr.rfind(unisub) + if pos == -1: + return space.newtuple([W_UnicodeObject.EMPTY, + W_UnicodeObject.EMPTY, w_unistr]) + else: + assert pos > 0 + return space.newtuple([space.wrap(unistr[:pos]), w_unisub, + space.wrap(unistr[pos+len(unisub):])]) + + +def unicode_expandtabs__RopeUnicode_ANY(space, w_self, w_tabsize): + self = w_self._node + tabsize = space.int_w(w_tabsize) + splitted = rope.split(node, rope.LiteralStringNode.PREBUILT[ord('\t')]) + last = splitted[0] + expanded = [last] + for i in range(1, len(splitted)): + expanded.append(rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], + _tabindent(last, tabsize))) + last = splitted[i] + expanded.append(last) + return W_RopeUnicodeObject(rope.rebalance(expanded)) + +def unicode_translate__RopeUnicode_ANY(space, w_self, w_table): + self = w_self._node + w_sys = space.getbuiltinmodule('sys') + maxunicode = space.int_w(space.getattr(w_sys, space.wrap("maxunicode"))) + result = [] + iter = rope.ItemIterator(self) + for i in range(self.length()): + crope = iter.nextrope() + char = crope.getint(0) + try: + w_newval = space.getitem(w_table, space.wrap(char)) + except OperationError, e: + if e.match(space, space.w_KeyError): + result.append(crope) + else: + raise + else: + if space.is_w(w_newval, space.w_None): + continue + elif space.is_true(space.isinstance(w_newval, space.w_int)): + newval = space.int_w(w_newval) + if newval < 0 or newval > maxunicode: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must be in range(0x%x)" % (maxunicode + 1,))) + result.append(unichr(newval)) + elif space.is_true(space.isinstance(w_newval, space.w_unicode)): + result.append(unicoderope_w(w_newval)) + else: + raise OperationError( + space.w_TypeError, + space.wrap("character mapping must return integer, None or unicode")) + return W_RopeUnicodeObject(rope.join(rope.LiteralStringNode.EMPTY, result)) + +# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' +def repr__RopeUnicode(space, w_unicode): + hexdigits = "0123456789abcdef" + chars = w_unicode._node + size = node.length() + + singlequote = doublequote = False + iter = rope.ItemIterator() + for i in range(size): + c = iter.nextunichar() + if singlequote and doublequote: + break + if c == u'\'': + singlequote = True + elif c == u'"': + doublequote = True + if singlequote and not doublequote: + quote = '"' + else: + quote = '\'' + result = ['u', quote] + iter = rope.ItemIterator() + j = 0 + while j < size: + code = iter.nextint() + if code >= 0x10000: + result.extend(['\\', "U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) + j += 1 + continue + if code >= 0xD800 and code < 0xDC00: + if j < size - 1: + code2 = iter.nextint() + if code2 >= 0xDC00 and code2 <= 0xDFFF: + code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 + result.extend(["U", + hexdigits[(code >> 28) & 0xf], + hexdigits[(code >> 24) & 0xf], + hexdigits[(code >> 20) & 0xf], + hexdigits[(code >> 16) & 0xf], + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) + j += 2 + continue + + if code >= 0x100: + result.extend(['\\', "u", + hexdigits[(code >> 12) & 0xf], + hexdigits[(code >> 8) & 0xf], + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) + j += 1 + continue + if code == ord('\\') or code == ord(quote): + result.append('\\') + result.append(chr(code)) + j += 1 + continue + if code == ord('\t'): + result.append('\\') + result.append('t') + j += 1 + continue + if code == ord('\r'): + result.append('\\') + result.append('r') + j += 1 + continue + if code == ord('\n'): + result.append('\\') + result.append('n') + j += 1 + continue + if code < ord(' ') or code >= 0x7f: + result.extend(['\\', "x", + hexdigits[(code >> 4) & 0xf], + hexdigits[(code >> 0) & 0xf], + ]) + j += 1 + continue + result.append(chr(code)) + j += 1 + result.append(quote) + return W_RopeObject(rope.rope_from_charlist(result)) + +def mod__RopeUnicode_ANY(space, w_format, w_values): + return mod_format(space, w_format, w_values, do_unicode=True) + + +import unicodetype +register_all(vars(), unicodetype) + +# str.strip(unicode) needs to convert self to unicode and call unicode.strip we +# use the following magic to register strip_string_unicode as a String +# multimethod. + +# XXX couldn't string and unicode _share_ the multimethods that make up their +# methods? + +class str_methods: + import stringtype + W_RopeUnicodeObject = W_RopeUnicodeObject + from pypy.objspace.std.ropeobject import W_RopeObject + def str_strip__Rope_RopeUnicode(space, w_self, w_chars): + return space.call_method(unicode_from_string(space, w_self), + 'strip', w_chars) + def str_lstrip__Rope_RopeUnicode(space, w_self, w_chars): + return space.call_method(unicode_from_string(space, w_self), + 'lstrip', w_chars) + def str_rstrip__Rope_RopeUnicode(space, w_self, w_chars): + return space.call_method(unicode_from_string(space, w_self), + 'rstrip', w_chars) + def str_count__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + return space.call_method(unicode_from_string(space, w_self), + 'count', w_substr, w_start, w_end) + def str_find__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + return space.call_method(unicode_from_string(space, w_self), + 'find', w_substr, w_start, w_end) + def str_rfind__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + return space.call_method(unicode_from_string(space, w_self), + 'rfind', w_substr, w_start, w_end) + def str_index__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + return space.call_method(unicode_from_string(space, w_self), + 'index', w_substr, w_start, w_end) + def str_rindex__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): + return space.call_method(unicode_from_string(space, w_self), + 'rindex', w_substr, w_start, w_end) + def str_replace__Rope_RopeUnicode_RopeUnicode_ANY(space, w_self, w_old, w_new, w_maxsplit): + return space.call_method(unicode_from_string(space, w_self), + 'replace', w_old, w_new, w_maxsplit) + def str_split__Rope_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): + return space.call_method(unicode_from_string(space, w_self), + 'split', w_delim, w_maxsplit) + def str_rsplit__Rope_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): + return space.call_method(unicode_from_string(space, w_self), + 'rsplit', w_delim, w_maxsplit) + register_all(vars(), stringtype) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py Sun Nov 18 00:04:31 2007 @@ -5,6 +5,14 @@ from sys import maxint +def wrapunicode(space, uni): + from pypy.objspace.std.unicodeobject import W_UnicodeObject + from pypy.objspace.std.ropeunicodeobject import wrapunicode + if space.config.objspace.std.withropeunicode: + return wrapunicode(space, uni) + return W_UnicodeObject(uni) + + unicode_capitalize = SMM('capitalize', 1, doc='S.capitalize() -> unicode\n\nReturn a' ' capitalized version of S, i.e. make the first' @@ -218,8 +226,11 @@ def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack - from pypy.objspace.std.unicodeobject import W_UnicodeObject + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_from_string + return unicode_from_string(space, w_str) encoding = getdefaultencoding(space) + from pypy.objspace.std.unicodeobject import W_UnicodeObject if encoding != 'ascii': return unicode_from_encoded_object(space, w_str, encoding, "strict") s = space.str_w(w_str) @@ -227,13 +238,14 @@ return W_UnicodeObject(s.decode("ascii")) except UnicodeDecodeError: # raising UnicodeDecodeError is messy, "please crash for me" - return unicode_from_object(space, w_str) + return unicode_from_encoded_object(space, w_str, "ascii", "strict") def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work from pypy.objspace.std.unicodeobject import W_UnicodeObject + from pypy.objspace.std.ropeunicodeobject import W_RopeUnicodeObject w_obj_type = space.type(w_obj) encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) @@ -253,7 +265,12 @@ w_value = unicode_from_object(space, w_obj) else: w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) - # XXX this is not true when there are different unicode implementations + if space.config.objspace.std.withrope: + assert isinstance(w_value, W_RopeUnicodeObject) + w_newobj = space.allocate_instance(W_RopeUnicodeObject, w_unicodetype) + W_RopeUnicodeObject.__init__(w_newobj, w_value._node) + return w_newobj + assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) W_UnicodeObject.__init__(w_newobj, w_value._value) From fijal at codespeak.net Sun Nov 18 12:39:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Nov 2007 12:39:42 +0100 (CET) Subject: [pypy-svn] r48774 - pypy/extradoc/talk/pycon2008 Message-ID: <20071118113942.023878164@code0.codespeak.net> Author: fijal Date: Sun Nov 18 12:39:41 2007 New Revision: 48774 Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt Log: Add my bio Modified: pypy/extradoc/talk/pycon2008/proxy-abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/proxy-abstract.txt (original) +++ pypy/extradoc/talk/pycon2008/proxy-abstract.txt Sun Nov 18 12:39:41 2007 @@ -55,3 +55,10 @@ as a "proxying operation" and finally, why design matters when you want to get a butterfly out of a caterpillar instead of just a bigger, healthier caterpillar. + +Bio +=== + +Maciej Fijalkowski is a PyPy developer for two years now. He has been +presenting PyPy at the EuroPython 2007 in Vilnius as well as Rupy +2007 in Poznan. \ No newline at end of file From fijal at codespeak.net Sun Nov 18 12:40:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Nov 2007 12:40:01 +0100 (CET) Subject: [pypy-svn] r48775 - pypy/extradoc/talk/pycon2008 Message-ID: <20071118114001.747568164@code0.codespeak.net> Author: fijal Date: Sun Nov 18 12:40:01 2007 New Revision: 48775 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Add previous experience section and website outline Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Sun Nov 18 12:40:01 2007 @@ -1,13 +1,11 @@ Hands on PyPy - how to modify your virtual machine in 15 minutes ================================================================ -XXX We need a sexier title. - Presenter name(s) & contact information ======================================= -Laura Creighton -Maciej Fijalkowski +Laura Creighton - lac at openend.se +Maciej Fijalkowski - fijall at gmail.com Intended audience ================= @@ -17,13 +15,13 @@ Perhaps you are uncomfortable with some design decisions? want an extra feature? desire fast modules but don't want to write them -in C? or just wish to experiment? +in C? or just wish to experiment? -Attendees will be expected to be fluent in Python, and have prior -experience with meta-programming (in any language). No knowledge of C +Attendees will be expected to be fluent in Python, preferably have prior +experience with meta-programming. No knowledge of C is needed nor required, although prior knowledge of any virtual -machine (for example CPython) might be useful. If you are wondering -'what's a virtual machine' then this tutorial is not for you. +machine construction (for example CPython) might be helpful. If you are +wondering 'what's a virtual machine' then this tutorial is not for you. Tutorial format description =========================== @@ -51,8 +49,8 @@ * python2.4 or 2.5 -Note for Reviewers -================== +Notes for Reviewers +=================== Note to the organisers: This is a very advanced topic, and therefore it is hard to predict how many people will show up. We realise that @@ -145,7 +143,13 @@ Tutorial outline for website publication ======================================== -XXX fill in +* Introduction into PyPy + +* Skim over codebase + +* Live coding session, modifying minor virtual machine aspect + +* Hands-on PyPy virtual machine Presenter bio ============= @@ -155,7 +159,9 @@ Previous experience =================== -Europython 2007, Ruby 2007, sprints +Maciej Fijalkowski was giving talks at the EuroPython 2007 in Vilnius and +Rupy 2007 in Poznan, as well as participating in PyPy sprints for the last +two years. Laura Creighton was a partner with Jim Joyce in Jim Joyce's Unix bookstore/The Gawain Group. She has designed hundreds of hands-on From fijal at codespeak.net Sun Nov 18 13:21:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Nov 2007 13:21:13 +0100 (CET) Subject: [pypy-svn] r48776 - pypy/extradoc/talk/pycon2008 Message-ID: <20071118122113.26FEB816B@code0.codespeak.net> Author: fijal Date: Sun Nov 18 13:21:11 2007 New Revision: 48776 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Kill laura's paragraph. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Sun Nov 18 13:21:11 2007 @@ -151,11 +151,6 @@ * Hands-on PyPy virtual machine -Presenter bio -============= - -XXX fill in - Previous experience =================== @@ -165,6 +160,4 @@ Laura Creighton was a partner with Jim Joyce in Jim Joyce's Unix bookstore/The Gawain Group. She has designed hundreds of hands-on -tutorials -- though she nearly always left them for Jim Joyce, or -one of our hired presenters to deliver. In the 15 years since -Jim's death, she has learned to make better presentations. +tutorials From fijal at codespeak.net Sun Nov 18 13:26:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Nov 2007 13:26:51 +0100 (CET) Subject: [pypy-svn] r48777 - pypy/extradoc/talk/pycon2008 Message-ID: <20071118122651.AC1338154@code0.codespeak.net> Author: fijal Date: Sun Nov 18 13:26:51 2007 New Revision: 48777 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: Update bio as I realised that this is send by mail (ARGH!) Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Sun Nov 18 13:26:51 2007 @@ -151,6 +151,16 @@ * Hands-on PyPy virtual machine +Presenter Bio +============= + +Maciej Fijalkowski is a PyPy developer for two years now. +XXX Give more technical details here? Besides, I've got no idea what to + put here + +XXX Laura, put your's here, although I've got no idea what's the real + difference between previous experience and short bio. + Previous experience =================== From lac at codespeak.net Sun Nov 18 15:02:56 2007 From: lac at codespeak.net (lac at codespeak.net) Date: Sun, 18 Nov 2007 15:02:56 +0100 (CET) Subject: [pypy-svn] r48778 - pypy/extradoc/talk/pycon2008 Message-ID: <20071118140256.081CA80FA@code0.codespeak.net> Author: lac Date: Sun Nov 18 15:02:55 2007 New Revision: 48778 Modified: pypy/extradoc/talk/pycon2008/tutorial.txt Log: ok, added my bio. maciek, the presenter history is supposed to tell david Goodger that the presenter(s) know what the heck they are doing, in case lots of people want to present on the same topic. Then ones with a proven track record will be favoured over unknowns. Modified: pypy/extradoc/talk/pycon2008/tutorial.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/tutorial.txt (original) +++ pypy/extradoc/talk/pycon2008/tutorial.txt Sun Nov 18 15:02:55 2007 @@ -154,12 +154,11 @@ Presenter Bio ============= -Maciej Fijalkowski is a PyPy developer for two years now. -XXX Give more technical details here? Besides, I've got no idea what to - put here +Maciej Fijalkowski has been a PyPy developer for the past two years. -XXX Laura, put your's here, although I've got no idea what's the real - difference between previous experience and short bio. +Laura Creighton is a founder of the Swedish company Open End AB, +which develops products in python and was a member of the consortium +which received EU Sixth Framework IST funding to develop PyPy. Previous experience =================== From exarkun at codespeak.net Mon Nov 19 02:43:26 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Mon, 19 Nov 2007 02:43:26 +0100 (CET) Subject: [pypy-svn] r48780 - pypy/dist/pypy/module/posix Message-ID: <20071119014326.0F0CB8141@code0.codespeak.net> Author: exarkun Date: Mon Nov 19 02:43:24 2007 New Revision: 48780 Modified: pypy/dist/pypy/module/posix/__init__.py Log: expose os.setuid and os.setgid Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Nov 19 02:43:24 2007 @@ -88,6 +88,10 @@ interpleveldefs['geteuid'] = 'interp_posix.geteuid' if hasattr(os, 'getgid'): interpleveldefs['getgid'] = 'interp_posix.getgid' + if hasattr(os, 'setuid'): + interpleveldefs['setuid'] = 'interp_posix.setuid' + if hasattr(os, 'setgid'): + interpleveldefs['setgid'] = 'interp_posix.setgid' # not visible via os, inconsistency in nt: if hasattr(posix, '_getfullpathname'): interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' From cfbolz at codespeak.net Mon Nov 19 12:16:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 12:16:47 +0100 (CET) Subject: [pypy-svn] r48781 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119111647.0D0D38133@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 12:16:46 2007 New Revision: 48781 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/floattype.py pypy/branch/ropes-unicode/pypy/objspace/std/inttype.py pypy/branch/ropes-unicode/pypy/objspace/std/longtype.py pypy/branch/ropes-unicode/pypy/objspace/std/model.py pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/stringobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py Log: try to fix ropes that represent unicode chars Modified: pypy/branch/ropes-unicode/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/floattype.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/floattype.py Mon Nov 19 12:16:46 2007 @@ -19,7 +19,10 @@ raise OperationError(space.w_ValueError, space.wrap(e.msg)) elif space.is_true(space.isinstance(w_value, space.w_unicode)): - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: if USE_NEW_S2F: Modified: pypy/branch/ropes-unicode/pypy/objspace/std/inttype.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/inttype.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/inttype.py Mon Nov 19 12:16:46 2007 @@ -66,7 +66,10 @@ except ParseStringOverflowError, e: w_longval = retry_to_w_long(space, e.parser) elif space.is_true(space.isinstance(w_value, space.w_unicode)): - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w string = unicode_to_decimal_w(space, w_value) try: value = string_to_int(string) @@ -95,7 +98,10 @@ base = space.int_w(w_base) if space.is_true(space.isinstance(w_value, space.w_unicode)): - from pypy.objspace.std.unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w s = unicode_to_decimal_w(space, w_value) else: try: Modified: pypy/branch/ropes-unicode/pypy/objspace/std/longtype.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/longtype.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/longtype.py Mon Nov 19 12:16:46 2007 @@ -18,7 +18,10 @@ space.wrap(e.msg)) elif space.is_true(space.isinstance(w_value, space.w_unicode)): try: - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w w_value = string_to_w_long(space, unicode_to_decimal_w(space, w_value)) except ParseStringError, e: raise OperationError(space.w_ValueError, Modified: pypy/branch/ropes-unicode/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/model.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/model.py Mon Nov 19 12:16:46 2007 @@ -17,6 +17,8 @@ "withmultilist" : ["listmultiobject.W_ListMultiObject"], "withrope" : ["ropeobject.W_RopeObject", "ropeobject.W_RopeIterObject"], + "withropeunicode": ["ropeunicodeobject.W_RopeUnicodeObject", + "ropeunicodeobject.W_RopeUnicodeIterObject"], "withrangelist" : ["rangeobject.W_RangeListObject", "rangeobject.W_RangeIterObject"], "withtproxy" : ["proxyobject.W_TransparentList", @@ -68,6 +70,7 @@ from pypy.objspace.std import listmultiobject from pypy.objspace.std import stringobject from pypy.objspace.std import ropeobject + from pypy.objspace.std import ropeunicodeobject from pypy.objspace.std import strsliceobject from pypy.objspace.std import strjoinobject from pypy.objspace.std import typeobject @@ -186,9 +189,15 @@ (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), ] else: - self.typeorder[ropeobject.W_RopeObject] += [ - (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), - ] + if config.objspace.std.withropeunicode: + self.typeorder[ropeobject.W_RopeObject] += [ + (ropeunicodeobject.W_RopeUnicodeObject, + ropeunicodeobject.delegate_Rope2RopeUnicode), + ] + else: + self.typeorder[ropeobject.W_RopeObject] += [ + (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), + ] if config.objspace.std.withstrslice: self.typeorder[strsliceobject.W_StringSliceObject] += [ Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Mon Nov 19 12:16:46 2007 @@ -151,7 +151,7 @@ return self.s[index] def getunichar(self, index): - return unicode(self.s[index]) + return unichr(ord(self.s[index])) def getint(self, index): return ord(self.s[index]) @@ -404,12 +404,18 @@ start, stop, node = find_straddling(node, start, stop) iter = SeekableItemIterator(node) iter.seekforward(start) - #XXX doesn't work for unicode - result = [iter.nextchar()] - for i in range(slicelength - 1): - iter.seekforward(step - 1) - result.append(iter.nextchar()) - return rope_from_charlist(result) + if node.is_bytestring(): + result = [iter.nextchar()] + for i in range(slicelength - 1): + iter.seekforward(step - 1) + result.append(iter.nextchar()) + return rope_from_charlist(result) + else: + result = [iter.nextunichar()] + for i in range(slicelength - 1): + iter.seekforward(step - 1) + result.append(iter.nextunichar()) + return rope_from_unicharlist(result) return getslice_one(node, start, stop) def getslice_one(node, start, stop): @@ -589,32 +595,61 @@ def rope_from_unicharlist(charlist): nodelist = [] length = len(charlist) - if length: + if not length: return LiteralStringNode.EMPTY i = 0 while i < length: - chunk = [] + unichunk = [] while i < length: c = ord(charlist[i]) if c < 256: break - chunk.append(unichr(c)) + unichunk.append(unichr(c)) i += 1 - if chunk: - nodelist.append(LiteralUnicodeNode("".join(chunk))) - chunck = [] + if unichunk: + nodelist.append(LiteralUnicodeNode("".join(unichunk))) + strchunk = [] while i < length: c = ord(charlist[i]) if c >= 256: break - chunk.append(chr(c)) + strchunk.append(chr(c)) i += 1 - if chunk: - nodelist.append(LiteralStringNode("".join(chunk))) + if strchunk: + nodelist.append(LiteralStringNode("".join(strchunk))) return rebalance(nodelist, length) -rope_from_unicharlist._annspecialcase_ = "specialize:argtype(0)" -rope_from_unicode = rope_from_unicharlist +def rope_from_unicode(uni): + nodelist = [] + length = len(uni) + if not length: + return LiteralStringNode.EMPTY + i = 0 + while i < length: + start = i + while i < length: + c = ord(uni[i]) + if c < 256: + break + i += 1 + if i != start: + nodelist.append(LiteralUnicodeNode(uni[start:i])) + start = i + strchunk = [] + while i < length: + c = ord(uni[i]) + if c >= 256: + break + i += 1 + if i != start: + nodelist.append(LiteralStringNode(uni[start:i].encode("latin-1"))) + return rebalance(nodelist, length) + +def rope_from_unichar(unichar): + intval = ord(unichar) + if intval > 256: + return LiteralUnicodeNode(unichar) + return LiteralStringNode.PREBUILT[intval] # __________________________________________________________________________ # searching @@ -881,12 +916,27 @@ class ItemIterator(object): - def __init__(self, node): + def __init__(self, node, start=0): self.iter = FringeIterator(node) self.node = None self.nodelength = 0 self.index = 0 - + if start: + self._advance_to(start) + + def _advance_to(self, index): + assert index > 0 + assert self.index == 0 + while 1: + node = self.iter.next() + length = node.length() + if index < length: + self.index = index + self.node = node + self.nodelength = length + break + index -= length + assert index >= 0 def getnode(self): node = self.node @@ -1160,8 +1210,7 @@ stop = start + prefix.length() if stop > end: return False - iter1 = SeekableItemIterator(self) - iter1.seekforward(start) + iter1 = ItemIterator(self, start) iter2 = ItemIterator(prefix) for i in range(prefix.length()): if iter1.nextint() != iter2.nextint(): @@ -1176,15 +1225,15 @@ begin = end - suffix.length() if begin < start: return False - iter1 = SeekableItemIterator(self) - iter1.seekforward(begin) + iter1 = ItemIterator(self, begin) iter2 = ItemIterator(suffix) for i in range(suffix.length()): if iter1.nextint() != iter2.nextint(): return False return True -def strip(node, left=True, right=True, predicate=lambda i: chr(i).isspace()): +def strip(node, left=True, right=True, predicate=lambda i: chr(i).isspace(), + *extraargs): length = node.length() lpos = 0 @@ -1192,12 +1241,12 @@ if left: iter = ItemIterator(node) - while lpos < rpos and predicate(iter.nextint()): + while lpos < rpos and predicate(iter.nextint(), *extraargs): lpos += 1 if right: iter = ReverseItemIterator(node) - while rpos > lpos and predicate(iter.nextint()): + while rpos > lpos and predicate(iter.nextint(), *extraargs): rpos -= 1 assert rpos >= lpos @@ -1219,6 +1268,76 @@ substrings.append(getslice_one(node, startidx, node.length())) return substrings + +def split_chars(node, maxsplit=-1, predicate=lambda x: chr(x).isspace()): + result = [] + length = node.length() + if not length: + return result + i = 0 + iter = ItemIterator(node) + while True: + # find the beginning of the next word + while i < length: + if not predicate(iter.nextint()): + break # found + i += 1 + else: + break # end of string, finished + + # find the end of the word + if maxsplit == 0: + j = length # take all the rest of the string + else: + j = i + 1 + while j < length and not predicate(iter.nextint()): + j += 1 + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + # the word is value[i:j] + result.append(getslice_one(node, i, j)) + + # continue to look from the character following the space after the word + i = j + 1 + return result + + +def rsplit_chars(node, maxsplit=-1, predicate=lambda x: chr(x).isspace()): + result = [] + length = node.length() + i = length - 1 + iter = ReverseItemIterator(node) + while True: + # starting from the end, find the end of the next word + while i >= 0: + if not predicate(iter.nextint()): + break # found + i -= 1 + else: + break # end of string, finished + + # find the start of the word + # (more precisely, 'j' will be the space character before the word) + if maxsplit == 0: + j = -1 # take all the rest of the string + else: + j = i - 1 + while j >= 0 and not predicate(iter.nextint()): + j -= 1 + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + # the word is value[j+1:i+1] + j1 = j + 1 + assert j1 >= 0 + result.append(getslice_one(node, j1, i + 1)) + + # continue to look from the character before the space before the word + i = j - 1 + + result.reverse() + return result + + def split_completely(node, maxsplit=-1): upper = node.length() if maxsplit > 0 and maxsplit < upper + 2: @@ -1231,6 +1350,53 @@ substrings.append(rope.getslice_one(node, upper, length)) +def splitlines(node, keepends=False): + length = node.length() + if length == 0: + return [] + + result = [] + iter = ItemIterator(node) + i = j = 0 + last = ord(" ") + char = iter.nextint() + while i < length: + # Find a line and append it + while char != ord('\n') and char != ord('\r'): + try: + i += 1 + last = char + char = iter.nextint() + except StopIteration: + break + # Skip the line break reading CRLF as one line break + eol = i + i += 1 + last = char + try: + char = iter.nextint() + except StopIteration: + pass + else: + if last == ord('\r') and char == ord('\n'): + i += 1 + try: + last = char + char = iter.nextint() + except StopIteration: + pass + if keepends: + eol = i + result.append(getslice_one(node, j, eol)) + j = i + + if j == 0: + result.append(node) + elif j < length: + result.append(getslice_one(node, j, length)) + + return result + # __________________________________________________________________________ # misc @@ -1247,54 +1413,65 @@ # ____________________________________________________________ # to and from unicode conversion -def str_decode(rope, encoding): +def str_decode_ascii(rope): assert rope.is_bytestring() - if encoding == "ascii": - if rope.is_ascii(): - return rope - elif encoding == "latin-1": + if rope.is_ascii(): + return rope + return None + +def str_decode_latin1(rope): + assert rope.is_bytestring() + return rope + +def str_decode_utf8(rope): + from pypy.rlib.runicode import str_decode_utf_8 + if rope.is_ascii(): return rope - elif encoding == "utf-8": - from pypy.rlib.runicode import str_decode_utf_8 - if rope.is_ascii(): - return rope - elif isinstance(rope, BinaryConcatNode): - lresult = str_decode(rope.left, "utf-8") - if result is not None: - return BinaryConcatNode(lresult, - str_decode(rope.right, "utf-8")) - elif isinstance(rope, LiteralStringNode): + elif isinstance(rope, BinaryConcatNode): + lresult = str_decode_utf8(rope.left) + if lresult is not None: + return BinaryConcatNode(lresult, + str_decode_utf8(rope.right)) + elif isinstance(rope, LiteralStringNode): + try: result, consumed = str_decode_utf_8(rope.s, len(rope.s), False, "strict") - if consumed < len(rope.s): - return None - return rope_from_unicode(result) - s = rope.flatten_string() - return str_decode_utf_8(s, len(s), True) - else: - raise NotImplementedError("unknown encoding") + except UnicodeDecodeError: + return None + if consumed < len(rope.s): + return None + return rope_from_unicode(result) + s = rope.flatten_string() + try: + result, consumed = str_decode_utf_8(s, len(s), True) + return rope_from_unicode(result) + except UnicodeDecodeError: + pass + -def unicode_encode(rope, encoding): - if encoding == "ascii": - if rope.is_ascii(): - return rope - elif encoding == "latin-1": +def unicode_encode_ascii(rope): + if rope.is_ascii(): return rope - elif encoding == "utf-8": - from pypy.rlib.runicode import unicode_encode_utf_8 - if rope.is_ascii(): - return rope - elif isinstance(rope, BinaryConcatNode): - return BinaryConcatNode(unicode_encode(rope.left, "utf-8"), - unicode_encode(rope.right, "utf-8")) - elif isinstance(rope, LiteralUnicodeNode): - return unicode_encode_utf_8(rope.u, len(rope.u), "strict") - elif isinstance(rope, LiteralStringNode): - return LiteralStringNode(_str_encode_utf_8(rope.s)) - s = rope.flatten_string() - return str_decode_utf_8(s, len(s), True) - else: - raise NotImplementedError("unknown encoding") + +def unicode_encode_latin1(rope): + if rope.is_bytestring(): + return rope + +def unicode_encode_utf8(rope): + from pypy.rlib.runicode import unicode_encode_utf_8 + if rope.is_ascii(): + return rope + elif isinstance(rope, BinaryConcatNode): + return BinaryConcatNode(unicode_encode_utf8(rope.left), + unicode_encode_utf8(rope.right)) + elif isinstance(rope, LiteralUnicodeNode): + try: + return LiteralStringNode( + unicode_encode_utf_8(rope.u, len(rope.u), "strict")) + except UnicodeDecodeError: + return None + elif isinstance(rope, LiteralStringNode): + return LiteralStringNode(_str_encode_utf_8(rope.s)) def _str_encode_utf_8(s): size = len(s) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py Mon Nov 19 12:16:46 2007 @@ -219,41 +219,14 @@ return W_RopeObject(rope.rope_from_charlist(buffer)) def str_split__Rope_None_ANY(space, w_self, w_none, w_maxsplit=-1): + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - res_w = [] - node = w_self._node - length = node.length() - i = 0 - iter = rope.ItemIterator(node) - while True: - # find the beginning of the next word - while i < length: - if not iter.nextchar().isspace(): - break # found - i += 1 - else: - break # end of string, finished - - # find the end of the word - if maxsplit == 0: - j = length # take all the rest of the string - else: - j = i + 1 - while j < length and not iter.nextchar().isspace(): - j += 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[i:j] - res_w.append(W_RopeObject(rope.getslice_one(node, i, j))) - - # continue to look from the character following the space after the word - i = j + 1 - + res_w = [W_RopeObject(node) + for node in rope.split_chars(selfnode, maxsplit)] return space.newlist(res_w) def str_split__Rope_Rope_ANY(space, w_self, w_by, w_maxsplit=-1): maxsplit = space.int_w(w_maxsplit) - start = 0 selfnode = w_self._node bynode = w_by._node bylen = bynode.length() @@ -263,42 +236,15 @@ for node in rope.split(selfnode, bynode, maxsplit)] return space.newlist(res_w) + def str_rsplit__Rope_None_ANY(space, w_self, w_none, w_maxsplit=-1): - # XXX works but flattens + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - res_w = [] - value = w_self._node.flatten_string() - i = len(value)-1 - while True: - # starting from the end, find the end of the next word - while i >= 0: - if not value[i].isspace(): - break # found - i -= 1 - else: - break # end of string, finished - - # find the start of the word - # (more precisely, 'j' will be the space character before the word) - if maxsplit == 0: - j = -1 # take all the rest of the string - else: - j = i - 1 - while j >= 0 and not value[j].isspace(): - j -= 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[j+1:i+1] - j1 = j + 1 - assert j1 >= 0 - res_w.append(space.wrap(value[j1:i+1])) - - # continue to look from the character before the space before the word - i = j - 1 - - res_w.reverse() + res_w = [W_RopeObject(node) + for node in rope.rsplit_chars(selfnode, maxsplit)] return space.newlist(res_w) + def str_rsplit__Rope_Rope_ANY(space, w_self, w_by, w_maxsplit=-1): # XXX works but flattens maxsplit = space.int_w(w_maxsplit) @@ -498,7 +444,7 @@ substrings = [by] iter = rope.ItemIterator(node) for i in range(upper): - substrings.append(iter.nextrope()]) + substrings.append(iter.nextrope()) substrings.append(by) substrings.append(rope.getslice_one(node, upper, length)) try: @@ -515,43 +461,27 @@ raise OperationError(space.w_OverflowError, space.wrap("string too long")) -def _strip(space, w_self, w_chars, left, right): - "internal function called by str_xstrip methods" - node = w_self._node - length = node.length() - u_chars = w_chars._node.flatten_string() - - lpos = 0 - rpos = length - - if left: - #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.ItemIterator(node) - while lpos < rpos and iter.nextchar() in u_chars: - lpos += 1 - - if right: - iter = rope.ReverseItemIterator(node) - while rpos > lpos and iter.nextchar() in u_chars: - rpos -= 1 - - return W_RopeObject(rope.getslice_one(node, lpos, rpos)) +def _contains(i, string): + return chr(i) in string def str_strip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=1, right=1) + return W_RopeObject(rope.strip(w_self._node, True, True, + _contains, w_chars._node.flatten_string())) def str_strip__Rope_None(space, w_self, w_chars): return W_RopeObject(rope.strip(w_self._node, left=True, right=True)) def str_rstrip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=0, right=1) + return W_RopeObject(rope.strip(w_self._node, False, True, + _contains, w_chars._node.flatten_string())) def str_rstrip__Rope_None(space, w_self, w_chars): - return W_RopeObject(rope.strip(w_self._node, left=False, right=True)) + return W_RopeObject(rope.strip(w_self._node, False, True)) def str_lstrip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=1, right=0) + return W_RopeObject(rope.strip(w_self._node, True, False, + _contains, w_chars._node.flatten_string())) def str_lstrip__Rope_None(space, w_self, w_chars): return W_RopeObject(rope.strip(w_self._node, left=True, right=False)) @@ -606,6 +536,10 @@ (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): + if space.is_true(space.isinstance(w_suffix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "endswith", w_suffixes, w_start, + w_end) suffix = rope_w(space, w_suffix) if rope.endswith(self, suffix, start, end): return space.w_True @@ -621,6 +555,10 @@ (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): + if space.is_true(space.isinstance(w_prefix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "startswith", w_prefixes, w_start, + w_end) prefix = rope_w(space, w_prefix) if rope.startswith(self, prefix, start, end): return space.w_True @@ -673,53 +611,10 @@ def str_splitlines__Rope_ANY(space, w_self, w_keepends): - keepends = bool(space.int_w(w_keepends)) # truth value, but type checked + keepends = bool(space.int_w(w_keepends)) # truth value, but type checked node = w_self._node - length = node.length() - if length == 0: - return space.newlist([]) - - strs_w = [] - iter = rope.ItemIterator(node) - i = j = 0 - last = " " - char = iter.nextchar() - while i < length: - # Find a line and append it - while char != '\n' and char != '\r': - try: - i += 1 - last = char - char = iter.nextchar() - except StopIteration: - break - # Skip the line break reading CRLF as one line break - eol = i - i += 1 - last = char - try: - char = iter.nextchar() - except StopIteration: - pass - else: - if last == '\r' and char == '\n': - i += 1 - try: - last = char - char = iter.nextchar() - except StopIteration: - pass - if keepends: - eol = i - strs_w.append(W_RopeObject(rope.getslice_one(node, j, eol))) - j = i - - if j == 0: - strs_w.append(w_self.create_if_subclassed()) - elif j < length: - strs_w.append(W_RopeObject(rope.getslice_one(node, j, length))) - - return space.newlist(strs_w) + return space.newlist( + [W_RopeObject(n) for n in rope.splitlines(node, keepends)]) def str_zfill__Rope_ANY(space, w_self, w_width): node = w_self._node @@ -839,8 +734,8 @@ return w_str return W_RopeObject(w_str._node) -def iter__Rope(space, w_list): - return W_RopeIterObject(w_list) +def iter__Rope(space, w_str): + return W_RopeIterObject(w_str) def ord__Rope(space, w_str): node = w_str._node @@ -934,11 +829,12 @@ def str_decode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ - unicode_from_string, decode_object + getdefaultencoding + from pypy.objspace.std.ropeunicodeobject import decode_string encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - if encoding is None and errors is None: - return unicode_from_string(space, w_string) - return decode_object(space, w_string, encoding, errors) + if encoding is None: + encoding = getdefaultencoding(space) + return decode_string(space, w_string, encoding, errors) def str_encode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 12:16:46 2007 @@ -19,23 +19,47 @@ return W_RopeUnicodeObject(rope.rope_from_unicode(uni)) def unicode_from_string(space, w_str): - from pypy.objspace.std.unicodetype import getdefaultencoding, \ - unicode_from_encoded_object + from pypy.objspace.std.unicodetype import getdefaultencoding assert isinstance(w_str, W_RopeObject) - node = w_str._node encoding = getdefaultencoding(space) - if encoding == 'ascii': - result = rope.str_decode_ascii(node) - if result is not None: - return W_RopeUnicodeObject(result) - elif encoding == 'latin-1': - assert node.is_bytestring() - return W_RopeUnicodeObject(node) - elif encoding == "utf-8": - result = rope.str_decode_utf8(node) - if result is not None: - return W_RopeUnicodeObject(result) - return unicode_from_encoded_object(space, w_str, encoding, "strict") + return decode_string(space, w_str, encoding, "strict") + +def decode_string(space, w_str, encoding, errors): + from pypy.objspace.std.unicodetype import unicode_from_encoded_object + if errors is None or errors == "strict": + node = w_str._node + if encoding == 'ascii': + result = rope.str_decode_ascii(node) + if result is not None: + return W_RopeUnicodeObject(result) + elif encoding == 'latin-1': + assert node.is_bytestring() + return W_RopeUnicodeObject(node) + elif encoding == "utf-8": + result = rope.str_decode_utf8(node) + if result is not None: + return W_RopeUnicodeObject(result) + return unicode_from_encoded_object(space, w_str, encoding, errors) + +def encode_unicode(space, w_unistr, encoding, errors): + from pypy.objspace.std.unicodetype import getdefaultencoding, \ + _get_encoding_and_errors, encode_object + from pypy.objspace.std.ropeobject import W_RopeObject + if errors is None or errors == "strict": + node = w_unistr._node + if encoding == 'ascii': + result = rope.unicode_encode_ascii(node) + if result is not None: + return W_RopeObject(result) + elif encoding == 'latin-1': + result = rope.unicode_encode_latin1(node) + if result is not None: + return W_RopeObject(result) + elif encoding == "utf-8": + result = rope.unicode_encode_utf8(node) + if result is not None: + return W_RopeObject(result) + return encode_object(space, w_unistr, encoding, errors) class W_RopeUnicodeObject(W_Object): @@ -55,20 +79,34 @@ def create_if_subclassed(w_self): if type(w_self) is W_RopeUnicodeObject: return w_self - return W_RopeUnicodeObject(w_self._value) + return W_RopeUnicodeObject(w_self._node) W_RopeUnicodeObject.EMPTY = W_RopeUnicodeObject(rope.LiteralStringNode.EMPTY) registerimplementation(W_RopeUnicodeObject) +def _isspace(uchar_ord): + return unicodedb.isspace(uchar_ord) def ropeunicode_w(space, w_str): if isinstance(w_str, W_RopeUnicodeObject): return w_str._node - # XXX do the right thing for W_RopeObject + if isinstance(w_str, W_RopeObject): + return unicode_from_string(space, w_str)._node return rope.LiteralUnicodeNode(space.unicode_w(w_str)) +class W_RopeUnicodeIterObject(W_Object): + from pypy.objspace.std.itertype import iter_typedef as typedef + + def __init__(w_self, w_rope, index=0): + w_self.node = node = w_rope._node + w_self.item_iter = rope.ItemIterator(node) + w_self.index = index + +def iter__RopeUnicode(space, w_uni): + return W_RopeUnicodeIterObject(w_uni) + # Helper for converting int/long def unicode_to_decimal_w(space, w_unistr): if not isinstance(w_unistr, W_RopeUnicodeObject): @@ -145,7 +183,7 @@ def ord__RopeUnicode(space, w_uni): - if w_uni.length() != 1: + if w_uni._node.length() != 1: raise OperationError(space.w_TypeError, space.wrap('ord() expected a character')) return space.wrap(w_uni._node.getint(0)) @@ -177,7 +215,7 @@ def unicode_join__RopeUnicode_ANY(space, w_self, w_list): l_w = space.unpackiterable(w_list) - delim = w_self._value + delim = w_self._node totlen = 0 if len(l_w) == 0: return W_RopeUnicodeObject.EMPTY @@ -199,7 +237,7 @@ raise OperationError(space.w_TypeError, w_msg) values_list.append(item) try: - return W_RopeUnicodeObject(rope.join(self, values_list)) + return W_RopeUnicodeObject(rope.join(w_self._node, values_list)) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) @@ -223,7 +261,7 @@ return W_RopeUnicodeObject(uni.getrope(ival)) def getitem__RopeUnicode_Slice(space, w_uni, w_slice): - node = w_uni._noed + node = w_uni._node length = node.length() start, stop, step, sl = w_slice.indices4(space, length) if sl == 0: @@ -239,7 +277,7 @@ raise node = w_uni._node try: - return W_RopeUnicodeObject(rope.multiply(node, mul)) + return W_RopeUnicodeObject(rope.multiply(node, times)) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) @@ -317,47 +355,33 @@ previous_is_cased = False -def _strip(space, w_self, w_chars, left, right): - "internal function called by str_xstrip methods" - XXX - u_self = w_self._value - u_chars = w_chars._value - - lpos = 0 - rpos = len(u_self) - - if left: - while lpos < rpos and u_self[lpos] in u_chars: - lpos += 1 - - if right: - while rpos > lpos and u_self[rpos - 1] in u_chars: - rpos -= 1 - - assert rpos >= 0 - result = u_self[lpos: rpos] - return W_UnicodeObject(result) +def _contains(i, uni): + return unichr(i) in uni def unicode_strip__RopeUnicode_None(space, w_self, w_chars): - return W_RopeUnicodeObject(rope.strip(w_self._none, True, True, _isspace)) + return W_RopeUnicodeObject(rope.strip(w_self._node, True, True, _isspace)) def unicode_strip__RopeUnicode_RopeUnicode(space, w_self, w_chars): - return _strip(space, w_self, w_chars, 1, 1) + return W_RopeUnicodeObject(rope.strip(w_self._node, True, True, _contains, + w_chars._node.flatten_unicode())) + def unicode_strip__RopeUnicode_Rope(space, w_self, w_chars): return space.call_method(w_self, 'strip', unicode_from_string(space, w_chars)) def unicode_lstrip__RopeUnicode_None(space, w_self, w_chars): - return W_RopeUnicodeObject(rope.strip(w_self._none, True, False, _isspace)) + return W_RopeUnicodeObject(rope.strip(w_self._node, True, False, _isspace)) def unicode_lstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars): - return _strip(space, w_self, w_chars, 1, 0) + return W_RopeUnicodeObject(rope.strip(w_self._node, True, False, _contains, + w_chars._node.flatten_unicode())) def unicode_lstrip__RopeUnicode_Rope(space, w_self, w_chars): return space.call_method(w_self, 'lstrip', unicode_from_string(space, w_chars)) def unicode_rstrip__RopeUnicode_None(space, w_self, w_chars): - return W_RopeUnicodeObject(rope.strip(w_self._none, False, True, _isspace)) + return W_RopeUnicodeObject(rope.strip(w_self._node, False, True, _isspace)) def unicode_rstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars): - return _strip(space, w_self, w_chars, 0, 1) + return W_RopeUnicodeObject(rope.strip(w_self._node, False, True, _contains, + w_chars._node.flatten_unicode())) def unicode_rstrip__RopeUnicode_Rope(space, w_self, w_chars): return space.call_method(w_self, 'rstrip', unicode_from_string(space, w_chars)) @@ -383,7 +407,7 @@ iter = rope.ItemIterator(input) previous_is_cased = False - for i in range(len(input)): + for i in range(input.length()): unichar = iter.nextint() if previous_is_cased: result[i] = unichr(unicodedb.tolower(unichar)) @@ -436,30 +460,6 @@ return (self, start, end) -def _check_startswith_substring(str, substr, start, end): - XXX - substr_len = len(substr) - - if end - start < substr_len: - return False # substring is too long - - for i in range(substr_len): - if str[start + i] != substr[i]: - return False - return True - -def _check_endswith_substring(str, substr, start, end): - XXX - substr_len = len(substr) - - if end - start < substr_len: - return False # substring is too long - start = end - substr_len - for i in range(substr_len): - if str[start + i] != substr[i]: - return False - return True - def unicode_endswith__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) return space.newbool(rope.endswith(self, w_substr._node, start, end)) @@ -477,8 +477,8 @@ w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): - prefix = unicoderope_w(space, w_prefix) - if rope.startswith(self, prefxix, start, end) + prefix = ropeunicode_w(space, w_prefix) + if rope.startswith(unistr, prefix, start, end): return space.w_True return space.w_False @@ -486,8 +486,8 @@ w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): - suffix = unicoderope_w(space, w_suffix) - if _check_endswith_substring(unistr, suffix, start, end): + suffix = ropeunicode_w(space, w_suffix) + if rope.endswith(unistr, suffix, start, end): return space.w_True return space.w_False @@ -508,7 +508,7 @@ def unicode_center__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._node - length = self.length + length = self.length() width = space.int_w(w_width) fillchar = _to_unichar_w(space, w_fillchar) padding = width - length @@ -517,82 +517,57 @@ offset = padding // 2 pre = rope.multiply(fillchar, offset) post = rope.multiply(fillchar, (padding - offset)) - centered = rope.rebalance([pre, node, post]) + centered = rope.rebalance([pre, self, post]) return W_RopeUnicodeObject(centered) def unicode_ljust__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._node - length = self.length + length = self.length() width = space.int_w(w_width) fillchar = _to_unichar_w(space, w_fillchar) padding = width - length if padding < 0: return w_self.create_if_subclassed() - resultnode = rope.concatenate(self, rope.multiply(fillchar, d)) + resultnode = rope.concatenate(self, rope.multiply(fillchar, padding)) return W_RopeUnicodeObject(resultnode) def unicode_rjust__RopeUnicode_ANY_ANY(space, w_self, w_width, w_fillchar): self = w_self._node - length = self.length + length = self.length() width = space.int_w(w_width) fillchar = _to_unichar_w(space, w_fillchar) padding = width - length if padding < 0: return w_self.create_if_subclassed() - resultnode = rope.concatenate(rope.multiply(fillchar, d), self) + resultnode = rope.concatenate(rope.multiply(fillchar, padding), self) return W_RopeUnicodeObject(resultnode) def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width): - self = w_self._value - length = self.length + self = w_self._node + length = self.length() width = space.int_w(w_width) + zero = rope.LiteralStringNode.PREBUILT[ord("0")] if self.length() == 0: return W_RopeUnicodeObject( - rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], width)) - padding = width - self.length() + rope.multiply(zero, width)) + padding = width - length if padding <= 0: return w_self.create_if_subclassed() firstchar = self.getunichar(0) if firstchar in (u'+', u'-'): return W_RopeUnicodeObject(rope.rebalance( [rope.LiteralStringNode.PREBUILT[ord(firstchar)], - rope.multiply(zero, middle), - rope.getslice_one(node, 1, length)])) + rope.multiply(zero, padding), + rope.getslice_one(self, 1, length)])) else: - middle = width - length return W_RopeUnicodeObject(rope.concatenate( - rope.multiply(zero, middle), node)) + rope.multiply(zero, padding), self)) def unicode_splitlines__RopeUnicode_ANY(space, w_self, w_keepends): - XXX - self = w_self._value - keepends = 0 - if space.int_w(w_keepends): - keepends = 1 - if len(self) == 0: - return space.newlist([]) - - start = 0 - end = len(self) - pos = 0 - lines = [] - while pos < end: - if unicodedb.islinebreak(ord(self[pos])): - if (self[pos] == u'\r' and pos + 1 < end and - self[pos + 1] == u'\n'): - # Count CRLF as one linebreak - lines.append(W_UnicodeObject(self[start:pos + keepends * 2])) - pos += 1 - else: - lines.append(W_UnicodeObject(self[start:pos + keepends])) - pos += 1 - start = pos - else: - pos += 1 - if not unicodedb.islinebreak(ord(self[end - 1])): - lines.append(W_UnicodeObject(self[start:])) - return space.newlist(lines) - + keepends = bool(space.int_w(w_keepends)) # truth value, but type checked + node = w_self._node + return space.newlist( + [W_RopeUnicodeObject(n) for n in rope.splitlines(node, keepends)]) def unicode_find__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) @@ -629,86 +604,44 @@ def unicode_count__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) - substr = w_substr._value - return space.wrap(self.count(substr, start, end)) + assert start >= 0 + assert end >= 0 + iter = rope.FindIterator(self, w_substr._node, start, end) + i = 0 + while 1: + try: + index = iter.next() + except StopIteration: + break + i += 1 + return wrapint(space, i) def unicode_split__RopeUnicode_None_ANY(space, w_self, w_none, w_maxsplit): - self = w_self._value + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - parts = [] - if len(self) == 0: - return space.newlist([]) - start = 0 - end = len(self) - inword = 0 - - while maxsplit != 0 and start < end: - index = start - for index in range(start, end): - if _isspace(self[index]): - break - else: - inword = 1 - else: - break - if inword == 1: - parts.append(W_UnicodeObject(self[start:index])) - maxsplit -= 1 - # Eat whitespace - for start in range(index + 1, end): - if not _isspace(self[start]): - break - else: - return space.newlist(parts) - - parts.append(W_UnicodeObject(self[start:])) - return space.newlist(parts) + res_w = [W_RopeUnicodeObject(node) + for node in rope.split_chars(selfnode, maxsplit, _isspace)] + return space.newlist(res_w) def unicode_split__RopeUnicode_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): maxsplit = space.int_w(w_maxsplit) start = 0 selfnode = w_self._node - bynode = w_by._node - bylen = bynode.length() - if bylen == 0: + delimnode = w_delim._node + delimlen = delimnode.length() + if delimlen == 0: raise OperationError(space.w_ValueError, space.wrap("empty separator")) res_w = [W_RopeUnicodeObject(node) - for node in rope.split(selfnode, bynode, maxsplit)] + for node in rope.split(selfnode, delimnode, maxsplit)] return space.newlist(res_w) def unicode_rsplit__RopeUnicode_None_ANY(space, w_self, w_none, w_maxsplit): - XXX - self = w_self._value + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - parts = [] - if len(self) == 0: - return space.newlist([]) - start = 0 - end = len(self) - inword = 0 - - while maxsplit != 0 and start < end: - index = end - for index in range(end-1, start-1, -1): - if _isspace(self[index]): - break - else: - inword = 1 - else: - break - if inword == 1: - parts.append(W_UnicodeObject(self[index+1:end])) - maxsplit -= 1 - # Eat whitespace - for end in range(index, start-1, -1): - if not _isspace(self[end-1]): - break - else: - return space.newlist(parts) + res_w = [W_RopeUnicodeObject(node) + for node in rope.rsplit_chars(selfnode, maxsplit, _isspace)] + return space.newlist(res_w) - parts.append(W_UnicodeObject(self[:end])) - parts.reverse() - return space.newlist(parts) def unicode_rsplit__RopeUnicode_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit): # XXX works but flattens @@ -733,7 +666,7 @@ end = index maxsplit -= 1 parts.append(W_RopeUnicodeObject( - rope.getslice_one(w_self._node, 0, :end))) + rope.getslice_one(w_self._node, 0, end))) parts.reverse() return space.newlist(parts) @@ -741,7 +674,7 @@ if maxsplit == 0: return [self] index = 0 - end = len(self) + end = self.length() parts = [rope.LiteralStringNode.EMPTY] maxsplit -= 1 while maxsplit != 0: @@ -750,50 +683,40 @@ parts.append(self.getrope(index)) index += 1 maxsplit -= 1 - parts.append(rope.getslice_one(self, index, self.length()) + parts.append(rope.getslice_one(self, index, self.length())) return parts def unicode_replace__RopeUnicode_RopeUnicode_RopeUnicode_ANY( space, w_self, w_old, w_new, w_maxsplit): + self = w_self._node old = w_old._node - oldlength = old.length + maxsplit = space.int_w(w_maxsplit) + oldlength = old.length() if not oldlength: - self = w_self._value - maxsplit = space.int_w(w_maxsplit) parts = _split_into_chars(self, maxsplit) return W_RopeUnicodeObject(rope.join(w_new._node, parts)) - substrings = rope.split(node, old, maxsplit) + substrings = rope.split(self, old, maxsplit) if not substrings: return w_self.create_if_subclassed() - substrings = rope.split(node, sub, maxsplit) - if substrings is None: - return w_self.create_if_subclassed() try: - return W_RopeObject(rope.join(by, substrings)) + return W_RopeUnicodeObject(rope.join(w_new._node, substrings)) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) - try: - return W_RopeObject(rope.join(by, substrings)) - except OverflowError: - raise OperationError(space.w_OverflowError, - space.wrap("string too long")) - return W_UnicodeObject(w_new._value.join(parts)) -def unicode_encode__Unicode_ANY_ANY(space, w_unistr, - w_encoding=None, - w_errors=None): +def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr, + w_encoding=None, + w_errors=None): from pypy.objspace.std.unicodetype import getdefaultencoding, \ - _get_encoding_and_errors, encode_object + _get_encoding_and_errors encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None: encoding = getdefaultencoding(space) - w_retval = encode_object(space, w_unistr, encoding, errors) - return w_retval + return encode_unicode(space, w_unistr, encoding, errors) -def unicode_partition__Unicode_Unicode(space, w_unistr, w_unisub): +def unicode_partition__RopeUnicode_RopeUnicode(space, w_unistr, w_unisub): self = w_unistr._node sub = w_unisub._node if not sub.length(): @@ -810,7 +733,7 @@ W_RopeUnicodeObject(rope.getslice_one(self, pos + sub.length(), self.length()))]) -def unicode_rpartition__Unicode_Unicode(space, w_unistr, w_unisub): +def unicode_rpartition__RopeUnicode_RopeUnicode(space, w_unistr, w_unisub): # XXX works but flattens unistr = w_unistr._node.flatten_unicode() unisub = w_unisub._node.flatten_unicode() @@ -819,8 +742,8 @@ space.wrap("empty separator")) pos = unistr.rfind(unisub) if pos == -1: - return space.newtuple([W_UnicodeObject.EMPTY, - W_UnicodeObject.EMPTY, w_unistr]) + return space.newtuple([W_RopeUnicodeObject.EMPTY, + W_RopeUnicodeObject.EMPTY, w_unistr]) else: assert pos > 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, @@ -828,9 +751,10 @@ def unicode_expandtabs__RopeUnicode_ANY(space, w_self, w_tabsize): + from pypy.objspace.std.ropeobject import _tabindent self = w_self._node tabsize = space.int_w(w_tabsize) - splitted = rope.split(node, rope.LiteralStringNode.PREBUILT[ord('\t')]) + splitted = rope.split(self, rope.LiteralStringNode.PREBUILT[ord('\t')]) last = splitted[0] expanded = [last] for i in range(1, len(splitted)): @@ -865,9 +789,9 @@ raise OperationError( space.w_TypeError, space.wrap("character mapping must be in range(0x%x)" % (maxunicode + 1,))) - result.append(unichr(newval)) + result.append(rope.rope_from_unichar(unichr(newval))) elif space.is_true(space.isinstance(w_newval, space.w_unicode)): - result.append(unicoderope_w(w_newval)) + result.append(ropeunicode_w(space, w_newval)) else: raise OperationError( space.w_TypeError, @@ -877,11 +801,11 @@ # Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)' def repr__RopeUnicode(space, w_unicode): hexdigits = "0123456789abcdef" - chars = w_unicode._node + node = w_unicode._node size = node.length() singlequote = doublequote = False - iter = rope.ItemIterator() + iter = rope.ItemIterator(node) for i in range(size): c = iter.nextunichar() if singlequote and doublequote: @@ -895,7 +819,7 @@ else: quote = '\'' result = ['u', quote] - iter = rope.ItemIterator() + iter = rope.ItemIterator(node) j = 0 while j < size: code = iter.nextint() @@ -915,6 +839,8 @@ if code >= 0xD800 and code < 0xDC00: if j < size - 1: code2 = iter.nextint() + # XXX this is wrong: if the next if is false, + # code2 is lost if code2 >= 0xDC00 and code2 <= 0xDFFF: code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000 result.extend(["U", @@ -975,6 +901,34 @@ return mod_format(space, w_format, w_values, do_unicode=True) +# methods of the iterator + +def iter__RopeUnicodeIter(space, w_ropeiter): + return w_ropeiter + +def next__RopeUnicodeIter(space, w_ropeiter): + if w_ropeiter.node is None: + raise OperationError(space.w_StopIteration, space.w_None) + try: + unichar = w_ropeiter.item_iter.nextunichar() + w_item = space.wrap(unichar) + except StopIteration: + w_ropeiter.node = None + w_ropeiter.char_iter = None + raise OperationError(space.w_StopIteration, space.w_None) + w_ropeiter.index += 1 + return w_item + +def len__RopeUnicodeIter(space, w_ropeiter): + if w_ropeiter.node is None: + return wrapint(space, 0) + index = w_ropeiter.index + length = w_ropeiter.node.length() + result = length - index + if result < 0: + return wrapint(space, 0) + return wrapint(space, result) + import unicodetype register_all(vars(), unicodetype) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/stringobject.py Mon Nov 19 12:16:46 2007 @@ -589,6 +589,10 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): + if space.is_true(space.isinstance(w_suffix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "endswith", w_suffixes, w_start, + w_end) suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -603,6 +607,10 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): + if space.is_true(space.isinstance(w_prefix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "startswith", w_prefixes, w_start, + w_end) prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Mon Nov 19 12:16:46 2007 @@ -129,6 +129,21 @@ for step in range(1, stop - start): assert getslice(s, start, stop, step).flatten_string() == result[start:stop:step] +def test_getslice_step_unicode(): + s1 = (LiteralUnicodeNode(u"\uaaaa") + + LiteralUnicodeNode(u"\ubbbb" * 5) + + LiteralUnicodeNode(u"\uaaaa\ubbbb\u1000\u2000") + + LiteralUnicodeNode(u"vwxyz") + + LiteralUnicodeNode(u"zyxwvu\u1234" * 2) + + LiteralUnicodeNode(u"12355")) + s2 = s1.rebalance() + result = s1.flatten_unicode() + assert s2.flatten_unicode() == result + for s in [s1, s2]: + for start in range(0, len(result)): + for stop in range(start, len(result)): + for step in range(1, stop - start): + assert getslice(s, start, stop, step).flatten_unicode() == result[start:stop:step] def test_random_addition_and_slicing(): seed = random.randrange(10000) @@ -181,6 +196,17 @@ assert c2 == ord(c) py.test.raises(StopIteration, iter.nextchar) +def test_iteration_startpos(): + rope, real_st = make_random_string(200) + for i in range(0, len(real_st), len(real_st) // 20): + iter = ItemIterator(rope, i) + x = i + for c in real_st[i:]: + x += 1 + c2 = iter.nextchar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + def test_iteration_unicode(): rope, real_st = make_random_string(200, unicode=True) iter = ItemIterator(rope) @@ -381,7 +407,6 @@ rope = getslice_one(rope, 10, 100) st = st[10:100] for i in range(len(st)): - print i for j in range(i + 1, len(st)): c = st[i:j][(j - i) // 2] pos = find_int(rope, ord(c), i, j) @@ -518,7 +543,6 @@ hashes[(h & 0xff0000) >> 16] += 1 for h in hashes: assert h > 300 - print hashes def test_hash_distribution_small_strings(): random.seed(42) # prevent randomly failing test @@ -581,7 +605,7 @@ assert s.hash_part() == h def test_hash_part_unicode(): - a, st = make_random_string(unicode=True) + a, st = make_random_string(5, unicode=True) h = a.hash_part() for split in range(1, len(st) - 1): s1 = LiteralUnicodeNode(st[:split]) @@ -714,3 +738,31 @@ assert len(l1) == len(l2) for n, s in zip(l1, l2): assert n.flatten_string() == s + +def test_splitlines(): + seps = [(LiteralStringNode("\n"), "\n"), (LiteralStringNode("\r"), "\r"), + (LiteralStringNode("\r\n"), "\r\n")] + l, strs = zip(*[(LiteralStringNode("xafnarsp"), "xafnarsp"), + (LiteralStringNode("xyzaaaa"), "xyzaaaa"), + (LiteralStringNode("wxxxx"), "wxxxx")]) + l = list(l) + for s, st in seps: + node = join(s, l) + l2 = splitlines(node) + for n1, n2 in zip(l, l2): + assert n1.flatten_string() == n2.flatten_string() + for keepends in [True, False]: + l1 = splitlines(LiteralStringNode("ab\nab\n\raba\rba"), keepends) + l2 = "ab\nab\n\raba\rba".splitlines(keepends) + assert len(l1) == len(l2) + for n, s in zip(l1, l2): + assert n.flatten_string() == s + +def test_rope_from_unicode(): + node = rope_from_unicode(u"aaabbbbbcccdddeeefffggggnnn") + assert node.is_bytestring() + assert node.is_ascii() + node = rope_from_unicode(u"a" * 30 + u"\ufffd" * 30 + "x" * 30) + assert node.length() == 90 + assert not node.is_ascii() + assert not node.is_bytestring() Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 12:16:46 2007 @@ -307,11 +307,10 @@ raises(TypeError, u'hello'.startswith, (42,)) def test_startswith_endswith_convert(self): - skip("fix me") assert 'hello'.startswith((u'he\u1111', u'he')) assert not 'hello'.startswith((u'lo\u1111', u'llo')) assert 'hello'.startswith((u'hellox\u1111', u'hello')) - assert 'hello'.startswith((u'lo', u'he\u1111'), 0, -1) + assert not 'hello'.startswith((u'lo', u'he\u1111'), 0, -1) assert not 'hello'.endswith((u'he\u1111', u'he')) assert 'hello'.endswith((u'\u1111lo', u'llo')) assert 'hello'.endswith((u'\u1111hellox', u'hello')) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py Mon Nov 19 12:16:46 2007 @@ -265,7 +265,7 @@ w_value = unicode_from_object(space, w_obj) else: w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) - if space.config.objspace.std.withrope: + if space.config.objspace.std.withropeunicode: assert isinstance(w_value, W_RopeUnicodeObject) w_newobj = space.allocate_instance(W_RopeUnicodeObject, w_unicodetype) W_RopeUnicodeObject.__init__(w_newobj, w_value._node) From rxe at codespeak.net Mon Nov 19 12:36:09 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 12:36:09 +0100 (CET) Subject: [pypy-svn] r48782 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071119113609.0A0DA8133@code0.codespeak.net> Author: rxe Date: Mon Nov 19 12:36:08 2007 New Revision: 48782 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/rpython/module/test/test_ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: (fijal, rxe) clean up 1 XXX Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Mon Nov 19 12:36:08 2007 @@ -143,18 +143,14 @@ _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] -# XXX need someone to look this over please... - from pypy.rpython.module.ll_os import CConfig from pypy.rpython.tool import rffi_platform as platform class CConfig: _includes_ = INCLUDES STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) config = platform.configure(CConfig) -STAT_STRUCT = config['STAT_STRUCT'] -_LL_STAT_FIELDS = [(n[2:], STAT_STRUCT._flds[n]) for n in STAT_STRUCT._names] -STRUCT_STAT = rffi.CStructPtr(_name_struct_stat, *_LL_STAT_FIELDS) +STAT_STRUCT = lltype.Ptr(config['STAT_STRUCT']) def build_stat_result(st): # only for LL backends @@ -199,11 +195,11 @@ ARG1 = rffi.CCHARP else: ARG1 = rffi.INT - os_mystat = rffi.llexternal(c_func_name, [ARG1, STRUCT_STAT], rffi.INT, + os_mystat = rffi.llexternal(c_func_name, [ARG1, STAT_STRUCT], rffi.INT, includes=INCLUDES) def os_mystat_llimpl(arg): - stresult = lltype.malloc(STRUCT_STAT.TO, flavor='raw') + stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: arg = rffi.str2charp(arg) Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Mon Nov 19 12:36:08 2007 @@ -62,7 +62,7 @@ def setup_class(cls): if not hasattr(os, 'ttyname'): py.test.skip("no ttyname") - py.test.skip("XXX get_errno() does not work with ll2ctypes") + #py.test.skip("XXX get_errno() does not work with ll2ctypes") def test_ttyname(self): import os Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Mon Nov 19 12:36:08 2007 @@ -146,5 +146,5 @@ class TestOOtype(BaseTestPosix, OORtypeMixin): def test_fstat(self): - py.test.skip("not working") + py.test.skip("ootypesystem does not support os.fstat") From rxe at codespeak.net Mon Nov 19 12:42:27 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 12:42:27 +0100 (CET) Subject: [pypy-svn] r48783 - pypy/dist/pypy/rpython/module/test Message-ID: <20071119114227.90EEE80C2@code0.codespeak.net> Author: rxe Date: Mon Nov 19 12:42:27 2007 New Revision: 48783 Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: arigo fixed this when fix errno for ll2ctypes Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Mon Nov 19 12:42:27 2007 @@ -62,7 +62,6 @@ def setup_class(cls): if not hasattr(os, 'ttyname'): py.test.skip("no ttyname") - #py.test.skip("XXX get_errno() does not work with ll2ctypes") def test_ttyname(self): import os From rxe at codespeak.net Mon Nov 19 13:00:13 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 13:00:13 +0100 (CET) Subject: [pypy-svn] r48784 - pypy/dist/pypy/rlib Message-ID: <20071119120013.EED508134@code0.codespeak.net> Author: rxe Date: Mon Nov 19 13:00:13 2007 New Revision: 48784 Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rsocket.py Log: (fijal,rxe) move a fixed array pointer to a carray Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Mon Nov 19 13:00:13 2007 @@ -461,7 +461,7 @@ if _POSIX: fcntl = external('fcntl', [socketfd_type, rffi.INT, rffi.INT], rffi.INT) - socketpair_t = rffi.CFixedArray(socketfd_type, 2) + socketpair_t = rffi.CArray(socketfd_type) socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(socketpair_t)], rffi.INT) Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Mon Nov 19 13:00:13 2007 @@ -1027,7 +1027,7 @@ The arguments are the same as for socket() except the default family is AF_UNIX if defined on the platform; otherwise, the default is AF_INET. """ - result = lltype.malloc(_c.socketpair_t, flavor='raw') + result = lltype.malloc(_c.socketpair_t, 2, flavor='raw') res = _c.socketpair(family, type, proto, result) if res < 0: raise last_error() From fijal at codespeak.net Mon Nov 19 13:02:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Nov 2007 13:02:41 +0100 (CET) Subject: [pypy-svn] r48785 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071119120241.A29218134@code0.codespeak.net> Author: fijal Date: Mon Nov 19 13:02:41 2007 New Revision: 48785 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: Add module cleanup Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Mon Nov 19 13:02:41 2007 @@ -54,6 +54,11 @@ - remove prebuilt costate vars (might happen before) +Modules ++++++++ + + - Take a look at module failures on Mac OS X (_socket) + JIT +++ From rxe at codespeak.net Mon Nov 19 13:15:35 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 13:15:35 +0100 (CET) Subject: [pypy-svn] r48786 - pypy/dist/pypy/module/rctime/test Message-ID: <20071119121535.F3FD380C9@code0.codespeak.net> Author: rxe Date: Mon Nov 19 13:15:34 2007 New Revision: 48786 Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py Log: (fijal, rxe) fix anti-british tests Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Mon Nov 19 13:15:34 2007 @@ -101,7 +101,7 @@ t = rctime.time() assert long(rctime.mktime(rctime.localtime(t))) == long(t) - assert long(rctime.mktime(rctime.gmtime(t))) != long(t) + assert long(rctime.mktime(rctime.gmtime(t))) + rctime.timezone == long(t) ltime = rctime.localtime() assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime) @@ -120,7 +120,8 @@ rctime.asctime(rctime.localtime()) t = rctime.time() assert rctime.ctime(t) == rctime.asctime(rctime.localtime(t)) - assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t)) + if rctime.timezone: + assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t)) ltime = rctime.localtime() assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime) From rxe at codespeak.net Mon Nov 19 13:16:36 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 13:16:36 +0100 (CET) Subject: [pypy-svn] r48787 - pypy/dist/pypy/module/rctime Message-ID: <20071119121636.46EF28135@code0.codespeak.net> Author: rxe Date: Mon Nov 19 13:16:35 2007 New Revision: 48787 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: remove another FixArray Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Mon Nov 19 13:16:35 2007 @@ -66,7 +66,7 @@ if cConfig.has_gettimeofday: c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT) -TIME_TP = lltype.Ptr(rffi.CFixedArray(time_t, 1)) +TIME_TP = rffi.CArrayPtr(time_t) TM_P = lltype.Ptr(tm) c_clock = external('clock', [TIME_TP], clock_t) c_time = external('time', [TIME_TP], time_t) @@ -106,7 +106,7 @@ t = (((c_time(lltype.nullptr(TIME_TP.TO))) / YEAR) * YEAR) # we cannot have reference to stack variable, put it on the heap - t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref = lltype.malloc(TIME_TP.TO, 1, flavor='raw') t_ref[0] = t p = c_localtime(t_ref) janzone = -p.c_tm_gmtoff @@ -185,7 +185,7 @@ if allowNone and space.is_w(w_tup, space.w_None): # default to the current local time tt = int(pytime.time()) - t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref = lltype.malloc(TIME_TP.TO, 1, flavor='raw') t_ref[0] = tt pbuf = c_localtime(t_ref) lltype.free(t_ref, flavor='raw') @@ -275,7 +275,7 @@ seconds = _get_inttime(space, w_seconds) - t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref = lltype.malloc(TIME_TP.TO, 1, flavor='raw') t_ref[0] = seconds p = c_ctime(t_ref) lltype.free(t_ref, flavor='raw') @@ -314,7 +314,7 @@ # rpython does not support that a variable has two incompatible builtins # as value so we have to duplicate the code. NOT GOOD! see localtime() too seconds = _get_inttime(space, w_seconds) - t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref = lltype.malloc(TIME_TP.TO, 1, flavor='raw') t_ref[0] = seconds p = c_gmtime(t_ref) lltype.free(t_ref, flavor='raw') @@ -332,7 +332,7 @@ When 'seconds' is not passed in, convert the current time instead.""" seconds = _get_inttime(space, w_seconds) - t_ref = lltype.malloc(TIME_TP.TO, flavor='raw') + t_ref = lltype.malloc(TIME_TP.TO, 1, flavor='raw') t_ref[0] = seconds p = c_localtime(t_ref) lltype.free(t_ref, flavor='raw') From cfbolz at codespeak.net Mon Nov 19 14:51:54 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 14:51:54 +0100 (CET) Subject: [pypy-svn] r48788 - in pypy/branch/ropes-unicode/pypy: annotation rpython rpython/lltypesystem rpython/ootypesystem rpython/test Message-ID: <20071119135154.3E2B78141@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 14:51:53 2007 New Revision: 48788 Modified: pypy/branch/ropes-unicode/pypy/annotation/unaryop.py pypy/branch/ropes-unicode/pypy/rpython/lltypesystem/rstr.py pypy/branch/ropes-unicode/pypy/rpython/ootypesystem/rstr.py pypy/branch/ropes-unicode/pypy/rpython/rstr.py pypy/branch/ropes-unicode/pypy/rpython/test/test_runicode.py Log: add encoding and decoding with latin-1 Modified: pypy/branch/ropes-unicode/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/annotation/unaryop.py (original) +++ pypy/branch/ropes-unicode/pypy/annotation/unaryop.py Mon Nov 19 14:51:53 2007 @@ -466,7 +466,7 @@ if not s_enc.is_constant(): raise TypeError("Non-constant encoding not supported") enc = s_enc.const - if enc != 'ascii': + if enc not in ('ascii', 'latin-1'): raise TypeError("Encoding %s not supported for unicode" % (enc,)) return SomeString() method_encode.can_only_throw = [UnicodeEncodeError] @@ -482,7 +482,7 @@ if not s_enc.is_constant(): raise TypeError("Non-constant encoding not supported") enc = s_enc.const - if enc != 'ascii': + if enc not in ('ascii', 'latin-1'): raise TypeError("Encoding %s not supported for strings" % (enc,)) return SomeUnicodeString() method_decode.can_only_throw = [UnicodeDecodeError] Modified: pypy/branch/ropes-unicode/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/ropes-unicode/pypy/rpython/lltypesystem/rstr.py Mon Nov 19 14:51:53 2007 @@ -108,6 +108,13 @@ self.ll = LLHelpers self.malloc = mallocstr + def ll_decode_latin1(self, value): + lgt = len(value.chars) + s = mallocunicode(lgt) + for i in range(lgt): + s.chars[i] = cast_primitive(UniChar, value.chars[i]) + return s + class UnicodeRepr(BaseLLStringRepr, AbstractUnicodeRepr): lowleveltype = Ptr(UNICODE) basetype = basestring @@ -131,6 +138,17 @@ result.chars[i] = cast_primitive(Char, c) return result + def ll_encode_latin1(self, s): + length = len(s.chars) + result = mallocstr(length) + for i in range(length): + c = s.chars[i] + if ord(c) > 255: + raise UnicodeEncodeError("character not in latin1 range") + result.chars[i] = cast_primitive(Char, c) + return result + + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char Modified: pypy/branch/ropes-unicode/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/ropes-unicode/pypy/rpython/ootypesystem/rstr.py Mon Nov 19 14:51:53 2007 @@ -48,6 +48,16 @@ def make_string(self, value): return ootype.make_string(value) + def ll_decode_latin1(self, value): + sb = ootype.new(ootype.UnicodeBuilder) + length = value.ll_strlen() + sb.ll_allocate(length) + for i in range(length): + c = value.ll_stritem_nonneg(i) + sb.ll_append_char(cast_primitive(UniChar, c)) + return sb.ll_build() + + class UnicodeRepr(BaseOOStringRepr, AbstractUnicodeRepr): lowleveltype = ootype.Unicode basetype = basestring @@ -66,6 +76,17 @@ sb.ll_append_char(cast_primitive(Char, c)) return sb.ll_build() + def ll_encode_latin1(self, value): + sb = ootype.new(ootype.StringBuilder) + length = value.ll_strlen() + sb.ll_allocate(length) + for i in range(length): + c = value.ll_stritem_nonneg(i) + if ord(c) > 255: + raise UnicodeEncodeError("%d > 255, not latin-1" % ord(c)) + sb.ll_append_char(cast_primitive(Char, c)) + return sb.ll_build() + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char Modified: pypy/branch/ropes-unicode/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rpython/rstr.py (original) +++ pypy/branch/ropes-unicode/pypy/rpython/rstr.py Mon Nov 19 14:51:53 2007 @@ -256,9 +256,17 @@ return hop.gendirectcall(self.ll.ll_str2unicode, v_str) def rtype_method_decode(self, hop): - v_self = hop.inputarg(self, 0) + if not hop.args_s[1].is_constant(): + raise TyperError("encoding must be a constant") + encoding = hop.args_s[1].const + v_self = hop.inputarg(self.repr, 0) hop.exception_is_here() - return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + if encoding == 'ascii': + return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + elif encoding == 'latin-1': + return hop.gendirectcall(self.ll_decode_latin1, v_self) + else: + raise TyperError("encoding %s not implemented" % (encoding, )) def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it @@ -272,9 +280,18 @@ class __extend__(AbstractUnicodeRepr): def rtype_method_encode(self, hop): - v_self = hop.inputarg(self, 0) + if not hop.args_s[1].is_constant(): + raise TyperError("encoding must be constant") + encoding = hop.args_s[1].const + v_self = hop.inputarg(self.repr, 0) hop.exception_is_here() - return hop.gendirectcall(self.ll_str, v_self) + if encoding == "ascii": + return hop.gendirectcall(self.ll_str, v_self) + elif encoding == "latin-1": + return hop.gendirectcall(self.ll_encode_latin1, v_self) + else: + raise TyperError("encoding %s not implemented" % (encoding, )) + class __extend__(pairtype(AbstractStringRepr, Repr)): def rtype_mod((r_str, _), hop): Modified: pypy/branch/ropes-unicode/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rpython/test/test_runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rpython/test/test_runicode.py Mon Nov 19 14:51:53 2007 @@ -84,26 +84,37 @@ def test_unicode_encode(self): def f(x): y = u'xxx' - return (y + unichr(x)).encode('ascii') + return (y + unichr(x)).encode('ascii') + y.encode('latin-1') assert self.ll_to_string(self.interpret(f, [38])) == f(38) def test_unicode_encode_error(self): - def f(x): - y = u'xxx' - try: - x = (y + unichr(x)).encode('ascii') - return len(x) - except UnicodeEncodeError: - return -1 - - assert self.interpret(f, [38]) == f(38) - assert self.interpret(f, [138]) == f(138) + def f(x, which): + if which: + y = u'xxx' + try: + x = (y + unichr(x)).encode('ascii') + return len(x) + except UnicodeEncodeError: + return -1 + else: + y = u'xxx' + try: + x = (y + unichr(x)).encode('latin-1') + return len(x) + except UnicodeEncodeError: + return -1 + + assert self.interpret(f, [38, True]) == f(38, True) + assert self.interpret(f, [138, True]) == f(138, True) + assert self.interpret(f, [38, False]) == f(38, False) + assert self.interpret(f, [138, False]) == f(138, False) + assert self.interpret(f, [300, False]) == f(300, False) def test_unicode_decode(self): def f(x): y = 'xxx' - return (y + chr(x)).decode('ascii') + return (y + chr(x)).decode('ascii') + chr(x).decode("latin-1") assert self.ll_to_string(self.interpret(f, [38])) == f(38) From cfbolz at codespeak.net Mon Nov 19 15:05:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 15:05:45 +0100 (CET) Subject: [pypy-svn] r48789 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119140545.BAB778146@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 15:05:44 2007 New Revision: 48789 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: kill some dead code and some useless XXXs Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Mon Nov 19 15:05:44 2007 @@ -133,7 +133,6 @@ return self.s def flatten_unicode(self): - # XXX not RPython yet return self.s.decode('latin-1') def hash_part(self): @@ -706,40 +705,6 @@ restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) -def _find(node, substring, start, stop, restart): - # XXX - assert node.is_bytestring() - len2 = len(substring) - i = 0 - m = start - iter = SeekableItemIterator(node) - iter.seekforward(start) - c = iter.nextchar() - while m + i < stop: - if c == substring[i]: - i += 1 - if i == len2: - return m - if m + i < stop: - c = iter.nextchar() - else: - # mismatch, go back to the last possible starting pos - if i==0: - m += 1 - if m + i < stop: - c = iter.nextchar() - else: - e = restart[i-1] - new_m = m + i - e - assert new_m <= m + i - seek = m + i - new_m - if seek: - iter.seekback(m + i - new_m) - c = iter.nextchar() - m = new_m - i = e - return -1 - def _find_node(node, subnode, start, stop, restart): len2 = subnode.length() m = start @@ -777,27 +742,7 @@ i = e return -1 -def construct_restart_positions(s): - length = len(s) - restart = [0] * length - restart[0] = 0 - i = 1 - j = 0 - while i < length: - if s[i] == s[j]: - j += 1 - restart[i] = j - i += 1 - elif j>0: - j = restart[j-1] - else: - restart[i] = 0 - i += 1 - j = 0 - return restart - def construct_restart_positions_node(node): - # really a bit overkill length = node.length() restart = [0] * length restart[0] = 0 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Mon Nov 19 15:05:44 2007 @@ -423,18 +423,11 @@ restart = construct_restart_positions_node( BinaryConcatNode(LiteralStringNode("aba"), LiteralStringNode("bcabab"))) assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] - restart = construct_restart_positions("ababcabab") - assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] - restart = construct_restart_positions("ababcababb") - assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4, 0] restart = construct_restart_positions_node( BinaryConcatNode(LiteralStringNode("aba"), LiteralStringNode("bcababb"))) assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4, 0] - restart = construct_restart_positions("ababb") - assert restart == [0, 0, 1, 2, 0] restart = construct_restart_positions_node(LiteralStringNode("ababb")) assert restart == [0, 0, 1, 2, 0] - #abababcabcabb def test_find(): @@ -447,6 +440,17 @@ pos = find(node, LiteralStringNode("a"), 0, node.length()) assert pos == 6 + +def test_find_unicode(): + node = BinaryConcatNode(LiteralUnicodeNode(u"\uaaaa\ubbbb\uaaaa"), + LiteralUnicodeNode(u"\ubbbb\ucccc\uaaaa\ubbbb\uaaaa\ubbbb")) + pos = find(node, LiteralUnicodeNode(u"\uaaaa\ubbbb\ucccc"), 0, node.length()) + assert pos == 2 + node = BinaryConcatNode(LiteralUnicodeNode(u"btffp"), + LiteralUnicodeNode(u"b\uaaaacbb")) + pos = find(node, LiteralUnicodeNode(u"\uaaaa"), 0, node.length()) + assert pos == 6 + def test_fromcharlist(): for i in range(0, 100, 10): chars = ["a"] * 50 + ["b"] * i From cfbolz at codespeak.net Mon Nov 19 15:28:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 15:28:34 +0100 (CET) Subject: [pypy-svn] r48790 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119142834.6A32D8142@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 15:28:31 2007 New Revision: 48790 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: bug in partition Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 15:28:31 2007 @@ -724,12 +724,12 @@ space.wrap("empty separator")) pos = rope.find(self, sub) if pos == -1: - return space.newtuple([w_self, W_RopeUnicodeObject.EMPTY, + return space.newtuple([w_unistr, W_RopeUnicodeObject.EMPTY, W_RopeUnicodeObject.EMPTY]) else: return space.newtuple( [W_RopeUnicodeObject(rope.getslice_one(self, 0, pos)), - w_sub, + w_unisub, W_RopeUnicodeObject(rope.getslice_one(self, pos + sub.length(), self.length()))]) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 15:28:31 2007 @@ -472,3 +472,34 @@ "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'") assert (repr(u'\n') == "u'\\n'") + + + def test_partition(self): + + assert (u'this is the par', u'ti', u'tion method') == \ + u'this is the partition method'.partition(u'ti') + + # from raymond's original specification + S = u'http://www.python.org' + assert (u'http', u'://', u'www.python.org') == S.partition(u'://') + assert (u'http://www.python.org', u'', u'') == S.partition(u'?') + assert (u'', u'http://', u'www.python.org') == S.partition(u'http://') + assert (u'http://www.python.', u'org', u'') == S.partition(u'org') + + raises(ValueError, S.partition, u'') + raises(TypeError, S.partition, None) + + def test_rpartition(self): + + assert ('this is the rparti', 'ti', 'on method') == \ + 'this is the rpartition method'.rpartition('ti') + + # from raymond's original specification + S = 'http://www.python.org' + assert ('http', '://', 'www.python.org') == S.rpartition('://') + assert ('', '', 'http://www.python.org') == S.rpartition('?') + assert ('', 'http://', 'www.python.org') == S.rpartition('http://') + assert ('http://www.python.', 'org', '') == S.rpartition('org') + + raises(ValueError, S.rpartition, '') + raises(TypeError, S.rpartition, None) From cfbolz at codespeak.net Mon Nov 19 15:30:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 15:30:05 +0100 (CET) Subject: [pypy-svn] r48791 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119143005.C11D88133@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 15:30:05 2007 New Revision: 48791 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: fix test and another bug Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 15:30:05 2007 @@ -745,7 +745,7 @@ return space.newtuple([W_RopeUnicodeObject.EMPTY, W_RopeUnicodeObject.EMPTY, w_unistr]) else: - assert pos > 0 + assert pos >= 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, space.wrap(unistr[pos+len(unisub):])]) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 15:30:05 2007 @@ -491,15 +491,15 @@ def test_rpartition(self): - assert ('this is the rparti', 'ti', 'on method') == \ - 'this is the rpartition method'.rpartition('ti') + assert (u'this is the rparti', u'ti', u'on method') == \ + u'this is the rpartition method'.rpartition(u'ti') # from raymond's original specification - S = 'http://www.python.org' - assert ('http', '://', 'www.python.org') == S.rpartition('://') - assert ('', '', 'http://www.python.org') == S.rpartition('?') - assert ('', 'http://', 'www.python.org') == S.rpartition('http://') - assert ('http://www.python.', 'org', '') == S.rpartition('org') + S = u'http://www.python.org' + assert (u'http', u'://', u'www.python.org') == S.rpartition(u'://') + assert (u'', u'', u'http://www.python.org') == S.rpartition(u'?') + assert (u'', u'http://', u'www.python.org') == S.rpartition(u'http://') + assert (u'http://www.python.', u'org', u'') == S.rpartition(u'org') - raises(ValueError, S.rpartition, '') + raises(ValueError, S.rpartition, u'') raises(TypeError, S.rpartition, None) From cfbolz at codespeak.net Mon Nov 19 15:38:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 15:38:07 +0100 (CET) Subject: [pypy-svn] r48792 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119143807.2DF918142@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 15:38:06 2007 New Revision: 48792 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: fix a bug in rindex Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 15:38:06 2007 @@ -594,13 +594,13 @@ # XXX works but flattens string self, start, end = _convert_idx_params(space, w_self, w_start, w_end) self = self.flatten_unicode() - sub = sub.flatten_unicode() + sub = w_substr._node.flatten_unicode() res = self.rfind(sub, start, end) if res < 0: raise OperationError(space.w_ValueError, space.wrap("substring not found in string.rindex")) - return wrapint(space, res) + return space.wrap(res) def unicode_count__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 15:38:06 2007 @@ -503,3 +503,20 @@ raises(ValueError, S.rpartition, u'') raises(TypeError, S.rpartition, None) + + + def test_rindex(self): + from sys import maxint + assert u'abcdefghiabc'.rindex(u'') == 12 + assert u'abcdefghiabc'.rindex(u'def') == 3 + assert u'abcdefghiabc'.rindex(u'abc') == 9 + assert u'abcdefghiabc'.rindex(u'abc', 0, -1) == 0 + assert u'abcdefghiabc'.rindex(u'abc', -4*maxint, 4*maxint) == 9 + raises(ValueError, u'abcdefghiabc'.rindex, u'hib') + raises(ValueError, u'defghiabc'.rindex, u'def', 1) + raises(ValueError, u'defghiabc'.rindex, u'abc', 0, -1) + raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, 8) + raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, -1) + raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', 0, 0.0) + raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', -10.0, 30) + From cfbolz at codespeak.net Mon Nov 19 15:55:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 15:55:17 +0100 (CET) Subject: [pypy-svn] r48793 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119145517.1C3C48183@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 15:55:16 2007 New Revision: 48793 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: more bugs Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 15:55:16 2007 @@ -577,9 +577,9 @@ def unicode_rfind__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) self = self.flatten_unicode() - sub = sub.flatten_unicode() + sub = w_substr._node.flatten_unicode() res = self.rfind(sub, start, end) - return wrapint(space, res) + return space.wrap(res) def unicode_index__RopeUnicode_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end): self, start, end = _convert_idx_params(space, w_self, w_start, w_end) @@ -921,13 +921,13 @@ def len__RopeUnicodeIter(space, w_ropeiter): if w_ropeiter.node is None: - return wrapint(space, 0) + return space.wrap(0) index = w_ropeiter.index length = w_ropeiter.node.length() result = length - index if result < 0: - return wrapint(space, 0) - return wrapint(space, result) + return space.wrap(0) + return space.wrap(result) import unicodetype register_all(vars(), unicodetype) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 15:55:16 2007 @@ -520,3 +520,13 @@ raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', 0, 0.0) raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', -10.0, 30) + def test_len_iter(self): + assert len(iter(u"abcdef\uffff")) == 7 + for i in range(10): + assert len(iter(unicode(i))) == 1 + + def test_rfind(self): + assert u'abcdefghiabc'.rfind(u'abc') == 9 + assert u'abcdefghiabc'.rfind(u'') == 12 + assert u'abcdefghiabc'.rfind(u'abcd') == 0 + assert u'abcdefghiabc'.rfind(u'abcz') == -1 From fijal at codespeak.net Mon Nov 19 15:57:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Nov 2007 15:57:07 +0100 (CET) Subject: [pypy-svn] r48794 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071119145707.C213E8183@code0.codespeak.net> Author: fijal Date: Mon Nov 19 15:57:07 2007 New Revision: 48794 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py Log: Memoize itemoffset, otherwise you cannot call it from inside rpython function, even with constant arguments. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Mon Nov 19 15:57:07 2007 @@ -302,6 +302,7 @@ def itemoffsetof(TYPE, n=0): return ArrayItemsOffset(TYPE) + ItemOffset(TYPE.OF) * n +itemoffsetof._annspecialcase_ = 'specialize:memo' # ------------------------------------------------------------- class fakeaddress(object): From fijal at codespeak.net Mon Nov 19 15:57:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Nov 2007 15:57:45 +0100 (CET) Subject: [pypy-svn] r48795 - pypy/dist/pypy/translator/goal Message-ID: <20071119145745.9175A8183@code0.codespeak.net> Author: fijal Date: Mon Nov 19 15:57:45 2007 New Revision: 48795 Added: pypy/dist/pypy/translator/goal/targetlbench.py (contents, props changed) Log: Add a benchmark for list and memcpy operations, intermediate checkin. Added: pypy/dist/pypy/translator/goal/targetlbench.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/goal/targetlbench.py Mon Nov 19 15:57:45 2007 @@ -0,0 +1,38 @@ + +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr, raw_memclear,\ + raw_memcopy, sizeof, itemoffsetof + +TP = lltype.GcArray(lltype.Signed) + +def f(x): + if 1: + a = lltype.malloc(TP, x) + for i in range(x): + a[i] = i + b = lltype.malloc(TP, x, zero=False) + for j in range(1000): + #for i in range(x): + # b[i] = a[i] + baseofs = itemoffsetof(TP, 0) + onesize = sizeof(TP.OF) + size = baseofs + onesize*(x - 1) + raw_memcopy(cast_ptr_to_adr(b)+baseofs, cast_ptr_to_adr(a)+baseofs, size) + else: + a = [] + for i in range(x): + a.append(i) + return 0 + +def entry_point(argv): + print f(int(argv[1])) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + import sys + entry_point(sys.argv) From cfbolz at codespeak.net Mon Nov 19 16:01:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 16:01:01 +0100 (CET) Subject: [pypy-svn] r48796 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119150101.51094814B@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 16:01:00 2007 New Revision: 48796 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: bugs in count Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 16:01:00 2007 @@ -614,7 +614,7 @@ except StopIteration: break i += 1 - return wrapint(space, i) + return space.wrap(i) def unicode_split__RopeUnicode_None_ANY(space, w_self, w_none, w_maxsplit): selfnode = w_self._node Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 16:01:00 2007 @@ -530,3 +530,17 @@ assert u'abcdefghiabc'.rfind(u'') == 12 assert u'abcdefghiabc'.rfind(u'abcd') == 0 assert u'abcdefghiabc'.rfind(u'abcz') == -1 + + + def test_count(self): + assert u"".count(u"x") ==0 + assert u"".count(u"") ==1 + assert u"Python".count(u"") ==7 + assert u"ab aaba".count(u"ab") ==2 + assert 'aaa'.count('a') == 3 + assert 'aaa'.count('b') == 0 + assert 'aaa'.count('a', -1) == 1 + assert 'aaa'.count('a', -10) == 3 + assert 'aaa'.count('a', 0, -1) == 2 + assert 'aaa'.count('a', 0, -10) == 0 + assert 'ababa'.count('aba') == 1 From rxe at codespeak.net Mon Nov 19 16:20:33 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 16:20:33 +0100 (CET) Subject: [pypy-svn] r48797 - pypy/dist/pypy/rpython/module Message-ID: <20071119152033.0694F8141@code0.codespeak.net> Author: rxe Date: Mon Nov 19 16:20:32 2007 New Revision: 48797 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/ll_os_environ.py Log: more FixedSizeArray -> CArrayPtr Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Mon Nov 19 16:20:32 2007 @@ -168,14 +168,14 @@ ('tv_usec', rffi.LONG)]) config = platform.configure(CConfig) TIMEVAL = config['TIMEVAL'] - TIMEVAL2P = lltype.Ptr(lltype.FixedSizeArray(TIMEVAL, 2)) + TIMEVAL2P = rffi.CArrayPtr(TIMEVAL) os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], rffi.INT, includes=['sys/time.h']) def os_utime_platform(path, actime, modtime): import math - l_times = lltype.malloc(TIMEVAL2P.TO, flavor='raw') + l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw') fracpart, intpart = math.modf(actime) l_times[0].c_tv_sec = int(intpart) l_times[0].c_tv_usec = int(fracpart * 1E6) @@ -764,11 +764,11 @@ return (fdread, fdwrite) else: - INT_ARRAY_P = lltype.Ptr(lltype.FixedSizeArray(rffi.INT, 2)) + INT_ARRAY_P = rffi.CArrayPtr(rffi.INT) os_pipe = self.llexternal('pipe', [INT_ARRAY_P], rffi.INT) def os_pipe_llimpl(): - filedes = lltype.malloc(INT_ARRAY_P.TO, flavor='raw') + filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw') error = os_pipe(filedes) read_fd = filedes[0] write_fd = filedes[1] Modified: pypy/dist/pypy/rpython/module/ll_os_environ.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_environ.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_environ.py Mon Nov 19 16:20:32 2007 @@ -132,7 +132,7 @@ # Access to the 'environ' external variable if sys.platform.startswith('darwin'): - CCHARPPP = lltype.Ptr(lltype.FixedSizeArray(rffi.CCHARPP, 1)) + CCHARPPP = rffi.CArrayPtr(rffi.CCHARPP)) _os_NSGetEnviron = rffi.llexternal('_NSGetEnviron', [], CCHARPPP, includes=['crt_externs.h']) def os_get_environ(): From rxe at codespeak.net Mon Nov 19 16:31:37 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 16:31:37 +0100 (CET) Subject: [pypy-svn] r48798 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071119153137.2BED5817C@code0.codespeak.net> Author: rxe Date: Mon Nov 19 16:31:35 2007 New Revision: 48798 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Log: tests run on 2.0 as well. enable isolate again. remove some print Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Mon Nov 19 16:31:35 2007 @@ -430,7 +430,6 @@ def repr_char(self, type_, value): x = ord(value) - print x if x >= 128: # XXX check this really works r = "trunc (i16 %s to i8)" % x Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Mon Nov 19 16:31:35 2007 @@ -384,7 +384,6 @@ def setinteriorfield(self, opr): op = opr.op if opr.argtypes[-1] != "void": - print op.args, op.args[1:-1] _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:-1]) tmpvar = self._tmp() self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Nov 19 16:31:35 2007 @@ -9,14 +9,14 @@ optimize_tests = False native_llvm_backend = True -MINIMUM_LLVM_VERSION = 2.1 +MINIMUM_LLVM_VERSION = 2.0 FLOAT_PRECISION = 8 # prevents resource leaking -use_isolate = False +use_isolate = True # if test can't be run using isolate, skip the test (useful for buildbots) -run_isolated_only = False +run_isolated_only = True from pypy import conftest Modified: pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rbuiltin.py Mon Nov 19 16:31:35 2007 @@ -4,9 +4,14 @@ from pypy.translator.llvm.test.runtest import * class TestLLVMBuiltin(LLVMTest, BaseTestRbuiltin): + def _isolate_skip(self): + py.test.skip("XXX isolate specialize this") + def _skip(self): py.test.skip("XXX specialize this") + test_os_write = _isolate_skip + test_os_write_single_char = _isolate_skip test_os_dup = _skip test_os_open = _skip test_debug_llinterpcall = _skip From rxe at codespeak.net Mon Nov 19 16:32:13 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 16:32:13 +0100 (CET) Subject: [pypy-svn] r48799 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071119153213.4C8E6817C@code0.codespeak.net> Author: rxe Date: Mon Nov 19 16:32:12 2007 New Revision: 48799 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/opaquenode.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: resolve some XXXs - mostly by just removing them. Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Mon Nov 19 16:32:12 2007 @@ -21,7 +21,6 @@ self.get_typerepr(), self.name, index) - #XXX probably why we are failing anyways return 'bitcast(i8* %s to [0 x i8]*)' % x def writeglobalconstants(self, codewriter): Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Mon Nov 19 16:32:12 2007 @@ -79,7 +79,6 @@ self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (base, opts, base)) def cmds_objects(self, base): - # XXX why this hack??? use_gcc = self.genllvm.config.translation.llvm_via_c if use_gcc: self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (base, base)) @@ -88,6 +87,7 @@ self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (base, base)) self.cmds.append("as %s.s -o %s.o" % (base, base)) +# XXX support profile? # if (self.genllvm.config.translation.profopt is not None and # not self.genllvm.config.translation.noprofopt): # cmd = "gcc -fprofile-generate %s.c -c -O3 -pipe -o %s.o" % (base, base) Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Mon Nov 19 16:32:12 2007 @@ -192,7 +192,7 @@ self._indent("%s = alloca %s" % (targetvar, vartype)) def malloc(self, targetvar, vartype, numelements=1): - XXX + XXX # we should use this for raw malloc (unless it is slow) if numelements == 1: self._indent("%s = malloc %s" % (targetvar, vartype)) else: @@ -203,6 +203,7 @@ def free(self, vartype, varref): + XXX # we should use this for raw malloc (unless it is slow) self._indent("free %s %s" % (vartype, varref)) def debug_print(self, s): Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Mon Nov 19 16:32:12 2007 @@ -150,8 +150,6 @@ self.db = db def genextern_code(self): - # XXX - # This is not finished: we must call the gc init function! r = '' r += '#define __GC_STARTUP_CODE__\n' r += '#define __GC_SETUP_CODE__\n' Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Nov 19 16:32:12 2007 @@ -89,9 +89,7 @@ c_db = cbuild.generate_graphs_for_llinterp() self.db = Database(self, self.translator) - - # XXX hardcoded for now - self.db.gcpolicy = GcPolicy.new(self.db, 'boehm') + self.db.gcpolicy = GcPolicy.new(self.db, self.config.translation.gc) # get entry point entry_point = self.get_entry_point(func) Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Mon Nov 19 16:32:12 2007 @@ -1,4 +1,4 @@ -" XXX THIS IS ONLY FOR TESTING XXX " +" THIS IS ONLY FOR TESTING " import py import ctypes Modified: pypy/dist/pypy/translator/llvm/opaquenode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opaquenode.py (original) +++ pypy/dist/pypy/translator/llvm/opaquenode.py Mon Nov 19 16:32:12 2007 @@ -8,7 +8,6 @@ self.name = "null" def writeglobalconstants(self, codewriter): - # XXX Dummy - not sure what what we want pass class ExtOpaqueNode(ConstantNode): @@ -37,7 +36,6 @@ return ref def writeglobalconstants(self, codewriter): - # XXX Dummy - not sure what what we want pass def constantvalue(self): Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Mon Nov 19 16:32:12 2007 @@ -41,7 +41,6 @@ class OpWriter(object): shift_operations = { - # ZZZ i guessed rshifts - ashr or lshr ?????? 'int_lshift': 'shl', 'int_rshift': 'lshr', @@ -90,7 +89,6 @@ for oo in 'lt le eq ne ge gt'.split(): binary_operations['%s_%s' % (tt, oo)] = 'fcmp o%s' % oo - # ZZZ check that operation should be checking unsigned binary_operations.update({'char_lt': 'icmp ult', 'char_le': 'icmp ule', 'char_eq': 'icmp eq', @@ -207,14 +205,7 @@ op = opr.op name = self.shift_operations[op.opname] - var = opr.argrefs[1] - # ZZZ why did we do this??? - #if isinstance(op.args[1], Constant): - # var = opr.argrefs[1] - #else: - # var = self._tmp() - # self.codewriter.cast(var, opr.argtypes[1], opr.argrefs[1], 'i8') - + var = opr.argrefs[1] self.codewriter.shiftop(name, opr.retref, opr.argtypes[0], opr.argrefs[0], var) def cast_primitive(self, opr): @@ -325,8 +316,8 @@ indices = [] for arg in args: name = None - # XXX this is because FixedSizeArray can sometimes be accessed - # like an Array and then sometimes a Struct + # this is because FixedSizeArray can sometimes be accessed like an + # Array and then sometimes a Struct if arg.concretetype is lltype.Void: name = arg.value assert name in list(TYPE._names) @@ -406,9 +397,7 @@ lengthref = self._tmp() self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices) else: - assert isinstance(TYPE, lltype.FixedSizeArray) - lengthref = TYPE.length - XXX # no tests for this - is it valid even since known at compile time ???? + assert False, "known at compile time" self.codewriter.load(opr.retref, opr.rettype, lengthref) @@ -418,7 +407,6 @@ def direct_fieldptr(self, opr): from pypy.translator.llvm.structnode import getindexhelper - # XXX use to_getelementptr ? op = opr.op assert opr.rettype != "void" index = getindexhelper(op.args[1].value, @@ -434,7 +422,6 @@ self.codewriter.cast(opr.retref, element_type, tmpvar, opr.rettype) def direct_arrayitems(self, opr): - # XXX use to_getelementptr ? assert opr.rettype != "void" array = opr.argrefs[0] @@ -504,9 +491,6 @@ def adr_ge(self, opr): self._op_adr_cmp(opr, "icmp sge") - # XXX Not sure any of this makes sense - maybe seperate policy for - # different flavours of mallocs? Well it depend on what happens the GC - # developments def raw_malloc(self, opr): self.codewriter.call(opr.retref, opr.rettype, "@raw_malloc", opr.argtypes, opr.argrefs) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Nov 19 16:32:12 2007 @@ -117,12 +117,13 @@ def genllvm_compile(function, annotation, + gcpolicy='boehm', # debug options debug=True, logging=False, isolate=True, - + # pass to compile optimize=True, extra_opts={}): @@ -138,7 +139,7 @@ 'translation.llvm.logging': logging, 'translation.llvm.isolate': isolate, 'translation.backendopt.none': not optimize, - 'translation.gc': 'boehm', + 'translation.gc': gcpolicy, 'translation.llvm_via_c' : not native_llvm_backend } From rxe at codespeak.net Mon Nov 19 16:32:38 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 19 Nov 2007 16:32:38 +0100 (CET) Subject: [pypy-svn] r48800 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071119153238.DD1F2816B@code0.codespeak.net> Author: rxe Date: Mon Nov 19 16:32:38 2007 New Revision: 48800 Modified: pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/genexterns.c pypy/dist/pypy/translator/llvm/test/runtest.py Log: resolve some XXXs - mostly by just removing them. Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Nov 19 16:32:38 2007 @@ -82,6 +82,30 @@ create ll file for c file create codewriter """ + if self.standalone: + # XXX this requires jumping through way too many hoops + + from pypy.rpython.lltypesystem import rffi + getargc = rffi.llexternal('__pypy_getargc', [], rffi.INT) + getargv = rffi.llexternal('__pypy_getargv', [], rffi.CCHARPP) + + def entry_point(args=None): + argc = getargc() + argv = getargv() + if args is None: + args = [] + for ii in range(argc): + s = rffi.charp2str(argv[ii]) + args.append(s) + return func(args) + from pypy.annotation.listdef import s_list_of_strings + graph = self.translator.rtyper.annotate_helper(entry_point, [s_list_of_strings]) + self.translator.rtyper.specialize_more_blocks() + # XXX this is really bad + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(self.translator) + func = entry_point + # XXX please dont ask! from pypy.translator.c.genc import CStandaloneBuilder cbuild = CStandaloneBuilder(self.translator, func, config=self.config) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Mon Nov 19 16:32:38 2007 @@ -47,41 +47,16 @@ #ifdef ENTRY_POINT_DEFINED -int __ENTRY_POINT__(RPyListOfString *); +int __ENTRY_POINT__(voidzd *); -int main(int argc, char *argv[]) -{ - XXX - char *errmsg; - int i, exitcode; - RPyListOfString *list; - errmsg = RPython_StartupCode(); - if (errmsg) goto error; - - list = _RPyListOfString_New(argc); - if (_RPyExceptionOccurred()) goto memory_out; - for (i=0; i Author: cfbolz Date: Mon Nov 19 17:09:13 2007 New Revision: 48801 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Log: rpython fix: don't mix string and unicode Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Mon Nov 19 17:09:13 2007 @@ -345,7 +345,7 @@ def flatten_unicode(self): f = fringe(self) - return "".join([node.flatten_unicode() for node in f]) + return u"".join([node.flatten_unicode() for node in f]) def hash_part(self): h = self.hash_cache From cfbolz at codespeak.net Mon Nov 19 17:09:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 17:09:23 +0100 (CET) Subject: [pypy-svn] r48802 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071119160923.AFDE28122@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 17:09:23 2007 New Revision: 48802 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Log: test and fix in swapcase Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Mon Nov 19 17:09:23 2007 @@ -439,10 +439,10 @@ return _local_transform(w_self._node, _toupper) def _swapcase(ordch): - if unicodedb.islower(unichar): - return unichr(unicodedb.toupper(unichar)) - elif unicodedb.isupper(unichar): - return unichr(unicodedb.tolower(unichar)) + if unicodedb.islower(ordch): + return unichr(unicodedb.toupper(ordch)) + elif unicodedb.isupper(ordch): + return unichr(unicodedb.tolower(ordch)) else: return unichr(ordch) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 19 17:09:23 2007 @@ -544,3 +544,6 @@ assert 'aaa'.count('a', 0, -1) == 2 assert 'aaa'.count('a', 0, -10) == 0 assert 'ababa'.count('aba') == 1 + + def test_swapcase(self): + assert u'\xe4\xc4\xdf'.swapcase() == u'\xc4\xe4\xdf' From fijal at codespeak.net Mon Nov 19 17:17:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Nov 2007 17:17:41 +0100 (CET) Subject: [pypy-svn] r48803 - pypy/dist/pypy/rpython/module Message-ID: <20071119161741.BE0A781B8@code0.codespeak.net> Author: fijal Date: Mon Nov 19 17:17:40 2007 New Revision: 48803 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: Remove circular import Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Mon Nov 19 17:17:40 2007 @@ -143,7 +143,6 @@ _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] -from pypy.rpython.module.ll_os import CConfig from pypy.rpython.tool import rffi_platform as platform class CConfig: _includes_ = INCLUDES From cfbolz at codespeak.net Mon Nov 19 17:26:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 19 Nov 2007 17:26:48 +0100 (CET) Subject: [pypy-svn] r48804 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071119162648.07AFF81B2@code0.codespeak.net> Author: cfbolz Date: Mon Nov 19 17:26:48 2007 New Revision: 48804 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: some more task changes Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Mon Nov 19 17:26:48 2007 @@ -13,7 +13,8 @@ - finish rctypes removal - - move away from implicit keepalives + - kill remaining suggested_primitives + - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) @@ -49,6 +50,8 @@ - fix the taint space + - fix the thunk space + - review the things implemented at applevel whether they are performance-critical @@ -62,7 +65,6 @@ JIT +++ - - repair the JIT tests (if it doesn't happen before), the keep alive killing merge broke them - start writing real unit tests for the merging logic From akuhn at codespeak.net Tue Nov 20 08:50:13 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Tue, 20 Nov 2007 08:50:13 +0100 (CET) Subject: [pypy-svn] r48806 - pypy/dist/pypy/translator/goal Message-ID: <20071120075013.5035781D4@code0.codespeak.net> Author: akuhn Date: Tue Nov 20 08:50:12 2007 New Revision: 48806 Added: pypy/dist/pypy/translator/goal/targetimageloadingsmalltalk.py - copied unchanged from r48269, pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Removed: pypy/dist/pypy/translator/goal/targetimageloadingmalltalk.py Log: fixed type in name of smalltalk target From antocuni at codespeak.net Tue Nov 20 10:18:17 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 10:18:17 +0100 (CET) Subject: [pypy-svn] r48807 - pypy/dist/pypy/rpython/module Message-ID: <20071120091817.337ED814B@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 10:18:15 2007 New Revision: 48807 Modified: pypy/dist/pypy/rpython/module/ll_os_environ.py Log: fix SyntaxError Modified: pypy/dist/pypy/rpython/module/ll_os_environ.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_environ.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_environ.py Tue Nov 20 10:18:15 2007 @@ -132,7 +132,7 @@ # Access to the 'environ' external variable if sys.platform.startswith('darwin'): - CCHARPPP = rffi.CArrayPtr(rffi.CCHARPP)) + CCHARPPP = rffi.CArrayPtr(rffi.CCHARPP) _os_NSGetEnviron = rffi.llexternal('_NSGetEnviron', [], CCHARPPP, includes=['crt_externs.h']) def os_get_environ(): From cfbolz at codespeak.net Tue Nov 20 11:41:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 11:41:34 +0100 (CET) Subject: [pypy-svn] r48808 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071120104134.D233781EF@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 11:41:33 2007 New Revision: 48808 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Tue Nov 20 11:41:33 2007 @@ -6,42 +6,50 @@ - Tracker-gardening - Review the cpython regression test suite (skips/failures) + - think about/remove orphaned parts + +Wednesday after the planning session: - Start slowly thinking about 1.1 plans + Translation toolchain +++++++++++++++++++++ - - finish rctypes removal + - fix the nightly CPython test runs (Samuele) + + - find out why our memcpy behaves so erratically - - kill remaining suggested_primitives + - finish rctypes removal + - kill remaining suggested_primitives (Armin, Alexander) - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) - delegate finding type stuff like vtables etc to GC, cleaner interface for rtti, simplify translator/c/gc.py + - kill opaque hacks in the C backend - clean up the tangle of including headers in the C backend - - review pdbplus, especially the graph commands, also in the light of - https://codespeak.net/issue/pypy-dev/issue303 and the fact that we - can have more than one translator/annotator around (with the - timeshifter) + - remove the use of pyobjs in the test wrapping functions (Carl Friedrich, + Richard) + + - remove the symbol table from the C backend - kill half concrete wrapper - - - kill opaque hacks on the C backend - decide how to implement constants in rffi - - think about/remove orphaned parts - implement callback mechanism for rffi/lltype/ll2ctypes - - find out why our memcpy behaves so erratically + - refactor OO external objects (Maciek, Anto) - - refactor OO external objects + - review pdbplus, especially the graph commands, also in the light of + https://codespeak.net/issue/pypy-dev/issue303 and the fact that we + can have more than one translator/annotator around (with the + timeshifter) Interpreter +++++++++++ From cfbolz at codespeak.net Tue Nov 20 11:42:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 11:42:55 +0100 (CET) Subject: [pypy-svn] r48809 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071120104255.CEAEC81F0@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 11:42:55 2007 New Revision: 48809 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: another task for armin and alexander Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Tue Nov 20 11:42:55 2007 @@ -17,7 +17,7 @@ - fix the nightly CPython test runs (Samuele) - - find out why our memcpy behaves so erratically + - find out why our memcpy behaves so erratically (Armin, Alexander, quickly) - finish rctypes removal From rxe at codespeak.net Tue Nov 20 11:49:16 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 11:49:16 +0100 (CET) Subject: [pypy-svn] r48810 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071120104916.6A31E8118@code0.codespeak.net> Author: rxe Date: Tue Nov 20 11:49:15 2007 New Revision: 48810 Modified: pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/genexterns.c Log: remove adding another entry point on top of entry point Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Tue Nov 20 11:49:15 2007 @@ -82,30 +82,6 @@ create ll file for c file create codewriter """ - if self.standalone: - # XXX this requires jumping through way too many hoops - - from pypy.rpython.lltypesystem import rffi - getargc = rffi.llexternal('__pypy_getargc', [], rffi.INT) - getargv = rffi.llexternal('__pypy_getargv', [], rffi.CCHARPP) - - def entry_point(args=None): - argc = getargc() - argv = getargv() - if args is None: - args = [] - for ii in range(argc): - s = rffi.charp2str(argv[ii]) - args.append(s) - return func(args) - from pypy.annotation.listdef import s_list_of_strings - graph = self.translator.rtyper.annotate_helper(entry_point, [s_list_of_strings]) - self.translator.rtyper.specialize_more_blocks() - # XXX this is really bad - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(self.translator) - func = entry_point - # XXX please dont ask! from pypy.translator.c.genc import CStandaloneBuilder cbuild = CStandaloneBuilder(self.translator, func, config=self.config) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Tue Nov 20 11:49:15 2007 @@ -16,8 +16,8 @@ #ifdef LL_NEED_STACK FAKE_ERROR(RuntimeError); - #include "c/src/thread.h" - #include "c/src/stack.h" + #include "src/thread.h" + #include "src/stack.h" #endif @@ -47,7 +47,16 @@ #ifdef ENTRY_POINT_DEFINED -int __ENTRY_POINT__(voidzd *); +int _argc; +char **argv; + +int _pypy_getargc() { + return _argc; +} + +char ** _pypy_getargv() { + return _argv; +} int main(int argc, char *argv[]) { char *errmsg = RPython_StartupCode(); @@ -56,7 +65,7 @@ return 1; } - return __ENTRY_POINT__(NULL); + return __ENTRY_POINT__(); } #else From rxe at codespeak.net Tue Nov 20 11:50:28 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 11:50:28 +0100 (CET) Subject: [pypy-svn] r48811 - pypy/dist/pypy/translator/llvm Message-ID: <20071120105028.AF39A8149@code0.codespeak.net> Author: rxe Date: Tue Nov 20 11:50:28 2007 New Revision: 48811 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/typedefnode.py Log: make an array of no length a pointer in llvm backend Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Nov 20 11:50:28 2007 @@ -529,7 +529,7 @@ return repr def repr_offset(self, value): - from_, indices, to = self.get_offset(value) + from_, indices, to = self.get_offset(value, []) # void array special cases if isinstance(from_, lltype.Array) and from_.OF is lltype.Void: @@ -554,12 +554,15 @@ r(from_), indices_as_str) - def get_offset(self, value, initialindices=None): + def get_offset(self, value, indices): " return (from_type, (indices, ...), to_type) " + word = self.database.get_machine_word() - indices = initialindices or [(word, 0)] if isinstance(value, llmemory.ItemOffset): + if not indices: + indices.append((word, 0)) + # skips over a fixed size item (eg array access) from_ = value.TYPE lasttype, lastvalue = indices[-1] @@ -568,6 +571,9 @@ to = value.TYPE elif isinstance(value, llmemory.FieldOffset): + if not indices: + indices.append((word, 0)) + # jumps to a field position in a struct from_ = value.TYPE pos = getindexhelper(value.fldname, value.TYPE) @@ -575,19 +581,32 @@ to = getattr(value.TYPE, value.fldname) elif isinstance(value, llmemory.ArrayLengthOffset): + assert not value.TYPE._hints.get("nolength", False) + + if not indices: + indices.append((word, 0)) + # jumps to the place where the array length is stored from_ = value.TYPE # or assert isinstance(value.TYPE, lltype.Array) - if not value.TYPE._hints.get("nolength", False): - indices.append((word, 0)) + indices.append((word, 0)) to = lltype.Signed elif isinstance(value, llmemory.ArrayItemsOffset): + if not indices: + if isinstance(value.TYPE, lltype.Array) and value.TYPE._hints.get("nolength", False): + pass + else: + indices.append((word, 0)) + # jumps to the beginning of array area from_ = value.TYPE if not isinstance(value.TYPE, lltype.FixedSizeArray) and not value.TYPE._hints.get("nolength", False): indices.append((word, 1)) - indices.append((word, 0)) # go to the 1st item + indices.append((word, 0)) # go to the 1st item + if isinstance(value.TYPE, lltype.FixedSizeArray): + indices.append((word, 0)) # go to the 1st item + to = value.TYPE.OF elif isinstance(value, llmemory.CompositeOffset): Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Tue Nov 20 11:50:28 2007 @@ -113,9 +113,11 @@ if isinstance(ARRAYTYPE, lltype.Array): if not ARRAYTYPE._hints.get("nolength", False): # skip the length field + indices.append((self.word, 0)) indices.append((self.word, 1)) else: assert isinstance(ARRAYTYPE, lltype.FixedSizeArray) + indices.append((self.word, 0)) return indices def write_operation(self, op): @@ -313,7 +315,10 @@ [word], [opr.argrefs[0]]) def to_getelementptr(self, TYPE, args): - indices = [] + if isinstance(TYPE, lltype.Array) and TYPE._hints.get("nolength", False): + indices = [] + else: + indices = [("i32", 0)] for arg in args: name = None # this is because FixedSizeArray can sometimes be accessed like an @@ -351,7 +356,7 @@ op = opr.op _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices, getptr=False) self.codewriter.load(opr.retref, opr.rettype, tmpvar) else: self._skipped(opr) @@ -365,7 +370,7 @@ assert opr.rettype != "void" op = opr.op _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) - self.codewriter.getelementptr(opr.retref, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.getelementptr(opr.retref, opr.argtypes[0], opr.argrefs[0], indices, getptr=False) # struct, name getsubstruct = _getinteriorpointer @@ -377,7 +382,7 @@ if opr.argtypes[-1] != "void": _, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:-1]) tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.getelementptr(tmpvar, opr.argtypes[0], opr.argrefs[0], indices, getptr=False) self.codewriter.store(opr.argtypes[-1], opr.argrefs[-1], tmpvar) else: self._skipped(opr) @@ -392,10 +397,11 @@ op = opr.op TYPE, indices = self.to_getelementptr(op.args[0].concretetype.TO, op.args[1:]) if isinstance(TYPE, lltype.Array): + assert not TYPE._hints.get("nolength", False) # gets the length indices.append(("i32", 0)) lengthref = self._tmp() - self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices) + self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices, getptr=False) else: assert False, "known at compile time" @@ -428,7 +434,7 @@ arraytype = opr.argtypes[0] indices = self._arrayindices(opr.op.args[0]) + [(self.word, 0)] tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, arraytype, array, indices) + self.codewriter.getelementptr(tmpvar, arraytype, array, indices, getptr=False) # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it @@ -440,7 +446,13 @@ arraytype, _ = opr.argtypes tmpvar = self._tmp() - self.codewriter.getelementptr(tmpvar, arraytype, array, [(self.word, incr)]) + + indices = [] + ARRAY = opr.op.args[0].concretetype.TO + if not (isinstance(ARRAY, lltype.Array) and ARRAY._hints.get("nolength", False)): + indices.append( (self.word, 0)) + indices.append((self.word, incr)) + self.codewriter.getelementptr(tmpvar, arraytype, array, indices, getptr=False) # getelementptr gets a pointer to the right type, except the generated code really expected # an array of size 1... so we just cast it Modified: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/typedefnode.py (original) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Tue Nov 20 11:50:28 2007 @@ -34,7 +34,7 @@ def writetypedef(self, codewriter): if self.ARRAY._hints.get("nolength", False): codewriter.typedef(self.ref, - "[0 x %s]" % self.db.repr_type(self.ARRAY.OF)) + "%s" % self.db.repr_type(self.ARRAY.OF)) else: codewriter.typedef(self.ref, "{ %s, [0 x %s] }" % (self.db.get_machine_word(), From rxe at codespeak.net Tue Nov 20 11:51:59 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 11:51:59 +0100 (CET) Subject: [pypy-svn] r48812 - pypy/dist/pypy/translator/llvm Message-ID: <20071120105159.BE7A18149@code0.codespeak.net> Author: rxe Date: Tue Nov 20 11:51:59 2007 New Revision: 48812 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: change path a bit Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Tue Nov 20 11:51:59 2007 @@ -193,8 +193,8 @@ return decls def get_c_cpath(): - from pypy.translator import translator - return os.path.dirname(translator.__file__) + from pypy.translator.c import genc + return os.path.dirname(genc.__file__) def get_llvm_cpath(): return os.path.join(os.path.dirname(__file__), "module") From rxe at codespeak.net Tue Nov 20 11:58:01 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 11:58:01 +0100 (CET) Subject: [pypy-svn] r48813 - pypy/branch/new-genc-tests-wrapper Message-ID: <20071120105801.4F9BE8149@code0.codespeak.net> Author: rxe Date: Tue Nov 20 11:58:00 2007 New Revision: 48813 Added: pypy/branch/new-genc-tests-wrapper/ - copied from r48812, pypy/dist/ Log: Make new branch to play about with tests that are not using test wrappers. From rxe at codespeak.net Tue Nov 20 13:05:46 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 13:05:46 +0100 (CET) Subject: [pypy-svn] r48814 - in pypy/branch/new-genc-tests-wrapper/pypy/translator/c: . test Message-ID: <20071120120546.2CACB81CC@code0.codespeak.net> Author: rxe Date: Tue Nov 20 13:05:44 2007 New Revision: 48814 Removed: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/symboltable.py pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_symboltable.py Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Log: (cfbolz, rxe) remove symbol table which was commented out and broken anyways Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 13:05:44 2007 @@ -20,7 +20,6 @@ class CBuilder(object): c_source_filename = None _compiled = False - symboltable = None modulename = None def __init__(self, translator, entrypoint, config, libraries=None, @@ -141,14 +140,10 @@ CBuilder.have___thread = check_under_under_thread() if not self.standalone: assert not self.config.translation.instrument - from pypy.translator.c.symboltable import SymbolTable - # XXX fix symboltable - #self.symboltable = SymbolTable() cfile, extra, include_dirs, library_dirs = \ gen_source(db, modulename, targetdir, defines = defines, exports = self.exports, - symboltable = self.symboltable, libraries = self.libraries) else: if self.config.translation.instrument: @@ -208,8 +203,6 @@ mod = import_module_from_directory(self.c_source_filename.dirpath(), self.c_source_filename.purebasename) self.c_ext_module = mod - if self.symboltable: - self.symboltable.attach(mod) # hopefully temporary hack return mod def isolated_import(self): @@ -712,7 +705,7 @@ def gen_source(database, modulename, targetdir, defines={}, exports={}, - symboltable=None, libraries=[]): + libraries=[]): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -759,27 +752,6 @@ sg.write_extra_sources(sources) # - # Debugging info - # - if symboltable: - print >> f - print >> f, '/*******************************************************/' - print >> f, '/*** Debugging info ***/' - print >> f - print >> f, 'static int debuginfo_offsets[] = {' - for node in database.structdefnodes.values(): - for expr in symboltable.generate_type_info(database, node): - print >> f, '\t%s,' % expr - print >> f, '\t0 };' - print >> f, 'static void *debuginfo_globals[] = {' - for node in database.globalcontainers(): - if not isinstance(node, PyObjectNode): - result = symboltable.generate_global_info(database, node) - print >> f, '\t%s,' % (result,) - print >> f, '\tNULL };' - print >> f, '#include "src/debuginfo.h"' - - # # PyObject support (strange) code # pyobjmaker = database.pyobjmaker From fijal at codespeak.net Tue Nov 20 13:14:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:14:52 +0100 (CET) Subject: [pypy-svn] r48816 - pypy/dist/pypy/translator Message-ID: <20071120121452.3B98B81AD@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:14:51 2007 New Revision: 48816 Modified: pypy/dist/pypy/translator/driver.py Log: (fijal, anto) Remove the squeak and lisp from the driver Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Nov 20 13:14:51 2007 @@ -589,42 +589,6 @@ "Running compiled llvm source", idemp=True) - def task_source_cl(self): - from pypy.translator.lisp.gencl import GenCL - self.gen = GenCL(self.translator, self.entry_point) - filename = self.gen.emitfile() - self.log.info("Wrote %s" % (filename,)) - task_source_cl = taskdef(task_source_cl, [OOTYPE], - 'Generating Common Lisp source') - - def task_compile_cl(self): - pass - task_compile_cl = taskdef(task_compile_cl, ['source_cl'], - 'XXX') - - def task_run_cl(self): - pass - task_run_cl = taskdef(task_run_cl, ['compile_cl'], - 'XXX') - - def task_source_squeak(self): - from pypy.translator.squeak.gensqueak import GenSqueak - self.gen = GenSqueak(dir, self.translator) - filename = self.gen.gen() - self.log.info("Wrote %s" % (filename,)) - task_source_squeak = taskdef(task_source_squeak, [OOTYPE], - 'Generating Squeak source') - - def task_compile_squeak(self): - pass - task_compile_squeak = taskdef(task_compile_squeak, ['source_squeak'], - 'XXX') - - def task_run_squeak(self): - pass - task_run_squeak = taskdef(task_run_squeak, ['compile_squeak'], - 'XXX') - def task_source_js(self): from pypy.translator.js.js import JS self.gen = JS(self.translator, functions=[self.entry_point], From fijal at codespeak.net Tue Nov 20 13:19:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:19:18 +0100 (CET) Subject: [pypy-svn] r48817 - pypy/dist/pypy/translator/test Message-ID: <20071120121918.BE03D81C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:19:18 2007 New Revision: 48817 Modified: pypy/dist/pypy/translator/test/test_driver.py Log: (fijal, anto) Refactor test to adapt of removal of squeak and cl Modified: pypy/dist/pypy/translator/test/test_driver.py ============================================================================== --- pypy/dist/pypy/translator/test/test_driver.py (original) +++ pypy/dist/pypy/translator/test/test_driver.py Tue Nov 20 13:19:18 2007 @@ -3,19 +3,12 @@ from pypy.translator.driver import TranslationDriver from py.compat import optparse -def cmpl(l1, l2): - l1 = list(l1) - l2 = list(l2) - l1.sort() - l2.sort() - return l1 == l2 - def test_ctr(): td = TranslationDriver() expected = ['annotate', 'backendopt', 'llinterpret', 'rtype', 'source', 'compile', 'run', 'prehannotatebackendopt', 'hintannotate', 'timeshift'] - assert cmpl(td.exposed, expected) + assert set(td.exposed) == set(expected) assert td.backend_select_goals(['compile_c']) == ['compile_c'] assert td.backend_select_goals(['compile']) == ['compile_c'] @@ -38,15 +31,15 @@ expected = ['annotate', 'backendopt_lltype', 'backendopt_ootype', 'llinterpret_lltype', - 'rtype_ootype', 'rtype_lltype', 'source_cl', 'source_js', - 'source_squeak', 'source_cli', 'source_c', 'source_llvm', - 'compile_cl', 'compile_cli', 'compile_c', 'compile_squeak', - 'compile_llvm', 'compile_js', 'run_cl', 'run_squeak', + 'rtype_ootype', 'rtype_lltype', 'source_js', + 'source_cli', 'source_c', 'source_llvm', + 'compile_cli', 'compile_c', + 'compile_llvm', 'compile_js', 'run_llvm', 'run_c', 'run_js', 'run_cli', 'compile_jvm', 'source_jvm', 'run_jvm', 'prehannotatebackendopt_lltype', 'hintannotate_lltype', 'timeshift_lltype'] - assert cmpl(td.exposed, expected) + assert set(td.exposed) == set(expected) td = TranslationDriver({'backend': None, 'type_system': 'lltype'}) @@ -62,4 +55,4 @@ 'source_llvm', 'compile_c', 'compile_llvm', 'run_llvm', 'run_c', 'prehannotatebackendopt', 'hintannotate', 'timeshift'] - assert cmpl(td.exposed, expected) + assert set(td.exposed) == set(expected) From fijal at codespeak.net Tue Nov 20 13:21:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:21:16 +0100 (CET) Subject: [pypy-svn] r48818 - pypy/dist/pypy/translator/test Message-ID: <20071120122116.E32B181AD@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:21:16 2007 New Revision: 48818 Modified: pypy/dist/pypy/translator/test/test_interactive.py Log: s/squeak/cli in rtyper tests Modified: pypy/dist/pypy/translator/test/test_interactive.py ============================================================================== --- pypy/dist/pypy/translator/test/test_interactive.py (original) +++ pypy/dist/pypy/translator/test/test_interactive.py Tue Nov 20 13:21:16 2007 @@ -138,20 +138,20 @@ t = Translation(f) s = t.annotate([int, int]) - t.rtype(backend='squeak') + t.rtype(backend='cli') assert 'rtype_ootype' in t.driver.done - t = Translation(f, backend='squeak', type_system='ootype') + t = Translation(f, backend='cli', type_system='ootype') s = t.annotate([int, int]) t.rtype() assert 'rtype_ootype' in t.driver.done t = Translation(f, type_system='lltype') s = t.annotate([int, int]) - py.test.raises(Exception, "t.rtype(backend='squeak')") + py.test.raises(Exception, "t.rtype(backend='cli')") - t = Translation(f, backend='squeak') + t = Translation(f, backend='cli') s = t.annotate([int, int]) py.test.raises(Exception, "t.rtype(type_system='lltype')") From fijal at codespeak.net Tue Nov 20 13:21:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:21:31 +0100 (CET) Subject: [pypy-svn] r48819 - pypy/dist/pypy/config Message-ID: <20071120122131.7E37481AD@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:21:31 2007 New Revision: 48819 Modified: pypy/dist/pypy/config/translationoption.py Log: Remove squeak and cl Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Tue Nov 20 13:21:31 2007 @@ -24,7 +24,7 @@ ] }), ChoiceOption("backend", "Backend to use for code generation", - ["c", "llvm", "cli", "jvm", "js", "squeak", "cl"], + ["c", "llvm", "cli", "jvm", "js"], requires={ "c": [("translation.type_system", "lltype")], "llvm": [("translation.type_system", "lltype"), @@ -33,8 +33,6 @@ "cli": [("translation.type_system", "ootype")], "jvm": [("translation.type_system", "ootype")], "js": [("translation.type_system", "ootype")], - "squeak": [("translation.type_system", "ootype")], - "cl": [("translation.type_system", "ootype")], }, cmdline="-b --backend"), BoolOption("llvm_via_c", "compile llvm via C", From fijal at codespeak.net Tue Nov 20 13:22:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:22:18 +0100 (CET) Subject: [pypy-svn] r48820 - in pypy/dist/pypy/translator: lisp squeak Message-ID: <20071120122218.83B6381C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:22:18 2007 New Revision: 48820 Removed: pypy/dist/pypy/translator/lisp/ pypy/dist/pypy/translator/squeak/ Log: Remove squeak and lisp backends. From fijal at codespeak.net Tue Nov 20 13:25:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:25:53 +0100 (CET) Subject: [pypy-svn] r48821 - pypy/dist/pypy/lib/ctypes Message-ID: <20071120122553.3DADE80CD@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:25:52 2007 New Revision: 48821 Added: pypy/dist/pypy/lib/ctypes/_basics.py (contents, props changed) pypy/dist/pypy/lib/ctypes/_pointers.py (contents, props changed) Modified: pypy/dist/pypy/lib/ctypes/__init__.py Log: Shuffle stuff around Modified: pypy/dist/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/dist/pypy/lib/ctypes/__init__.py (original) +++ pypy/dist/pypy/lib/ctypes/__init__.py Tue Nov 20 13:25:52 2007 @@ -28,69 +28,8 @@ self._handle = self.lib._handle.ptr(self.name, llargs, self.restype._type_) -class _SimpleCData(object): - def __init__(self, value): - self.value = value - -class c_ushort(_SimpleCData): - _type_ = 'H' - -class c_double(_SimpleCData): - _type_ = 'd' - -class c_ubyte(_SimpleCData): - _type_ = 'B' - -class c_float(_SimpleCData): - _type_ = 'f' - -class c_ulong(_SimpleCData): - _type_ = 'L' - -class c_short(_SimpleCData): - _type_ = 'h' - -class c_ubyte(_SimpleCData): - _type_ = 'b' - -class c_byte(_SimpleCData): - _type_ = 'B' - -class c_char(_SimpleCData): - _type_ = 'c' - -class c_long(_SimpleCData): - _type_ = 'l' - -class c_ulonglong(_SimpleCData): - _type_ = 'Q' - -class c_longlong(_SimpleCData): - _type_ = 'q' - -class c_int(_SimpleCData): - _type_ = 'i' - -class c_uint(_SimpleCData): - _type_ = 'I' - -class c_double(_SimpleCData): - _type_ = 'd' - -class c_float(_SimpleCData): - _type_ = 'f' - -c_size_t = c_ulong # XXX - -class POINTER(object): - def __init__(self, cls): - self.cls = cls - -class c_void_p(_SimpleCData): - _type_ = 'P' - -class c_char_p(_SimpleCData): - _type_ = 's' +from ctypes._basics import * +from ctypes._pointers import POINTER class CDLL(object): """An instance of this class represents a loaded dll/shared Added: pypy/dist/pypy/lib/ctypes/_basics.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/ctypes/_basics.py Tue Nov 20 13:25:52 2007 @@ -0,0 +1,60 @@ + +class _SimpleCData(object): + def __init__(self, value): + self.value = value + +class c_ushort(_SimpleCData): + _type_ = 'H' + +class c_double(_SimpleCData): + _type_ = 'd' + +class c_ubyte(_SimpleCData): + _type_ = 'B' + +class c_float(_SimpleCData): + _type_ = 'f' + +class c_ulong(_SimpleCData): + _type_ = 'L' + +class c_short(_SimpleCData): + _type_ = 'h' + +class c_ubyte(_SimpleCData): + _type_ = 'b' + +class c_byte(_SimpleCData): + _type_ = 'B' + +class c_char(_SimpleCData): + _type_ = 'c' + +class c_long(_SimpleCData): + _type_ = 'l' + +class c_ulonglong(_SimpleCData): + _type_ = 'Q' + +class c_longlong(_SimpleCData): + _type_ = 'q' + +class c_int(_SimpleCData): + _type_ = 'i' + +class c_uint(_SimpleCData): + _type_ = 'I' + +class c_double(_SimpleCData): + _type_ = 'd' + +class c_float(_SimpleCData): + _type_ = 'f' + +c_size_t = c_ulong # XXX + +class c_void_p(_SimpleCData): + _type_ = 'P' + +class c_char_p(_SimpleCData): + _type_ = 's' Added: pypy/dist/pypy/lib/ctypes/_pointers.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/ctypes/_pointers.py Tue Nov 20 13:25:52 2007 @@ -0,0 +1,31 @@ + +# This cache maps types to pointers to them. +_pointer_type_cache = {} + +class PointerMetaclass(type): + def __new__(self, name, superclass, attrs): + res = super(PointerMetaclass, self).__new__(self, name, superclass, + attrs) + return res + +class _Pointer(object): + __metaclass__ = PointerMetaclass + +def POINTER(cls): + try: + return _pointer_type_cache[cls] + except KeyError: + pass + if type(cls) is str: + klass = type(_Pointer)("LP_%s" % cls, + (_Pointer,), + {}) + _pointer_type_cache[id(klass)] = klass + return klass + else: + name = "LP_%s" % cls.__name__ + klass = type(_Pointer)(name, + (_Pointer,), + {'_type_': cls}) + _pointer_type_cache[cls] = klass + return klass From fijal at codespeak.net Tue Nov 20 13:26:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:26:02 +0100 (CET) Subject: [pypy-svn] r48822 - in pypy/dist/pypy/lib: . test2 Message-ID: <20071120122602.F241E8141@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:26:02 2007 New Revision: 48822 Removed: pypy/dist/pypy/lib/aop.py pypy/dist/pypy/lib/dbc.py pypy/dist/pypy/lib/test2/test_aop.py Log: Remove aop and dbc From fijal at codespeak.net Tue Nov 20 13:29:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:29:36 +0100 (CET) Subject: [pypy-svn] r48823 - in pypy/dist/pypy: config lib/cslib lib/pyontology module/_cslib module/cclp Message-ID: <20071120122936.C827C8124@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:29:35 2007 New Revision: 48823 Removed: pypy/dist/pypy/lib/cslib/ pypy/dist/pypy/lib/pyontology/ pypy/dist/pypy/module/_cslib/ pypy/dist/pypy/module/cclp/ Modified: pypy/dist/pypy/config/pypyoption.py Log: Remove constraint solving modules. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Tue Nov 20 13:29:35 2007 @@ -46,8 +46,6 @@ requires = { "logic": [#("objspace.geninterp", False), ("objspace.usemodules._stackless", True), - ("objspace.usemodules._cslib", True), - ("objspace.usemodules.cclp", True), ("translation.gc", 'marksweep'), ], }, From fijal at codespeak.net Tue Nov 20 13:30:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:30:25 +0100 (CET) Subject: [pypy-svn] r48824 - pypy/dist/pypy/doc/config Message-ID: <20071120123025.96F2E8124@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:30:25 2007 New Revision: 48824 Removed: pypy/dist/pypy/doc/config/objspace.usemodules._cslib.txt pypy/dist/pypy/doc/config/objspace.usemodules.cclp.txt Log: Remove documentation about constraint solving From antocuni at codespeak.net Tue Nov 20 13:32:12 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 13:32:12 +0100 (CET) Subject: [pypy-svn] r48825 - pypy/dist/pypy/doc Message-ID: <20071120123212.29F7581C3@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 13:32:11 2007 New Revision: 48825 Modified: pypy/dist/pypy/doc/_ref.txt pypy/dist/pypy/doc/dynamic-language-translation.txt pypy/dist/pypy/doc/faq.txt pypy/dist/pypy/doc/glossary.txt pypy/dist/pypy/doc/index.txt pypy/dist/pypy/doc/project-ideas.txt pypy/dist/pypy/doc/translation.txt Log: (antocuni, fijal) remove references to the dead lisp and squeak backends Modified: pypy/dist/pypy/doc/_ref.txt ============================================================================== --- pypy/dist/pypy/doc/_ref.txt (original) +++ pypy/dist/pypy/doc/_ref.txt Tue Nov 20 13:32:11 2007 @@ -122,7 +122,6 @@ .. _`pypy/translator/goal/targetprologstandalone.py`: ../../pypy/translator/goal/targetprologstandalone.py .. _`translator/js/`: ../../pypy/translator/js .. _`translator/jvm/`: ../../pypy/translator/jvm -.. _`translator/lisp/`: ../../pypy/translator/lisp .. _`translator/llvm/`: ../../pypy/translator/llvm .. _`translator/squeak/`: ../../pypy/translator/squeak .. _`translator/stackless/`: ../../pypy/translator/stackless Modified: pypy/dist/pypy/doc/dynamic-language-translation.txt ============================================================================== --- pypy/dist/pypy/doc/dynamic-language-translation.txt (original) +++ pypy/dist/pypy/doc/dynamic-language-translation.txt Tue Nov 20 13:32:11 2007 @@ -2050,8 +2050,8 @@ We can currently generate C-like low-level flow graphs and turn them into either C or LLVM_ code; or experimentally, PowerPC machine code or JavaScript_. If we go through slightly less low-level flow graphs -instead, we can also interface with an experimental back-end generating -Squeak_ and in the future Java and/or .NET. +instead, we can also interface with back-ends generating bytecode for +JVM, CLI/.NET and even Javascript. RTyper @@ -2171,10 +2171,9 @@ styles of code for different execution models like coroutines. We will give as an example an overview of the `GenC back-end`_. The -`LLVM back-end`_ works at the same level. The (undocumented) Squeak -back-end takes ootyped graphs instead, as described above, and faces -different problems (e.g. the graphs have unstructured control flow, so -they are difficult to render in a language with no ``goto`` equivalent). +`LLVM back-end`_ works at the same level. The CLI and JVM +back-ends take ootyped graphs instead, as described above, and faces +different problems. .. _`Generating C code`: @@ -2330,8 +2329,6 @@ * RTyper (RPython Low-level Typer): see [TR]_. -* Squeak: http://www.squeak.org/ - * SSA (Static Single Assignment): http://en.wikipedia.org/wiki/Static_single_assignment_form * Standard Object Space: see `Object Space`_. @@ -2354,7 +2351,6 @@ .. _`GenC back-end`: translation.html#genc .. _`LLVM back-end`: translation.html#llvm .. _JavaScript: http://www.ecma-international.org/publications/standards/Ecma-262.htm -.. _Squeak: http://www.squeak.org/ .. _lltype: rtyper.html#low-level-types .. _`post on the Perl 6 compiler mailing list`: http://www.nntp.perl.org/group/perl.perl6.compiler/1107 Modified: pypy/dist/pypy/doc/faq.txt ============================================================================== --- pypy/dist/pypy/doc/faq.txt (original) +++ pypy/dist/pypy/doc/faq.txt Tue Nov 20 13:32:11 2007 @@ -339,18 +339,13 @@ Backends that can actually translate all of PyPy: - * C_, LLVM_, CLI_ + * C_, LLVM_, CLI_, JVM_ Somewhat mature backends: * Low level backends: C_, LLVM_ * High level backends: CLI_, JVM_, JavaScript_ -Partially implemented backends (both high-level): - -* Squeak_, `Common Lisp`_ - - To learn more about backends take a look at the `translation document`_. .. _CLI: cli-backend.html @@ -359,8 +354,6 @@ .. _LLVM: translation.html#the-llvm-back-end .. _`translation document`: translation.html .. _JVM: translation.html#genjvm -.. _Squeak: translation.html#gensqueak -.. _`Common Lisp`: translation.html#gencl ---------------------- How do I compile PyPy? Modified: pypy/dist/pypy/doc/glossary.txt ============================================================================== --- pypy/dist/pypy/doc/glossary.txt (original) +++ pypy/dist/pypy/doc/glossary.txt Tue Nov 20 13:32:11 2007 @@ -116,8 +116,8 @@ **ootypesystem** An `object oriented type model `__ containing classes and instances. A backend_ that uses this type system - is also called a high-level backend. The common lisp, javascript and - cli backends all use this typesystem. + is also called a high-level backend. The JVM, Javascript and + CLI backends all use this typesystem. .. _`prebuilt constant`: Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Tue Nov 20 13:32:11 2007 @@ -301,15 +301,11 @@ `translator/js/`_ the `JavaScript backend`_ -`translator/jvm/`_ the Java backend (in-progress) - -`translator/lisp/`_ the `Common Lisp backend`_ (incomplete) +`translator/jvm/`_ the Java backend `translator/llvm/`_ contains the `LLVM backend`_ producing LLVM assembler from fully annotated RPython programs -`translator/squeak/`_ the `Squeak backend`_ (very incomplete) - `translator/stackless/`_ the `Stackless Transform`_ `translator/tool/`_ helper tools for translation, including the Pygame @@ -348,8 +344,6 @@ .. _`GenC backend`: translation.html#genc .. _`LLVM backend`: translation.html#llvm .. _`CLI backend`: cli-backend.html -.. _`Common Lisp backend`: translation.html#gencl -.. _`Squeak backend`: translation.html#gensqueak .. _`extension compiler`: extcompiler.html .. _`py.py`: getting-started.html#main-entry-point .. _`translatorshell.py`: getting-started.html#try-out-the-translator Modified: pypy/dist/pypy/doc/project-ideas.txt ============================================================================== --- pypy/dist/pypy/doc/project-ideas.txt (original) +++ pypy/dist/pypy/doc/project-ideas.txt Tue Nov 20 13:32:11 2007 @@ -66,11 +66,10 @@ Start or improve a back-end --------------------------- -PyPy has complete, or nearly so, back-ends for C, LLVM, CLI/.NET and JVM and partial -backends for JavaScript, Common Lisp, Squeak. It would be an -interesting project to improve either of these partial backends, or start one -for another platform (Objective C comes to mind as one that should not be too -terribly hard). +PyPy has complete, or nearly so, back-ends for C, LLVM, CLI/.NET and +JVM. It would be an interesting project to improve either of these +partial backends, or start one for another platform (Objective C comes +to mind as one that should not be too terribly hard). Improve JavaScript backend -------------------------- Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Tue Nov 20 13:32:11 2007 @@ -28,8 +28,8 @@ introduce them. As of the 1.0 release, RPython_ programs can be translated into the following -languages/platforms: C/POSIX, LLVM/POSIX, CLI/.NET, Squeak, Javascript, -Common Lisp and Java/JVM (in addition, there's `a backend`_ that translates +languages/platforms: C/POSIX, LLVM/POSIX, CLI/.NET, Javascript +and Java/JVM (in addition, there's `a backend`_ that translates `application-level`_ into `interpreter-level`_ code, but this is a special case in several ways). @@ -106,7 +106,6 @@ .. _`bytecode evaluator`: interpreter.html .. _`abstract interpretation`: theory.html#abstract-interpretation .. _`Flow Object Space`: objspace.html#the-flow-object-space -.. _`Common Lisp`: ../../pypy/translator/lisp/ .. _`interactive interface`: getting-started.html#try-out-the-translator .. _`translatorshell.py`: ../../pypy/bin/translatorshell.py @@ -791,20 +790,6 @@ GenJVM is almost entirely the work of Niko Matsakis, who worked on it also as part of the Summer of PyPy program. -GenSqueak -+++++++++ - -GenSqueak translates RPython programs into source that can be run in `Squeak -`__, the free Smalltalk implementation. - -It is probably the least complete of the backends listed here. - -The first version was written by Bert Freudenberg of Impara GmbH, mostly as a -way of motivating the design of the ootypesystem. More recently, it has been -worked on by Niklaus Haldimann. - -GenSqueak is considered dead now. - GenJS +++++ @@ -832,26 +817,6 @@ .. _`whatis document`: js/whatis.html .. _`using document`: js/using.html -GenCL -+++++ - -GenCL generates `ANSI Common Lisp`_ source. - -.. _`ANSI Common Lisp`: http://www.lisp.org/HyperSpec/FrontMatter/index.html - -GenCL is one of the oldest backends -- some version of it has been present -since before revision 2000, long long before any version of the RTyper -existed! At the Tokyo sprint in April 2006, GenCL was revived and ported to -work on OOTyped graphs. - -GenCL has seen work from many people, but the driving force has mostly been -Seo Sanghyeon. - -GenCL is considered dead now. - -.. _`Python again`: -.. _`Python back-end`: - The Interpreter-Level backend ----------------------------- From fijal at codespeak.net Tue Nov 20 13:33:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:33:13 +0100 (CET) Subject: [pypy-svn] r48826 - pypy/dist/pypy/rlib/cslib Message-ID: <20071120123313.D59BA81C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:33:13 2007 New Revision: 48826 Removed: pypy/dist/pypy/rlib/cslib/ Log: Remove constraint solving lib here as well. From fijal at codespeak.net Tue Nov 20 13:36:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:36:49 +0100 (CET) Subject: [pypy-svn] r48827 - in pypy/dist/pypy: lib/constraint objspace objspace/test Message-ID: <20071120123649.29EF081C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:36:48 2007 New Revision: 48827 Removed: pypy/dist/pypy/lib/constraint/ pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/_test_logic_build.py pypy/dist/pypy/objspace/test/test_logiccloning.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: Remove logic objspace From fijal at codespeak.net Tue Nov 20 13:38:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:38:16 +0100 (CET) Subject: [pypy-svn] r48828 - pypy/dist/pypy/lib Message-ID: <20071120123816.680C981C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:38:16 2007 New Revision: 48828 Modified: pypy/dist/pypy/lib/_exceptions.py Log: Remove logic-specific exceptions Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Tue Nov 20 13:38:16 2007 @@ -438,18 +438,3 @@ self.encoding, self.start, self.end - 1, self.reason) - -#-- Logic object space specific stuff -#XXX conditionalize me on '-o logic' - -class LogicError(Exception): pass - -class UnificationError(LogicError): pass -class RebindingError(UnificationError): pass -class FutureBindingError(LogicError): pass - -class AllBlockedError(LogicError): pass - -# constraints - -class ConsistencyError(LogicError): pass From antocuni at codespeak.net Tue Nov 20 13:39:17 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 13:39:17 +0100 (CET) Subject: [pypy-svn] r48829 - in pypy/dist/pypy/doc: . config Message-ID: <20071120123917.69A9081C3@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 13:39:16 2007 New Revision: 48829 Removed: pypy/dist/pypy/doc/constraints-and-logic.txt pypy/dist/pypy/doc/howto-logicobjspace.txt Modified: pypy/dist/pypy/doc/config/objspace.name.txt pypy/dist/pypy/doc/index.txt pypy/dist/pypy/doc/objspace-proxies.txt Log: remove references to logic objspace from the doc Modified: pypy/dist/pypy/doc/config/objspace.name.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.name.txt (original) +++ pypy/dist/pypy/doc/config/objspace.name.txt Tue Nov 20 13:39:16 2007 @@ -4,7 +4,6 @@ for normal usage): * thunk_: The thunk object space adds lazy evaluation to PyPy. - * logic_: The logic object space contains logical programming features. * taint_: The taint object space adds soft security features. * dump_: Using this object spaces results in the dumpimp of all operations to a log. @@ -13,6 +12,5 @@ .. _`Object Space Proxies`: ../objspace-proxies.html .. _`Standard Object Space`: ../objspace.html#standard-object-space .. _thunk: ../objspace-proxies.html#thunk -.. _logic: ../objspace-proxies.html#logic .. _taint: ../objspace-proxies.html#taint .. _dump: ../objspace-proxies.html#dump Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Tue Nov 20 13:39:16 2007 @@ -263,8 +263,6 @@ `objspace/thunk.py`_ the `thunk object space`_, providing unique object features -`objspace/logic.py`_ the `logic object space`_, providing Prolog-like logic variables - `objspace/cpy/`_ an object space supporting the `extension compiler`_ `objspace/flow/`_ the FlowObjSpace_ implementing `abstract interpretation` @@ -330,7 +328,6 @@ .. _`trace object space`: objspace.html#the-trace-object-space .. _`taint object space`: objspace-proxies.html#taint .. _`thunk object space`: objspace-proxies.html#thunk -.. _`logic object space`: objspace-proxies.html#logic .. _`transparent proxies`: objspace-proxies.html#tproxy .. _`What PyPy can do for your objects`: objspace-proxies.html .. _`Stackless and coroutines`: stackless.html Modified: pypy/dist/pypy/doc/objspace-proxies.txt ============================================================================== --- pypy/dist/pypy/doc/objspace-proxies.txt (original) +++ pypy/dist/pypy/doc/objspace-proxies.txt Tue Nov 20 13:39:16 2007 @@ -18,9 +18,6 @@ only if and when needed; and a way to globally replace an object with another. -* *Logic Object Space*: logical variables, i.e. placeholder objects - whose value can be bound once. - * *Taint Object Space*: a soft security system; your application cannot accidentally compute results based on tainted objects unless it explicitly untaints them first. @@ -136,33 +133,6 @@ function behaves lazily: all calls to it return a thunk object. -.. _logic: - -The Logic Object Space -====================== - -The Logic Object Space adds several logic programming features, -including the notion of logical variable. A logical variable is really -an object from the Python point of view; it is called "variable" for -consistency with logic programming terminology. It is an "empty" object -with no initial value: its value can be set once, and only once, at any -point in time. - -Logical variables are not entirely different from lazily-computed objects, -except that the objects have no built-in knowledge about how they should -compute themselves. - -Using a logical variable before it gets a value results in a lock: -the current thread is suspended, in the hope that another thread will -eventually give it a value. In practice, this works well -with microthreads instead of real threads (see `Stackless features`_). - -The `EU Interim Report`_ (PDF) describes the Logic Object Space in -more detail. - -.. _`EU Interim Report`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2007-02-28.pdf -.. _`Stackless features`: stackless.html - .. _taint: The Taint Object Space From fijal at codespeak.net Tue Nov 20 13:41:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:41:04 +0100 (CET) Subject: [pypy-svn] r48830 - pypy/dist/pypy/config Message-ID: <20071120124104.D42E881DA@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:41:04 2007 New Revision: 48830 Modified: pypy/dist/pypy/config/pypyoption.py Log: Remove logic objspace from options Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Tue Nov 20 13:41:04 2007 @@ -41,14 +41,8 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ ChoiceOption("name", "Object Space name", - ["std", "flow", "logic", "thunk", "cpy", "dump", "taint"], + ["std", "flow", "thunk", "cpy", "dump", "taint"], "std", - requires = { - "logic": [#("objspace.geninterp", False), - ("objspace.usemodules._stackless", True), - ("translation.gc", 'marksweep'), - ], - }, cmdline='--objspace -o'), ChoiceOption("parser", "which parser to use for app-level code", From fijal at codespeak.net Tue Nov 20 13:43:50 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:43:50 +0100 (CET) Subject: [pypy-svn] r48831 - pypy/dist/pypy/config/test Message-ID: <20071120124350.E1E6A81C3@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:43:50 2007 New Revision: 48831 Modified: pypy/dist/pypy/config/test/test_config.py Log: Make tests more sane Modified: pypy/dist/pypy/config/test/test_config.py ============================================================================== --- pypy/dist/pypy/config/test/test_config.py (original) +++ pypy/dist/pypy/config/test/test_config.py Tue Nov 20 13:43:50 2007 @@ -5,7 +5,7 @@ gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref') gcdummy = BoolOption('dummy', 'dummy', default=False) objspaceoption = ChoiceOption('objspace', 'Object space', - ['std', 'logic'], 'std') + ['std', 'thunk'], 'std') booloption = BoolOption('bool', 'Test boolean option', default=True) intoption = IntOption('int', 'Test int option', default=0) floatoption = FloatOption('float', 'Test float option', default=2.3) @@ -34,8 +34,8 @@ assert getattr(config, "gc.name") == 'framework' assert config.objspace == 'std' - config.objspace = 'logic' - assert config.objspace == 'logic' + config.objspace = 'thunk' + assert config.objspace == 'thunk' assert config.gc.float == 2.3 assert config.int == 0 From antocuni at codespeak.net Tue Nov 20 13:44:58 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 13:44:58 +0100 (CET) Subject: [pypy-svn] r48832 - pypy/dist/pypy/doc Message-ID: <20071120124458.4742381C3@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 13:44:57 2007 New Revision: 48832 Removed: pypy/dist/pypy/doc/pyontology.txt Log: remove this file too From fijal at codespeak.net Tue Nov 20 13:49:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 13:49:03 +0100 (CET) Subject: [pypy-svn] r48833 - pypy/dist/pypy/module/thread/test Message-ID: <20071120124903.682C181CC@code0.codespeak.net> Author: fijal Date: Tue Nov 20 13:49:03 2007 New Revision: 48833 Modified: pypy/dist/pypy/module/thread/test/test_ll_thread.py Log: Remove the never-to-work-and-not-needed-test Modified: pypy/dist/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_ll_thread.py Tue Nov 20 13:49:03 2007 @@ -90,16 +90,3 @@ freed_counter = fn() print freed_counter assert freed_counter > 0 - -def test_prebuilt_lock(): - py.test.skip("Does not work (prebuilt opaque object)") - l = allocate_lock() - - def f(): - l.acquire(True) - l.release() - - fn = compile(f, [], gcpolicy='boehm') - fn() - - From rxe at codespeak.net Tue Nov 20 13:51:08 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 13:51:08 +0100 (CET) Subject: [pypy-svn] r48834 - pypy/branch/new-genc-tests-wrapper/pypy/translator/c Message-ID: <20071120125108.E11CA81C3@code0.codespeak.net> Author: rxe Date: Tue Nov 20 13:51:08 2007 New Revision: 48834 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Log: (cfbolz, rxe) removed some code which did something sometime Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 13:51:08 2007 @@ -335,13 +335,6 @@ f.close() -def translator2database(translator, entrypoint): - pf = lltype.pyobjectptr(entrypoint) - db = LowLevelDatabase(translator) - db.get(pf) - db.complete() - return db, pf - # ____________________________________________________________ SPLIT_CRITERIA = 65535 # support VC++ 7.2 From rxe at codespeak.net Tue Nov 20 14:07:24 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 14:07:24 +0100 (CET) Subject: [pypy-svn] r48835 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c c/test tool Message-ID: <20071120130724.40E5B81C2@code0.codespeak.net> Author: rxe Date: Tue Nov 20 14:07:23 2007 New Revision: 48835 Removed: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_wrapping.py pypy/branch/new-genc-tests-wrapper/pypy/translator/tool/raymond.py Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Log: (cfbolz, rxe) remove the raymond tool Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 14:07:23 2007 @@ -37,7 +37,7 @@ self.libraries = libraries self.exports = {} - def build_database(self, exports=[], pyobj_options=None): + def build_database(self, pyobj_options=None): translator = self.translator gcpolicyclass = self.get_gcpolicyclass() @@ -75,20 +75,6 @@ pf = self.getentrypointptr() pfname = db.get(pf) self.exports[self.entrypoint.func_name] = pf - for obj in exports: - if type(obj) is tuple: - objname, obj = obj - elif hasattr(obj, '__name__'): - objname = obj.__name__ - else: - objname = None - po = self.getentrypointptr(obj) - poname = db.get(po) - objname = objname or poname - if objname in self.exports: - raise NameError, 'duplicate name in export: %s is %s and %s' % ( - objname, db.get(self.exports[objname]), poname) - self.exports[objname] = po db.complete() # add library dependencies From rxe at codespeak.net Tue Nov 20 15:01:57 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 15:01:57 +0100 (CET) Subject: [pypy-svn] r48836 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c llsupport llsupport/test llvm Message-ID: <20071120140157.03A3881A3@code0.codespeak.net> Author: rxe Date: Tue Nov 20 15:01:56 2007 New Revision: 48836 Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/ (props changed) pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/test/ (props changed) pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/test/__init__.py Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py Log: (cfbolz, rxe) add empty llsupport Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 15:01:56 2007 @@ -167,10 +167,9 @@ standalone = False c_ext_module = None - def getentrypointptr(self, obj=None): - if obj is None: - obj = self.entrypoint - return lltype.pyobjectptr(obj) + def getentrypointptr(self): + bk = self.translator.annotator.bookkeeper + return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) def compile(self): assert self.c_source_filename Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/test/__init__.py ============================================================================== Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py Tue Nov 20 15:01:56 2007 @@ -150,18 +150,19 @@ lltype.UniChar: "ctypes.c_uint", } - def __init__(self, genllvm, dllname): - self.genllvm = genllvm + def __init__(self, entryname, filename, graph, dllname): + self.entryname = entryname self.dllname = dllname - basename = self.genllvm.entry_name + '_wrapper.py' - self.modfilename = genllvm.filename.new(basename=basename) + basename = self.entryname + '_wrapper.py' + self.modfilename = filename.new(basename=basename) self.count = 0 - + self.graph = graph + def create(self): self.file = open(str(self.modfilename), 'w') self.file.write(self.prolog % self.dllname) - g = self.genllvm.entrynode.graph + g = self.graph name = self.genllvm.entry_name ARGS = [a.concretetype for a in g.startblock.inputargs] From rxe at codespeak.net Tue Nov 20 15:03:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 15:03:56 +0100 (CET) Subject: [pypy-svn] r48837 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: llsupport llvm Message-ID: <20071120140356.D240681A3@code0.codespeak.net> Author: rxe Date: Tue Nov 20 15:03:56 2007 New Revision: 48837 Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/__init__.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py - copied unchanged from r48836, pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py Removed: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/modwrapper.py Log: (cfbolz, rxe) move modwrapper Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/__init__.py ============================================================================== From rxe at codespeak.net Tue Nov 20 15:10:56 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 15:10:56 +0100 (CET) Subject: [pypy-svn] r48838 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: llsupport llvm Message-ID: <20071120141056.2950181C4@code0.codespeak.net> Author: rxe Date: Tue Nov 20 15:10:55 2007 New Revision: 48838 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py Log: (cfbolz, rxe) fix llvm tests Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 15:10:55 2007 @@ -163,7 +163,8 @@ self.file.write(self.prolog % self.dllname) g = self.graph - name = self.genllvm.entry_name + + name = self.entryname ARGS = [a.concretetype for a in g.startblock.inputargs] inputargtypes, to_llargs = self.build_args_to_ctypes_to_lltype(ARGS) Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py Tue Nov 20 15:10:55 2007 @@ -5,7 +5,7 @@ from pypy.translator.tool import stdoutcapture from pypy.translator.llvm.log import log -from pypy.translator.llvm.modwrapper import CtypesModule +from pypy.translator.llsupport.modwrapper import CtypesModule def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ @@ -133,7 +133,11 @@ try: self.execute_cmds() - modname = CtypesModule(self.genllvm, "%s.so" % base).create() + gen = self.genllvm + modname = CtypesModule(gen.entry_name, + gen.filename, + gen.entrynode.graph, + "%s.so" % base).create() finally: self.lastdir.chdir() From antocuni at codespeak.net Tue Nov 20 15:21:36 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 15:21:36 +0100 (CET) Subject: [pypy-svn] r48839 - pypy/dist/pypy/module/sys Message-ID: <20071120142136.AFE9F81E2@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 15:21:32 2007 New Revision: 48839 Modified: pypy/dist/pypy/module/sys/app.py Log: (antocuni, fijal) this seems to be rpython nowadays Modified: pypy/dist/pypy/module/sys/app.py ============================================================================== --- pypy/dist/pypy/module/sys/app.py (original) +++ pypy/dist/pypy/module/sys/app.py Tue Nov 20 15:21:32 2007 @@ -1,6 +1,3 @@ -# NOT_RPYTHON -- flowing results in -# AttributeError: << 'FlowObjSpace' object has no attribute 'w_AttributeError' -# XXX investigate! """ The 'sys' module. """ From xoraxax at codespeak.net Tue Nov 20 15:24:53 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 20 Nov 2007 15:24:53 +0100 (CET) Subject: [pypy-svn] r48840 - pypy/dist/pypy/translator/jvm Message-ID: <20071120142453.B4E8D81D4@code0.codespeak.net> Author: xoraxax Date: Tue Nov 20 15:24:53 2007 New Revision: 48840 Modified: pypy/dist/pypy/translator/jvm/genjvm.py Log: Remove a pass statement. Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Tue Nov 20 15:24:53 2007 @@ -49,7 +49,6 @@ print "vvv Stderr vvv\n" print self.stderr - pass class JvmGeneratedSource(object): From rxe at codespeak.net Tue Nov 20 15:27:00 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 15:27:00 +0100 (CET) Subject: [pypy-svn] r48841 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport Message-ID: <20071120142700.DA17681D4@code0.codespeak.net> Author: rxe Date: Tue Nov 20 15:27:00 2007 New Revision: 48841 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe) add gc offset to mod wrapper Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 15:27:00 2007 @@ -53,7 +53,8 @@ _fields_ = [("size", ctypes.c_int), ("data", ctypes.c_byte * len(arg))] class STR(ctypes.Structure): - _fields_ = [("hash", ctypes.c_int), + _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + ("hash", ctypes.c_int), ("chars", Chars)] s = STR() s.hash = 0 @@ -77,7 +78,8 @@ ("data", ctypes.c_char * 1)] class STR(ctypes.Structure): - _fields_ = [("hash", ctypes.c_int), + _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + ("hash", ctypes.c_int), ("array", Chars)] if res: @@ -89,7 +91,8 @@ def struct_to_tuple(res, C_TYPE_actions): if res: class S(ctypes.Structure): - _fields_ = [("item%%s" %% ii, C_TYPE) for ii, (C_TYPE, _) in enumerate(C_TYPE_actions)] + _fields_ = ([("padding", ctypes.c_byte * GC_OFFSET)] + + [("item%%s" %% ii, C_TYPE) for ii, (C_TYPE, _) in enumerate(C_TYPE_actions)]) s = ctypes.cast(res, ctypes.POINTER(S)).contents items = [action(getattr(s, 'item%%s' %% ii)) for ii, (_, action) in enumerate(C_TYPE_actions)] return {'type':'tuple', 'value':tuple(items)} @@ -99,7 +102,8 @@ def list_to_array(res, action): if res: class List(ctypes.Structure): - _fields_ = [("length", ctypes.c_int), + _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + ("length", ctypes.c_int), ("items", ctypes.c_void_p)] list = ctypes.cast(res, ctypes.POINTER(List)).contents size = list.length @@ -112,7 +116,8 @@ if size == -1: size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value class Array(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), + _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + ("size", ctypes.c_int), ("data", C_TYPE * size)] array = ctypes.cast(res, ctypes.POINTER(Array)).contents return [action(array.data[ii]) for ii in range(size)] @@ -120,7 +125,8 @@ return None def to_exception_type(addr): - addr_str = ctypes.cast(addr+12, ctypes.POINTER(ctypes.c_int)).contents.value + # XXX we should define the structure + addr_str = ctypes.cast(addr+GC_OFFSET+12, ctypes.POINTER(ctypes.c_int)).contents.value size = ctypes.cast(addr_str, ctypes.POINTER(ctypes.c_int)).contents.value - 1 name = ctypes.string_at(addr_str+4, size) return name @@ -150,7 +156,7 @@ lltype.UniChar: "ctypes.c_uint", } - def __init__(self, entryname, filename, graph, dllname): + def __init__(self, entryname, filename, graph, dllname, gcoffset=0): self.entryname = entryname self.dllname = dllname basename = self.entryname + '_wrapper.py' @@ -160,6 +166,7 @@ def create(self): self.file = open(str(self.modfilename), 'w') + self.file.write("GC_OFFSET = 0") self.file.write(self.prolog % self.dllname) g = self.graph From fijal at codespeak.net Tue Nov 20 15:31:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 15:31:41 +0100 (CET) Subject: [pypy-svn] r48842 - pypy/dist/pypy/objspace/std Message-ID: <20071120143141.8F7CD81D4@code0.codespeak.net> Author: fijal Date: Tue Nov 20 15:31:39 2007 New Revision: 48842 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: Mute the warning Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Nov 20 15:31:39 2007 @@ -134,6 +134,11 @@ self.dictimplementation = None return None + def next_entry(self): + """ Purely abstract method + """ + raise NotImplementedError + def length(self): if self.dictimplementation is not None: return self.len - self.pos From antocuni at codespeak.net Tue Nov 20 16:15:51 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 16:15:51 +0100 (CET) Subject: [pypy-svn] r48843 - pypy/dist/pypy/module/__builtin__/test Message-ID: <20071120151551.C0F6C818E@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 16:15:50 2007 New Revision: 48843 Modified: pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Log: another test for super Modified: pypy/dist/pypy/module/__builtin__/test/test_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Tue Nov 20 16:15:50 2007 @@ -80,3 +80,10 @@ return [cls, super(B, cls).f()] f = classmethod(f) assert B().f() == [B, B] + + def test_super_fail(self): + try: + super(list, 2) + except TypeError, e: + message = e.args[0] + assert message.startswith('super(type, obj): obj must be an instance or subtype of type') From xoraxax at codespeak.net Tue Nov 20 16:33:03 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 20 Nov 2007 16:33:03 +0100 (CET) Subject: [pypy-svn] r48844 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071120153303.299718197@code0.codespeak.net> Author: xoraxax Date: Tue Nov 20 16:33:02 2007 New Revision: 48844 Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: (armin, alexander) Remove unused imports to fix circular import. Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Tue Nov 20 16:33:02 2007 @@ -14,7 +14,6 @@ from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.annotation import model as annmodel -from pypy.rpython.rbuiltin import gen_cast def uaddressof(obj): @@ -636,6 +635,7 @@ return annmodel.lltype_to_annotation(RESTYPE) def specialize_call(self, hop): + from pypy.rpython.rbuiltin import gen_cast hop.exception_cannot_occur() s_RESTYPE = hop.args_s[0] assert s_RESTYPE.is_constant() Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Tue Nov 20 16:33:02 2007 @@ -1,13 +1,11 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import ll2ctypes from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic from pypy.rlib import rarithmetic -from pypy.rpython.rbuiltin import parse_kwds from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable from pypy.tool.sourcetools import func_with_new_name From rxe at codespeak.net Tue Nov 20 16:41:05 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 16:41:05 +0100 (CET) Subject: [pypy-svn] r48845 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c llsupport llvm Message-ID: <20071120154105.DF14B81A3@code0.codespeak.net> Author: rxe Date: Tue Nov 20 16:41:05 2007 New Revision: 48845 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/funcnode.py Log: (cfbolz, rxe) progress in the direction of using modwrapper in both ll backends Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 16:41:05 2007 @@ -35,7 +35,6 @@ if libraries is None: libraries = [] self.libraries = libraries - self.exports = {} def build_database(self, pyobj_options=None): translator = self.translator @@ -74,7 +73,7 @@ # build entrypoint and eventually other things to expose pf = self.getentrypointptr() pfname = db.get(pf) - self.exports[self.entrypoint.func_name] = pf + self.entrypoint_name = pfname db.complete() # add library dependencies @@ -129,7 +128,6 @@ cfile, extra, include_dirs, library_dirs = \ gen_source(db, modulename, targetdir, defines = defines, - exports = self.exports, libraries = self.libraries) else: if self.config.translation.instrument: @@ -169,7 +167,8 @@ def getentrypointptr(self): bk = self.translator.annotator.bookkeeper - return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) + self.graph_entrypoint = bk.getdesc(self.entrypoint).getuniquegraph() + return getfunctionptr(self.graph_entrypoint) def compile(self): assert self.c_source_filename @@ -180,8 +179,15 @@ include_dirs = [autopath.this_dir] + extra_includes, library_dirs = self.library_dirs, libraries=self.libraries) - self._compiled = True + from pypy.translator.llsupport.modwrapper import CtypesModule + dll_filename = self.c_source_filename.new(ext='so') + modname = CtypesModule(self.entrypoint_name, + self.graph_entrypoint, + dll_filename).create() + + self._compiled = True + def import_module(self): assert self._compiled assert not self.c_ext_module @@ -682,8 +688,7 @@ return filename, sg.getextrafiles(), include_dirs, library_dirs -def gen_source(database, modulename, targetdir, defines={}, exports={}, - libraries=[]): +def gen_source(database, modulename, targetdir, defines={}, libraries=[]): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -730,69 +735,6 @@ sg.write_extra_sources(sources) # - # PyObject support (strange) code - # - pyobjmaker = database.pyobjmaker - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Table of global PyObjects ***/' - print >> f - print >> f, 'static globalobjectdef_t globalobjectdefs[] = {' - for node in database.containerlist: - if isinstance(node, (PyObjectNode, PyObjHeadNode)): - for target in node.where_to_copy_me: - print >> f, '\t{%s, "%s"},' % (target, node.exported_name) - print >> f, '\t{ NULL, NULL }\t/* Sentinel */' - print >> f, '};' - print >> f - print >> f, 'static cpyobjheaddef_t cpyobjheaddefs[] = {' - for node in database.containerlist: - if isinstance(node, PyObjHeadNode): - print >> f, '\t{"%s", %s, %s},' % (node.exported_name, - node.ptrname, - node.get_setupfn_name()) - print >> f, '\t{ NULL, NULL, NULL }\t/* Sentinel */' - print >> f, '};' - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Table of functions ***/' - print >> f - print >> f, 'static globalfunctiondef_t globalfunctiondefs[] = {' - wrappers = pyobjmaker.wrappers.items() - wrappers.sort() - for globalobject_name, (base_name, wrapper_name, func_doc) in wrappers: - print >> f, ('\t{&%s, "%s", {"%s", (PyCFunction)%s, ' - 'METH_VARARGS|METH_KEYWORDS, %s}},' % ( - globalobject_name, - globalobject_name, - base_name, - wrapper_name, - func_doc and c_string_constant(func_doc) or 'NULL')) - print >> f, '\t{ NULL }\t/* Sentinel */' - print >> f, '};' - print >> f, 'static globalfunctiondef_t *globalfunctiondefsptr = &globalfunctiondefs[0];' - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Frozen Python bytecode: the initialization code ***/' - print >> f - print >> f, 'static char *frozen_initcode[] = {"\\' - bytecode, originalsource = database.pyobjmaker.getfrozenbytecode() - g = targetdir.join('frozen.py').open('w') - g.write(originalsource) - g.close() - def char_repr(c): - if c in '\\"': return '\\' + c - if ' ' <= c < '\x7F': return c - return '\\%03o' % ord(c) - for i in range(0, len(bytecode), 32): - print >> f, ''.join([char_repr(c) for c in bytecode[i:i+32]])+'\\' - if (i+32) % 1024 == 0: - print >> f, '", "\\' - print >> f, '"};' - print >> f, "#define FROZEN_INITCODE_SIZE %d" % len(bytecode) - print >> f - - # # Module initialization function # print >> f, '/***********************************************************/' @@ -800,16 +742,14 @@ print >> f gen_startupcode(f, database) print >> f - print >> f, 'MODULE_INITFUNC(%s)' % modulename - print >> f, '{' - print >> f, '\tSETUP_MODULE(%s);' % modulename - for publicname, pyobjptr in exports.items(): - # some fishing needed to find the name of the obj - pyobjnode = database.containernodes[pyobjptr._obj] - print >> f, '\tPyModule_AddObject(m, "%s", %s);' % (publicname, - pyobjnode.name) - print >> f, '\tcall_postsetup(m);' + + print >> f, 'int ctypes_RPython_StartupCode() {' + print >> f, ' char *res = RPython_StartupCode();' + print >> f, ' if (res == NULL) return 1;' + print >> f, ' return 0;' print >> f, '}' + print >> f + f.close() # Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 16:41:05 2007 @@ -14,11 +14,11 @@ _c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) -rpyexc_occured = _c.pypy__rpyexc_occured +rpyexc_occured = _c.pypy_g__RPyExceptionOccurred rpyexc_occured.argtypes = [] rpyexc_occured.restype = ctypes.c_int -rpyexc_fetch_type = _c.pypy_rpyexc_fetch_type +rpyexc_fetch_type = _c.pypy_g_RPyFetchExceptionType rpyexc_fetch_type.argtypes = [] rpyexc_fetch_type.restype = ctypes.c_void_p @@ -133,7 +133,7 @@ """ epilog = """ -__entrypoint__ = _c.pypy_%(name)s +__entrypoint__ = _c.%(name)s # %(RT)r to_llargs = %(to_llargs)s @@ -156,11 +156,11 @@ lltype.UniChar: "ctypes.c_uint", } - def __init__(self, entryname, filename, graph, dllname, gcoffset=0): + def __init__(self, entryname, graph, dllname, gcoffset=0): self.entryname = entryname self.dllname = dllname basename = self.entryname + '_wrapper.py' - self.modfilename = filename.new(basename=basename) + self.modfilename = dllname.new(basename=basename) self.count = 0 self.graph = graph Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py Tue Nov 20 16:41:05 2007 @@ -135,9 +135,8 @@ self.execute_cmds() gen = self.genllvm modname = CtypesModule(gen.entry_name, - gen.filename, gen.entrynode.graph, - "%s.so" % base).create() + py.path.local("%s.so" % base)).create() finally: self.lastdir.chdir() Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/funcnode.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/funcnode.py Tue Nov 20 16:41:05 2007 @@ -11,7 +11,7 @@ pass class FuncImplNode(FuncNode): - prefix = '@pypy_' + prefix = '@pypy_g_' __slots__ = "db value graph block_to_name bad_switch_block".split() def __init__(self, db, value): @@ -20,7 +20,9 @@ self.graph = value.graph self.bad_switch_block = False - self.make_name(value.graph.name) + name = value._name + #name = value.graph.name + self.make_name(name) def setup(self): assert self.graph, "cannot traverse" From cfbolz at codespeak.net Tue Nov 20 16:47:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 16:47:08 +0100 (CET) Subject: [pypy-svn] r48846 - pypy/branch/new-genc-tests-wrapper/pypy/translator/c Message-ID: <20071120154708.75FEF8146@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 16:47:08 2007 New Revision: 48846 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/pyobj.py Log: (cfbolz, rxe): pyc files are evil Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/pyobj.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/pyobj.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/pyobj.py Tue Nov 20 16:47:08 2007 @@ -8,7 +8,6 @@ from pypy.translator.gensupp import builtin_base, builtin_type_base from pypy.translator.c.support import log from pypy.translator.c.wrapper import gen_wrapper, new_method_graph -from pypy.translator.tool.raymond import should_expose from pypy.rlib.rarithmetic import r_int, r_uint from pypy.rpython.lltypesystem.lltype import pyobjectptr, LowLevelType @@ -584,9 +583,6 @@ if isinstance(value, FunctionType): func = value fname = '%s.%s' % (cls.__name__, func.__name__) - if not should_expose(func): - log.REMARK('method %s hidden from wrapper' % fname) - continue if func.__name__ == '__init__': init_seen = True # there is the problem with exposed classes inheriting from From pedronis at codespeak.net Tue Nov 20 16:52:25 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 20 Nov 2007 16:52:25 +0100 (CET) Subject: [pypy-svn] r48847 - pypy/dist/pypy/rpython/module Message-ID: <20071120155225.ECF438146@code0.codespeak.net> Author: pedronis Date: Tue Nov 20 16:52:25 2007 New Revision: 48847 Modified: pypy/dist/pypy/rpython/module/ll_os.py Log: kill unused imports Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Tue Nov 20 16:52:25 2007 @@ -7,7 +7,6 @@ import os, sys, errno from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.rpython.module.support import to_opaque_object, from_opaque_object from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rarithmetic import r_longlong from pypy.rpython.extfunc import BaseLazyRegistering From cfbolz at codespeak.net Tue Nov 20 16:55:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 16:55:00 +0100 (CET) Subject: [pypy-svn] r48848 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c llsupport Message-ID: <20071120155500.AF7E28156@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 16:55:00 2007 New Revision: 48848 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe): some tests are passing Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 16:55:00 2007 @@ -185,14 +185,14 @@ modname = CtypesModule(self.entrypoint_name, self.graph_entrypoint, dll_filename).create() - + self.modname = modname self._compiled = True def import_module(self): assert self._compiled assert not self.c_ext_module mod = import_module_from_directory(self.c_source_filename.dirpath(), - self.c_source_filename.purebasename) + self.modname) self.c_ext_module = mod return mod @@ -200,7 +200,7 @@ assert self._compiled assert not self.c_ext_module self.c_ext_module = isolate.Isolate((str(self.c_source_filename.dirpath()), - self.c_source_filename.purebasename)) + self.modname)) return self.c_ext_module def get_entry_point(self): Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 16:55:00 2007 @@ -142,7 +142,7 @@ # %(ARGS)r ll_to_res = %(ll_to_res)s __entrypoint__.restype = %(returntype)s - """ +""" TO_CTYPES = {lltype.Bool: "ctypes.c_byte", lltype.SingleFloat: "ctypes.c_float", @@ -183,6 +183,7 @@ returntype, ll_to_res = self.build_lltype_to_ctypes_to_res(RT) self.file.write(self.epilog % locals()) + self.file.write("\n%s = entrypoint" % (self.graph.name, )) self.file.close() return self.modfilename.purebasename From xoraxax at codespeak.net Tue Nov 20 17:21:14 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 20 Nov 2007 17:21:14 +0100 (CET) Subject: [pypy-svn] r48849 - in pypy/dist/pypy/rpython: . test Message-ID: <20071120162114.00273814A@code0.codespeak.net> Author: xoraxax Date: Tue Nov 20 17:21:14 2007 New Revision: 48849 Modified: pypy/dist/pypy/rpython/rcontrollerentry.py pypy/dist/pypy/rpython/test/test_controllerentry.py Log: (armin, alexander) Fix rtyping and actually using calls on controller instances. Modified: pypy/dist/pypy/rpython/rcontrollerentry.py ============================================================================== --- pypy/dist/pypy/rpython/rcontrollerentry.py (original) +++ pypy/dist/pypy/rpython/rcontrollerentry.py Tue Nov 20 17:21:14 2007 @@ -31,6 +31,9 @@ def rtype_is_true(self, hop): return self.controller.rtype_is_true(hop) + def rtype_simple_call(self, hop): + return self.controller.rtype_call(hop) + class __extend__(pairtype(ControlledInstanceRepr, Repr)): Modified: pypy/dist/pypy/rpython/test/test_controllerentry.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_controllerentry.py (original) +++ pypy/dist/pypy/rpython/test/test_controllerentry.py Tue Nov 20 17:21:14 2007 @@ -36,6 +36,9 @@ def setitem(self, obj, key, value): value.append(obj + key) + def call(self, obj, arg): + return obj + arg + class Entry(ControllerEntry): _about_ = C _controller_ = C_Controller @@ -96,14 +99,16 @@ lst = [] c = C(a) c['foo'] = lst # side-effect on lst! well, it's a test - return c['bar'], lst[0] + call_res = c("baz") + return c['bar'], lst[0], call_res def test_getsetitem_annotate(): a = RPythonAnnotator() s = a.build_types(fun3, [a.bookkeeper.immutablevalue("4")]) - assert s.const == ("4_bar", "4_foo") + assert s.const == ("4_bar", "4_foo", "4_baz") def test_getsetitem_specialize(): res = interpret(fun3, ["4"]) assert ''.join(res.item0.chars) == "4_bar" assert ''.join(res.item1.chars) == "4_foo" + assert ''.join(res.item2.chars) == "4_baz" From cfbolz at codespeak.net Tue Nov 20 17:38:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 17:38:38 +0100 (CET) Subject: [pypy-svn] r48850 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport Message-ID: <20071120163838.4DA19810C@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 17:38:37 2007 New Revision: 48850 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe): this is only there if we are running with Boehm GC. Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 17:38:37 2007 @@ -22,9 +22,13 @@ rpyexc_fetch_type.argtypes = [] rpyexc_fetch_type.restype = ctypes.c_void_p -GC_get_heap_size_wrapper = _c.GC_get_heap_size -GC_get_heap_size_wrapper.argtypes = [] -GC_get_heap_size_wrapper.restype = ctypes.c_int +try: + GC_get_heap_size_wrapper = _c.GC_get_heap_size + GC_get_heap_size_wrapper.argtypes = [] + GC_get_heap_size_wrapper.restype = ctypes.c_int +except AttributeError: + pass # only if there is Boehm + startup_code = _c.ctypes_RPython_StartupCode startup_code.argtypes = [] From cfbolz at codespeak.net Tue Nov 20 17:39:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 17:39:36 +0100 (CET) Subject: [pypy-svn] r48851 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c/test test Message-ID: <20071120163936.73916810C@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 17:39:36 2007 New Revision: 48851 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_typed.py pypy/branch/new-genc-tests-wrapper/pypy/translator/test/snippet.py Log: (cfbolz, rxe): this snippet created SomeObjects. stop doing that. Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_typed.py Tue Nov 20 17:39:36 2007 @@ -61,7 +61,7 @@ def test_inheritance2(self): inheritance2 = self.getcompiled(snippet.inheritance2) - assert inheritance2() == ((-12, -12), (3, "world")) + assert inheritance2() == ((-12, -12.0), (3, 14.0)) def test_factorial2(self): factorial2 = self.getcompiled(snippet.factorial2, [int]) Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/test/snippet.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/test/snippet.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/test/snippet.py Tue Nov 20 17:39:36 2007 @@ -371,7 +371,7 @@ d = D() d.stuff = (-12, -12) e = E() - e.stuff = (3, "world") + e.stuff = (3, 14.0) return _getstuff(d), _getstuff(e) class F: From ac at codespeak.net Tue Nov 20 17:43:27 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Tue, 20 Nov 2007 17:43:27 +0100 (CET) Subject: [pypy-svn] r48852 - pypy/dist/pypy/module/rctime/test Message-ID: <20071120164327.24CBC817C@code0.codespeak.net> Author: ac Date: Tue Nov 20 17:43:27 2007 New Revision: 48852 Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py Log: Typo, the test only worked in the UK. Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Tue Nov 20 17:43:27 2007 @@ -101,7 +101,7 @@ t = rctime.time() assert long(rctime.mktime(rctime.localtime(t))) == long(t) - assert long(rctime.mktime(rctime.gmtime(t))) + rctime.timezone == long(t) + assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t) ltime = rctime.localtime() assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime) From pedronis at codespeak.net Tue Nov 20 17:57:07 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 20 Nov 2007 17:57:07 +0100 (CET) Subject: [pypy-svn] r48853 - pypy/dist/pypy/jit/codegen/llgraph Message-ID: <20071120165707.2106B818E@code0.codespeak.net> Author: pedronis Date: Tue Nov 20 17:57:06 2007 New Revision: 48853 Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py Log: don't use extfunctable Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/llimpl.py Tue Nov 20 17:57:06 2007 @@ -8,28 +8,34 @@ from pypy.objspace.flow import model as flowmodel from pypy.translator.simplify import eliminate_empty_blocks from pypy.translator.unsimplify import varoftype -from pypy.rpython.module.support import init_opaque_object -from pypy.rpython.module.support import to_opaque_object, from_opaque_object from pypy.rpython.module.support import LLSupport from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem.rclass import fishllattr from pypy.rpython.lltypesystem.lloperation import llop +def _from_opaque(opq): + return opq._obj.externalobj + +_TO_OPAQUE = {} + +def _to_opaque(value): + return lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', + externalobj=value) # for debugging, sanity checks in non-RPython code -reveal = from_opaque_object +reveal = _from_opaque def isptrtype(gv_type): - c = from_opaque_object(gv_type) + c = _from_opaque(gv_type) return isinstance(c.value, lltype.Ptr) def newblock(): block = flowmodel.Block([]) - return to_opaque_object(block) + return _to_opaque(block) def newgraph(gv_FUNCTYPE, name): - FUNCTYPE = from_opaque_object(gv_FUNCTYPE).value + FUNCTYPE = _from_opaque(gv_FUNCTYPE).value # 'name' is just a way to track things if not isinstance(name, str): name = LLSupport.from_rstr(name) @@ -65,7 +71,7 @@ return genconst(fptr) def _getgraph(gv_func): - graph = from_opaque_object(gv_func).value._obj.graph + graph = _from_opaque(gv_func).value._obj.graph return graph def end(gv_func): @@ -76,22 +82,22 @@ graph = _getgraph(gv_func) [link] = graph.startblock.exits substartblock = link.target - return to_opaque_object(substartblock) + return _to_opaque(substartblock) def geninputarg(block, gv_CONCRETE_TYPE): - block = from_opaque_object(block) + block = _from_opaque(block) assert not block.operations, "block already contains operations" assert block.exits == [], "block already closed" - CONCRETE_TYPE = from_opaque_object(gv_CONCRETE_TYPE).value + CONCRETE_TYPE = _from_opaque(gv_CONCRETE_TYPE).value v = flowmodel.Variable() v.concretetype = lltype.erasedType(CONCRETE_TYPE) block.inputargs.append(v) - return to_opaque_object(v) + return _to_opaque(v) def getinputarg(block, i): - block = from_opaque_object(block) + block = _from_opaque(block) v = block.inputargs[i] - return to_opaque_object(v) + return _to_opaque(v) def _inputvars(vars): newvars = [] @@ -114,14 +120,14 @@ newvars.append(v) res = [] for v1 in newvars: - v = from_opaque_object(v1) + v = _from_opaque(v1) assert isinstance(v, (flowmodel.Constant, flowmodel.Variable)) res.append(v) return res def cast(block, gv_TYPE, gv_var): - TYPE = from_opaque_object(gv_TYPE).value - v = from_opaque_object(gv_var) + TYPE = _from_opaque(gv_TYPE).value + v = _from_opaque(gv_var) if TYPE != v.concretetype: if TYPE is llmemory.GCREF or v.concretetype is llmemory.GCREF: lltype.cast_opaque_ptr(TYPE, v.concretetype._defl()) # sanity check @@ -129,13 +135,13 @@ else: assert v.concretetype == lltype.erasedType(TYPE) opname = 'cast_pointer' - block = from_opaque_object(block) + block = _from_opaque(block) v2 = flowmodel.Variable() v2.concretetype = TYPE op = flowmodel.SpaceOperation(opname, [v], v2) block.operations.append(op) v = v2 - return to_opaque_object(v) + return _to_opaque(v) def erasedvar(v, block): T = lltype.erasedType(v.concretetype) @@ -152,7 +158,7 @@ # gv_RESULT_TYPE comes from constTYPE if not isinstance(opname, str): opname = LLSupport.from_rstr(opname) - block = from_opaque_object(block) + block = _from_opaque(block) assert block.exits == [], "block already closed" opvars = _inputvars(vars_gv) if gv_RESULT_TYPE is guess: @@ -160,12 +166,12 @@ elif isinstance(gv_RESULT_TYPE, lltype.LowLevelType): RESULT_TYPE = gv_RESULT_TYPE else: - RESULT_TYPE = from_opaque_object(gv_RESULT_TYPE).value + RESULT_TYPE = _from_opaque(gv_RESULT_TYPE).value v = flowmodel.Variable() v.concretetype = RESULT_TYPE op = flowmodel.SpaceOperation(opname, opvars, v) block.operations.append(op) - return to_opaque_object(erasedvar(v, block)) + return _to_opaque(erasedvar(v, block)) RESULT_TYPES = { 'cast_ptr_to_int': lltype.Signed, @@ -206,14 +212,14 @@ v.concretetype = T1 if v.concretetype == lltype.Void: # XXX genconst should not really be used for Void constants assert not isinstance(llvalue, str) and not isinstance(llvalue, lltype.LowLevelType) - return to_opaque_object(v) + return _to_opaque(v) def genzeroconst(gv_TYPE): - TYPE = from_opaque_object(gv_TYPE).value + TYPE = _from_opaque(gv_TYPE).value TYPE = lltype.erasedType(TYPE) c = flowmodel.Constant(TYPE._defl()) c.concretetype = TYPE - return to_opaque_object(c) + return _to_opaque(c) def _generalcast(T, value): if isinstance(T, lltype.Ptr): @@ -231,16 +237,16 @@ return lltype.cast_primitive(T, value) def revealconst(T, gv_value): - c = from_opaque_object(gv_value) + c = _from_opaque(gv_value) assert isinstance(c, flowmodel.Constant) return _generalcast(T, c.value) def revealconstrepr(gv_value): - c = from_opaque_object(gv_value) + c = _from_opaque(gv_value) return LLSupport.to_rstr(repr(c.value)) def isconst(gv_value): - c = from_opaque_object(gv_value) + c = _from_opaque(gv_value) return isinstance(c, flowmodel.Constant) @@ -252,48 +258,48 @@ self.base_and_offsets_gv = base_and_offsets_gv def gengetsubstruct(block, gv_ptr, gv_PTRTYPE, gv_fieldname): - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) # don't generate any operation for an interior getsubstruct, # but just return a special pseudo-variable if isinstance(v_ptr, InteriorPtrVariable): # a nested getsubstruct v = InteriorPtrVariable(v_ptr.base_and_offsets_gv + [gv_fieldname]) - return to_opaque_object(v) + return _to_opaque(v) # in all other cases we need a proper cast gv_ptr = cast(block, gv_PTRTYPE, gv_ptr) - PTRTYPE = from_opaque_object(gv_PTRTYPE).value + PTRTYPE = _from_opaque(gv_PTRTYPE).value if PTRTYPE.TO._gckind == 'gc': # reading from a GcStruct requires returning an interior pointer # pseudo-variable v = InteriorPtrVariable([gv_ptr, gv_fieldname]) - return to_opaque_object(v) + return _to_opaque(v) else: vars_gv = [gv_ptr, gv_fieldname] - c_fieldname = from_opaque_object(gv_fieldname) + c_fieldname = _from_opaque(gv_fieldname) RESULTTYPE = lltype.Ptr(getattr(PTRTYPE.TO, c_fieldname.value)) return genop(block, "getsubstruct", vars_gv, RESULTTYPE) def gengetarraysubstruct(block, gv_ptr, gv_index): - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) # don't generate any operation for an interior getarraysubstruct, # but just return a special pseudo-variable if isinstance(v_ptr, InteriorPtrVariable): # a nested getarraysubstruct v = InteriorPtrVariable(v_ptr.base_and_offsets_gv + [gv_index]) - return to_opaque_object(v) + return _to_opaque(v) PTRTYPE = v_ptr.concretetype if PTRTYPE.TO._gckind == 'gc': # reading from a GcArray requires returning an interior pointer # pseudo-variable v = InteriorPtrVariable([gv_ptr, gv_index]) - return to_opaque_object(v) + return _to_opaque(v) else: vars_gv = [gv_ptr, gv_index] RESULTTYPE = lltype.Ptr(PTRTYPE.TO.OF) return genop(block, "getarraysubstruct", vars_gv, RESULTTYPE) def gensetfield(block, gv_ptr, gv_PTRTYPE, gv_fieldname, gv_value): - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) if isinstance(v_ptr, InteriorPtrVariable): # this is really a setinteriorfield vars_gv = v_ptr.base_and_offsets_gv + [gv_fieldname, gv_value] @@ -306,10 +312,10 @@ genop(block, "setfield", vars_gv, lltype.Void) def gengetfield(block, gv_ptr, gv_PTRTYPE, gv_fieldname): - PTRTYPE = from_opaque_object(gv_PTRTYPE).value - c_fieldname = from_opaque_object(gv_fieldname) + PTRTYPE = _from_opaque(gv_PTRTYPE).value + c_fieldname = _from_opaque(gv_fieldname) RESULTTYPE = getattr(PTRTYPE.TO, c_fieldname.value) - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) if isinstance(v_ptr, InteriorPtrVariable): # this is really a getinteriorfield vars_gv = v_ptr.base_and_offsets_gv + [gv_fieldname] @@ -322,7 +328,7 @@ return genop(block, "getfield", vars_gv, RESULTTYPE) def gensetarrayitem(block, gv_ptr, gv_index, gv_value): - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) if isinstance(v_ptr, InteriorPtrVariable): # this is really a setinteriorfield vars_gv = v_ptr.base_and_offsets_gv + [gv_index, gv_value] @@ -332,8 +338,8 @@ genop(block, "setarrayitem", vars_gv, lltype.Void) def gengetarrayitem(block, gv_ITEMTYPE, gv_ptr, gv_index): - ITEMTYPE = from_opaque_object(gv_ITEMTYPE).value - v_ptr = from_opaque_object(gv_ptr) + ITEMTYPE = _from_opaque(gv_ITEMTYPE).value + v_ptr = _from_opaque(gv_ptr) if isinstance(v_ptr, InteriorPtrVariable): # this is really a getinteriorfield vars_gv = v_ptr.base_and_offsets_gv + [gv_index] @@ -343,7 +349,7 @@ return genop(block, "getarrayitem", vars_gv, ITEMTYPE) def gengetarraysize(block, gv_ptr): - v_ptr = from_opaque_object(gv_ptr) + v_ptr = _from_opaque(gv_ptr) if isinstance(v_ptr, InteriorPtrVariable): # this is really a getinteriorarraysize vars_gv = v_ptr.base_and_offsets_gv @@ -360,29 +366,29 @@ def placeholder(dummy): c = flowmodel.Constant(dummy) c.concretetype = lltype.Void - return to_opaque_object(c) + return _to_opaque(c) def constFieldName(name): assert isinstance(name, str) c = flowmodel.Constant(name) c.concretetype = lltype.Void - return to_opaque_object(c) + return _to_opaque(c) def constTYPE(TYPE): assert isinstance(TYPE, lltype.LowLevelType) c = flowmodel.Constant(TYPE) c.concretetype = lltype.Void - return to_opaque_object(c) + return _to_opaque(c) def closeblock1(block): - block = from_opaque_object(block) + block = _from_opaque(block) link = flowmodel.Link([], None) block.closeblock(link) - return to_opaque_object(link) + return _to_opaque(link) def closeblock2(block, exitswitch): - block = from_opaque_object(block) - exitswitch = from_opaque_object(exitswitch) + block = _from_opaque(block) + exitswitch = _from_opaque(exitswitch) assert isinstance(exitswitch, flowmodel.Variable) block.exitswitch = exitswitch false_link = flowmodel.Link([], None) @@ -392,8 +398,8 @@ true_link.exitcase = True true_link.llexitcase = True block.closeblock(false_link, true_link) - return pseudotuple(to_opaque_object(false_link), - to_opaque_object(true_link)) + return pseudotuple(_to_opaque(false_link), + _to_opaque(true_link)) _color_num = 1 _color_den = 2 @@ -409,9 +415,9 @@ return '#'+''.join(['%02x' % int(p*255) for p in rgb]) def closeblockswitch(block, exitswitch): - block = from_opaque_object(block) + block = _from_opaque(block) block.blockcolor = getcolor() - exitswitch = from_opaque_object(exitswitch) + exitswitch = _from_opaque(exitswitch) assert isinstance(exitswitch, flowmodel.Variable) TYPE = exitswitch.concretetype if isinstance(TYPE, lltype.Ptr): @@ -425,8 +431,8 @@ return def add_case(block, exitcase): - block = from_opaque_object(block) - exitcase = from_opaque_object(exitcase) + block = _from_opaque(block) + exitcase = _from_opaque(exitcase) assert isinstance(exitcase, flowmodel.Constant) assert isinstance(block.exitswitch, flowmodel.Variable) case_link = flowmodel.Link([], None) @@ -441,10 +447,10 @@ else: exits = block.exits + (case_link,) block.recloseblock(*exits) - return to_opaque_object(case_link) + return _to_opaque(case_link) def add_default(block): - block = from_opaque_object(block) + block = _from_opaque(block) assert isinstance(block.exitswitch, flowmodel.Variable) default_link = flowmodel.Link([], None) default_link.exitcase = 'default' @@ -454,7 +460,7 @@ else: exits = block.exits + (default_link,) block.recloseblock(*exits) - return to_opaque_object(default_link) + return _to_opaque(default_link) class pseudotuple(object): # something that looks both like a hl and a ll tuple @@ -488,19 +494,19 @@ raise TypeError def closelink(link, vars, targetblock): - link = from_opaque_object(link) - targetblock = from_opaque_object(targetblock) + link = _from_opaque(link) + targetblock = _from_opaque(targetblock) vars = _inputvars(vars) _closelink(link, vars, targetblock) def closereturnlink(link, returnvar, gv_func): - returnvar = from_opaque_object(returnvar) - link = from_opaque_object(link) + returnvar = _from_opaque(returnvar) + link = _from_opaque(link) graph = _getgraph(gv_func) _closelink(link, [returnvar], graph.prereturnblock) def closelinktofreshblock(link, inputargs=None, otherlink=None): - link = from_opaque_object(link) + link = _from_opaque(link) prevblockvars = link.prevblock.getvariables() # the next block's inputargs come from 'inputargs' if specified if inputargs is None: @@ -513,7 +519,7 @@ if otherlink is None: linkvars = list(inputvars) else: - otherlink = from_opaque_object(otherlink) + otherlink = _from_opaque(otherlink) linkvars = list(otherlink.args) # check linkvars for consistency existing_vars = dict.fromkeys(prevblockvars) @@ -525,7 +531,7 @@ nextblock = flowmodel.Block(inputvars) link.args = linkvars link.target = nextblock - return to_opaque_object(nextblock) + return _to_opaque(nextblock) def casting_link(source, sourcevars, target): assert len(sourcevars) == len(target.inputargs) @@ -579,7 +585,7 @@ return graph def buildgraph(graph, FUNCTYPE): - graph = from_opaque_object(graph) + graph = _from_opaque(graph) return _buildgraph(graph) def testgengraph(gengraph, args, viewbefore=False, executor=LLInterpreter): @@ -603,23 +609,18 @@ graph.show() # ____________________________________________________________ -# RTyping of the above functions - -from pypy.rpython.extfunctable import declareptrtype -blocktypeinfo = declareptrtype(flowmodel.Block, "Block") -consttypeinfo = declareptrtype(flowmodel.Constant, "ConstOrVar") -vartypeinfo = declareptrtype(flowmodel.Variable, "ConstOrVar") -vartypeinfo.set_lltype(consttypeinfo.get_lltype()) # force same lltype -interiorptrvartypeinfo = declareptrtype(InteriorPtrVariable, "ConstOrVar") -interiorptrvartypeinfo.set_lltype(vartypeinfo.get_lltype()) # force same lltype -linktypeinfo = declareptrtype(flowmodel.Link, "Link") -graphtypeinfo = declareptrtype(flowmodel.FunctionGraph, "FunctionGraph") - -CONSTORVAR = lltype.Ptr(consttypeinfo.get_lltype()) -BLOCK = lltype.Ptr(blocktypeinfo.get_lltype()) -LINK = lltype.Ptr(linktypeinfo.get_lltype()) -GRAPH = lltype.Ptr(graphtypeinfo.get_lltype()) +CONSTORVAR = lltype.Ptr(lltype.OpaqueType("ConstOrVar")) +BLOCK = lltype.Ptr(lltype.OpaqueType("Block")) +LINK = lltype.Ptr(lltype.OpaqueType("Link")) +GRAPH = lltype.Ptr(lltype.OpaqueType("FunctionGraph")) + +_TO_OPAQUE[flowmodel.Block] = BLOCK.TO +_TO_OPAQUE[flowmodel.Constant] = CONSTORVAR.TO +_TO_OPAQUE[flowmodel.Variable] = CONSTORVAR.TO +_TO_OPAQUE[InteriorPtrVariable] = CONSTORVAR.TO +_TO_OPAQUE[flowmodel.Link] = LINK.TO +_TO_OPAQUE[flowmodel.FunctionGraph] = GRAPH.TO # support constants and types @@ -664,8 +665,8 @@ # annotations from pypy.annotation import model as annmodel -s_ConstOrVar = annmodel.SomePtr(CONSTORVAR)#annmodel.SomeExternalObject(flowmodel.Variable) -s_Link = annmodel.SomePtr(LINK)#annmodel.SomeExternalObject(flowmodel.Link) +s_ConstOrVar = annmodel.SomePtr(CONSTORVAR) +s_Link = annmodel.SomePtr(LINK) s_LinkPair = annmodel.SomeTuple([s_Link, s_Link]) s_Block = annmodel.SomePtr(BLOCK) s_Graph = annmodel.SomePtr(GRAPH) @@ -714,7 +715,7 @@ def get_frame_info(block, vars_gv): genop(block, 'frame_info', vars_gv, lltype.Void) - block = from_opaque_object(block) + block = _from_opaque(block) frame_info = block.operations[-1] return lltype.opaqueptr(llmemory.GCREF.TO, 'frame_info', info=frame_info) From cfbolz at codespeak.net Tue Nov 20 18:07:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 18:07:52 +0100 (CET) Subject: [pypy-svn] r48854 - pypy/branch/new-genc-tests-wrapper/pypy/translator/c Message-ID: <20071120170752.7EA0181A3@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 18:07:52 2007 New Revision: 48854 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/c/node.py Log: (cfbolz, rxe): some small simplification in the GC policy Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py Tue Nov 20 18:07:52 2007 @@ -15,20 +15,20 @@ self.db = db self.thread_enabled = thread_enabled - def common_gcheader_definition(self, defnode): + def common_gcheader_definition(self): return [] def common_gcheader_initdata(self, defnode): return [] - def struct_gcheader_definition(self, defnode): - return self.common_gcheader_definition(defnode) + def struct_gcheader_definition(self): + return self.common_gcheader_definition() def struct_gcheader_initdata(self, defnode): return self.common_gcheader_initdata(defnode) - def array_gcheader_definition(self, defnode): - return self.common_gcheader_definition(defnode) + def array_gcheader_definition(self): + return self.common_gcheader_definition() def array_gcheader_initdata(self, defnode): return self.common_gcheader_initdata(defnode) @@ -79,9 +79,9 @@ class RefcountingGcPolicy(BasicGcPolicy): transformerclass = refcounting.RefcountingGCTransformer - def common_gcheader_definition(self, defnode): - if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR + def common_gcheader_definition(self): + if self.db.gctransformer is not None: + HDR = self.db.gctransformer.HDR return [(name, HDR._flds[name]) for name in HDR._names] else: return [] @@ -312,7 +312,7 @@ args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) - def common_gcheader_definition(self, defnode): + def common_gcheader_definition(self): return defnode.db.gctransformer.gc_fields() def common_gcheader_initdata(self, defnode): Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/node.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/node.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/node.py Tue Nov 20 18:07:52 2007 @@ -78,7 +78,7 @@ STRUCT = self.STRUCT varlength = self.varlength if needs_gcheader(self.STRUCT): - for fname, T in db.gcpolicy.struct_gcheader_definition(self): + for fname, T in db.gcpolicy.struct_gcheader_definition(): self.fields.append((fname, db.gettype(T, who_asks=self))) for name in STRUCT._names: T = self.c_struct_field_type(name) @@ -223,7 +223,7 @@ ARRAY = self.ARRAY self.gcinfo # force it to be computed if needs_gcheader(ARRAY): - for fname, T in db.gcpolicy.array_gcheader_definition(self): + for fname, T in db.gcpolicy.array_gcheader_definition(): self.gcfields.append((fname, db.gettype(T, who_asks=self))) self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) From cfbolz at codespeak.net Tue Nov 20 18:34:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 18:34:47 +0100 (CET) Subject: [pypy-svn] r48855 - pypy/branch/new-genc-tests-wrapper/pypy/translator/c Message-ID: <20071120173447.95AD6818A@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 18:34:46 2007 New Revision: 48855 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Log: (cfbolz, rxe): make tests only fail and not segfault. progress Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 18:34:46 2007 @@ -108,6 +108,7 @@ if db is None: db = self.build_database() + self.db = db pf = self.getentrypointptr() pfname = db.get(pf) @@ -182,9 +183,11 @@ from pypy.translator.llsupport.modwrapper import CtypesModule dll_filename = self.c_source_filename.new(ext='so') + gcfields = self.db.gcpolicy.common_gcheader_definition() modname = CtypesModule(self.entrypoint_name, self.graph_entrypoint, - dll_filename).create() + dll_filename, + gcfields).create() self.modname = modname self._compiled = True From cfbolz at codespeak.net Tue Nov 20 18:35:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 18:35:03 +0100 (CET) Subject: [pypy-svn] r48856 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport Message-ID: <20071120173503.42F218191@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 18:35:03 2007 New Revision: 48856 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe): this really goes with the last checkin Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 18:35:03 2007 @@ -29,6 +29,7 @@ except AttributeError: pass # only if there is Boehm +GC_OFFSET = getattr(_c, "pypy_g__size_of_gc_header", 0) startup_code = _c.ctypes_RPython_StartupCode startup_code.argtypes = [] @@ -57,7 +58,7 @@ _fields_ = [("size", ctypes.c_int), ("data", ctypes.c_byte * len(arg))] class STR(ctypes.Structure): - _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + _fields_ = get_gc_header() + [ ("hash", ctypes.c_int), ("chars", Chars)] s = STR() @@ -82,7 +83,7 @@ ("data", ctypes.c_char * 1)] class STR(ctypes.Structure): - _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + _fields_ = get_gc_header() + [ ("hash", ctypes.c_int), ("array", Chars)] @@ -95,7 +96,7 @@ def struct_to_tuple(res, C_TYPE_actions): if res: class S(ctypes.Structure): - _fields_ = ([("padding", ctypes.c_byte * GC_OFFSET)] + + _fields_ = (get_gc_header() + [("item%%s" %% ii, C_TYPE) for ii, (C_TYPE, _) in enumerate(C_TYPE_actions)]) s = ctypes.cast(res, ctypes.POINTER(S)).contents items = [action(getattr(s, 'item%%s' %% ii)) for ii, (_, action) in enumerate(C_TYPE_actions)] @@ -106,7 +107,7 @@ def list_to_array(res, action): if res: class List(ctypes.Structure): - _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + _fields_ = get_gc_header() + [ ("length", ctypes.c_int), ("items", ctypes.c_void_p)] list = ctypes.cast(res, ctypes.POINTER(List)).contents @@ -120,7 +121,7 @@ if size == -1: size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value class Array(ctypes.Structure): - _fields_ = [("padding", ctypes.c_byte * GC_OFFSET), + _fields_ = get_gc_header() + [ ("size", ctypes.c_int), ("data", C_TYPE * size)] array = ctypes.cast(res, ctypes.POINTER(Array)).contents @@ -130,6 +131,7 @@ def to_exception_type(addr): # XXX we should define the structure + XXX addr_str = ctypes.cast(addr+GC_OFFSET+12, ctypes.POINTER(ctypes.c_int)).contents.value size = ctypes.cast(addr_str, ctypes.POINTER(ctypes.c_int)).contents.value - 1 name = ctypes.string_at(addr_str+4, size) @@ -160,17 +162,19 @@ lltype.UniChar: "ctypes.c_uint", } - def __init__(self, entryname, graph, dllname, gcoffset=0): + def __init__(self, entryname, graph, dllname, gcfields=None): self.entryname = entryname self.dllname = dllname basename = self.entryname + '_wrapper.py' self.modfilename = dllname.new(basename=basename) self.count = 0 self.graph = graph + if gcfields is None: + gcfields = [] + self.gcfields = gcfields def create(self): self.file = open(str(self.modfilename), 'w') - self.file.write("GC_OFFSET = 0") self.file.write(self.prolog % self.dllname) g = self.graph @@ -185,12 +189,20 @@ RT = g.returnblock.inputargs[0].concretetype returntype, ll_to_res = self.build_lltype_to_ctypes_to_res(RT) + + self.create_gc_header_function() self.file.write(self.epilog % locals()) self.file.write("\n%s = entrypoint" % (self.graph.name, )) self.file.close() return self.modfilename.purebasename + def create_gc_header_function(self): + self.file.write("\ndef get_gc_header():\n") + fields = ['(%r, %s)' % (fieldname, self.to_ctype(T)) + for (fieldname, T) in self.gcfields] + self.file.write(" return [%s]" % ", ".join(fields)) + def create_simple_closure(self, args, code): name = 'tmpfunction_%s' % self.count self.count += 1 From rxe at codespeak.net Tue Nov 20 18:46:24 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 18:46:24 +0100 (CET) Subject: [pypy-svn] r48857 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm Message-ID: <20071120174624.6E26A8110@code0.codespeak.net> Author: rxe Date: Tue Nov 20 18:46:24 2007 New Revision: 48857 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py Log: (cfbolz, rxe) make llvm tests pass again Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/buildllvm.py Tue Nov 20 18:46:24 2007 @@ -134,7 +134,7 @@ try: self.execute_cmds() gen = self.genllvm - modname = CtypesModule(gen.entry_name, + modname = CtypesModule(gen.entrynode.ref[1:], gen.entrynode.graph, py.path.local("%s.so" % base)).create() From fijal at codespeak.net Tue Nov 20 18:47:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 18:47:30 +0100 (CET) Subject: [pypy-svn] r48858 - in pypy/dist/pypy: module/__builtin__ module/__builtin__/test objspace/std Message-ID: <20071120174730.7D13B813D@code0.codespeak.net> Author: fijal Date: Tue Nov 20 18:47:29 2007 New Revision: 48858 Added: pypy/dist/pypy/module/__builtin__/descriptor.py (contents, props changed) Modified: pypy/dist/pypy/module/__builtin__/__init__.py pypy/dist/pypy/module/__builtin__/app_descriptor.py pypy/dist/pypy/module/__builtin__/test/test_descriptor.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/typeobject.py Log: (anto, fijal) Move super() from app-level to interp-level Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Tue Nov 20 18:47:29 2007 @@ -58,7 +58,6 @@ 'property' : 'app_descriptor.property', 'staticmethod' : 'app_descriptor.staticmethod', 'classmethod' : 'app_descriptor.classmethod', - 'super' : 'app_descriptor.super', 'complex' : 'app_complex.complex', @@ -127,6 +126,7 @@ 'xrange' : 'functional.W_XRange', 'all' : 'functional.all', 'any' : 'functional.any', + 'super' : 'descriptor.W_Super', } def pick_builtin(self, w_globals): Modified: pypy/dist/pypy/module/__builtin__/app_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/app_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/app_descriptor.py Tue Nov 20 18:47:29 2007 @@ -142,62 +142,3 @@ self.fdel(obj) docstring.capture(property, 'slot__doc__') - - -# super is a modified version from Guido's tutorial -# http://www.python.org/2.2.3/descrintro.html -# it exposes the same special attributes as CPython's. -class super(object): - """super(type) -> unbound super object -super(type, obj) -> bound super object; requires isinstance(obj, type) -super(type, type2) -> bound super object; requires issubclass(type2, type) - -Typical use to call a cooperative superclass method: - -class C(B): - def meth(self, arg): - super(C, self).meth(arg)""" - __slots__ = ['__thisclass__', '__self__', '__self_class__'] - def __init__(self, typ, obj=None): - if obj is None: - objcls = None # unbound super object - elif _issubtype(type(obj), type) and _issubtype(obj, typ): - objcls = obj # special case for class methods - elif _issubtype(type(obj), typ): - objcls = type(obj) # normal case - else: - objcls = getattr(obj, '__class__', type(obj)) - if not _issubtype(objcls, typ): - raise TypeError, ("super(type, obj): " - "obj must be an instance or subtype of type") - self.__thisclass__ = typ - self.__self__ = obj - self.__self_class__ = objcls - def __get__(self, obj, type=None): - if obj is None or super.__self__.__get__(self) is not None: - return self - else: - return self.__class__(super.__thisclass__.__get__(self), obj) - def __getattribute__(self, attr): - _self_class_ = super.__self_class__.__get__(self) - if (attr != '__class__' # we want super().__class__ to be the real class - and _self_class_ is not None): # no magic for unbound type objects - _thisclass_ = super.__thisclass__.__get__(self) - mro = iter(_self_class_.__mro__) - for cls in mro: - if cls is _thisclass_: - break - # Note: mro is an iterator, so the second loop - # picks up where the first one left off! - for cls in mro: - try: - x = cls.__dict__[attr] - except KeyError: - continue - if hasattr(x, '__get__'): - _self_ = super.__self__.__get__(self) - if _self_ is _self_class_: - _self_ = None # performs an unbound __get__ - x = x.__get__(_self_, _self_class_) - return x - return object.__getattribute__(self, attr) # fall-back Added: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Nov 20 18:47:29 2007 @@ -0,0 +1,91 @@ + +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable, \ + Arguments +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError +# XXX +from pypy.interpreter.callmethod import object_getattribute + +class W_Super(Wrappable): + def __init__(self, space, w_selftype, w_starttype, w_type, w_self): + self.w_selftype = w_selftype + self.w_starttype = w_starttype + self.w_type = w_type + self.w_self = w_self + + def get(self, space, w_obj, w_type=None): + w = space.wrap + if self.w_self is None or space.is_w(w_obj, space.w_None): + return w(self) + else: + return space.call(self.w_selftype, space.newlist([ + self.w_starttype, w_obj])) + get.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def getattribute(self, space, name): + w = space.wrap + if name == '__class__': + return self.w_selftype + if self.w_type is None: + return space.call(object_getattribute(space), + space.newlist([w(self), w(name)])) + + w_value = space.lookup_in_type_starting_at(self.w_type, + self.w_starttype, + name) + if w_value is None: + return space.getattr(w(self), w(name)) + + try: + w_get = space.getattr(w_value, space.wrap('__get__')) + if space.is_w(self.w_self, self.w_type): + w_self = None + else: + w_self = self.w_self + except OperationError, o: + if not o.match(space, space.w_AttributeError): + raise + return w_value + return space.call(w_get, space.newlist([w_self, self.w_type])) + getattribute.unwrap_spec = ['self', ObjSpace, str] + +def descr_new_super(space, w_self, w_starttype, w_obj_or_type=None): + if space.is_w(w_obj_or_type, space.w_None): + w_type = None # unbound super object + else: + w_objtype = space.type(w_obj_or_type) + if space.is_true(space.issubtype(w_objtype, space.w_type)) and \ + space.is_true(space.issubtype(w_obj_or_type, w_starttype)): + w_type = w_obj_or_type # special case for class methods + elif space.is_true(space.issubtype(w_objtype, w_starttype)): + w_type = w_objtype # normal case + else: + try: + w_type = space.getattr(w_obj_or_type, space.wrap('__class__')) + except OperationError, o: + if not o.match(space, space.w_AttributeError): + raise + w_type = w_objtype + if not space.is_true(space.issubtype(w_type, w_starttype)): + raise OperationError(space.w_TypeError, + space.wrap("super(type, obj): " + "obj must be an instance or subtype of type")) + return space.wrap(W_Super(space, w_self, w_starttype, w_type, w_obj_or_type)) +descr_new_super.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] + +W_Super.typedef = TypeDef( + 'super', + __new__ = interp2app(descr_new_super), + __getattribute__ = interp2app(W_Super.getattribute), + __get__ = interp2app(W_Super.get), + __doc__ = """super(type) -> unbound super object +super(type, obj) -> bound super object; requires isinstance(obj, type) +super(type, type2) -> bound super object; requires issubclass(type2, type) + +Typical use to call a cooperative superclass method: + +class C(B): + def meth(self, arg): + super(C, self).meth(arg)""" +) Modified: pypy/dist/pypy/module/__builtin__/test/test_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Tue Nov 20 18:47:29 2007 @@ -87,3 +87,109 @@ except TypeError, e: message = e.args[0] assert message.startswith('super(type, obj): obj must be an instance or subtype of type') + + def test_super_various(self): + + class A(object): + def meth(self, a): + return "A(%r)" % a + + class B(A): + def __init__(self): + self.__super = super(B, self) + def meth(self, a): + return "B(%r)" % a + self.__super.meth(a) + + assert B().meth(2) == "B(2)A(2)" + + class C(A): + def meth(self, a): + return "C(%r)" % a + self.__super.meth(a) + C._C__super = super(C) + + assert C().meth(3) == "C(3)A(3)" + + class D(C, B): + def meth(self, a): + return "D(%r)" % a + super(D, self).meth(a) + + assert D().meth(4) == "D(4)C(4)B(4)A(4)" + + # subclasses + class mysuper(super): + def __init__(self, *args): + return super(mysuper, self).__init__(*args) + + class E(D): + def meth(self, a): + return "E(%r)" % a + mysuper(E, self).meth(a) + + assert E().meth(5) == "E(5)D(5)C(5)B(5)A(5)" + + class F(E): + def meth(self, a): + s = self.__super # == mysuper(F, self) + return "F(%r)[%s]" % (a, s.__class__.__name__) + s.meth(a) + F._F__super = mysuper(F) + + assert F().meth(6) == "F(6)[mysuper]E(6)D(6)C(6)B(6)A(6)" + + + def test_super_lookup(self): + class DDbase(object): + def getx(self): + return 42 + x = property(getx) + + class DDsub(DDbase): + def getx(self): + return "hello" + x = property(getx) + + dd = DDsub() + assert dd.x == "hello" + assert super(DDsub, dd).x == 42 + + def test_super_lookup2(self): + + class Base(object): + aProp = property(lambda self: "foo") + + class Sub(Base): + def test(klass): + return super(Sub,klass).aProp + test = classmethod(test) + + assert Sub.test() is Base.aProp + + def test_proxy_super(self): + class Proxy(object): + def __init__(self, obj): + self.__obj = obj + def __getattribute__(self, name): + if name.startswith("_Proxy__"): + return object.__getattribute__(self, name) + else: + return getattr(self.__obj, name) + + class B(object): + def f(self): + return "B.f" + + class C(B): + def f(self): + return super(C, self).f() + "->C.f" + + obj = C() + p = Proxy(obj) + assert C.__dict__["f"](p) == "B.f->C.f" + + def test_super_errors(self): + class C: + pass + class D(C): + pass + raises(TypeError, "super(D, 42)") + raises(TypeError, "super(D, C())") + raises(TypeError, "super(D).__get__(12)") + raises(TypeError, "super(D).__get__(C())") Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Tue Nov 20 18:47:29 2007 @@ -541,6 +541,14 @@ return w_type.lookup_where(name) lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where' + def lookup_in_type_starting_at(self, w_type, w_starttype, name): + """ Only supposed to be used to implement super, w_starttype + and w_type are the same as for super(starttype, type) + """ + assert isinstance(w_type, W_TypeObject) + assert isinstance(w_starttype, W_TypeObject) + return w_type.lookup_starting_at(w_starttype, name) + def allocate_instance(self, cls, w_subtype): """Allocate the memory needed for an instance of an internal or user-defined type, without actually __init__ializing the instance.""" Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Tue Nov 20 18:47:29 2007 @@ -294,6 +294,20 @@ return w_self._lookup_where(name) + def lookup_starting_at(w_self, w_starttype, name): + space = w_self.space + # XXX Optimize this with method cache + look = False + for w_class in w_self.mro_w: + if w_class is w_starttype: + look = True + elif look: + w_value = w_class.getdictvalue_w(space, name) + if w_value is not None: + return w_value + return None + + def _lookup(w_self, key): space = w_self.space for w_class in w_self.mro_w: From rxe at codespeak.net Tue Nov 20 18:48:30 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 18:48:30 +0100 (CET) Subject: [pypy-svn] r48859 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport Message-ID: <20071120174830.4D3B98140@code0.codespeak.net> Author: rxe Date: Tue Nov 20 18:48:29 2007 New Revision: 48859 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe) since types are immortal, no need for gc offset and remove XXX Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 18:48:29 2007 @@ -131,8 +131,7 @@ def to_exception_type(addr): # XXX we should define the structure - XXX - addr_str = ctypes.cast(addr+GC_OFFSET+12, ctypes.POINTER(ctypes.c_int)).contents.value + addr_str = ctypes.cast(addr+12, ctypes.POINTER(ctypes.c_int)).contents.value size = ctypes.cast(addr_str, ctypes.POINTER(ctypes.c_int)).contents.value - 1 name = ctypes.string_at(addr_str+4, size) return name From rxe at codespeak.net Tue Nov 20 19:00:15 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 19:00:15 +0100 (CET) Subject: [pypy-svn] r48860 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: llsupport llvm/test Message-ID: <20071120180015.7CC8B812D@code0.codespeak.net> Author: rxe Date: Tue Nov 20 19:00:15 2007 New Revision: 48860 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py Log: (cfbolz, rxe) move wrapfn() to modwrapper so genc can use it Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 19:00:15 2007 @@ -284,4 +284,25 @@ return 'ctypes.c_void_p' else: return self.TO_CTYPES[T] - + +def wrapfn(fn): + def wrapped(*args): + callargs = [] + for a in args: + if hasattr(a, 'chars'): + callargs.append(''.join(a.chars)) + else: + callargs.append(a) + res = fn(*callargs) + if isinstance(res, dict): + # these mappings are a simple protocol to work over isolate + mapping = { + "exceptiontypename": ExceptionWrapper, + "tuple": StructTuple, + "r_uint": r_uint, + "r_longlong": r_longlong, + "r_ulonglong": r_ulonglong, + } + res = mapping[res["type"]](res["value"]) + return res + return wrapped Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py Tue Nov 20 19:00:15 2007 @@ -6,6 +6,7 @@ from pypy.translator.llvm.genllvm import GenLLVM from pypy.annotation.model import lltype_to_annotation from pypy.rpython.lltypesystem.lltype import typeOf +from pypy.translator.llsupport.modwrapper import wrapfn optimize_tests = False native_llvm_backend = True @@ -92,28 +93,6 @@ return self[i] else: raise AttributeError, name - -def wrapfn(fn): - def wrapped(*args): - callargs = [] - for a in args: - if hasattr(a, 'chars'): - callargs.append(''.join(a.chars)) - else: - callargs.append(a) - res = fn(*callargs) - if isinstance(res, dict): - # these mappings are a simple protocol to work over isolate - mapping = { - "exceptiontypename": ExceptionWrapper, - "tuple": StructTuple, - "r_uint": r_uint, - "r_longlong": r_longlong, - "r_ulonglong": r_ulonglong, - } - res = mapping[res["type"]](res["value"]) - return res - return wrapped def genllvm_compile(function, annotation, From fijal at codespeak.net Tue Nov 20 19:02:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 19:02:20 +0100 (CET) Subject: [pypy-svn] r48861 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071120180220.470D6813F@code0.codespeak.net> Author: fijal Date: Tue Nov 20 19:02:19 2007 New Revision: 48861 Modified: pypy/dist/pypy/module/__builtin__/descriptor.py Log: Two performance-related moves Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Nov 20 19:02:19 2007 @@ -19,8 +19,8 @@ if self.w_self is None or space.is_w(w_obj, space.w_None): return w(self) else: - return space.call(self.w_selftype, space.newlist([ - self.w_starttype, w_obj])) + return space.call_function(self.w_selftype, self.w_starttype, w_obj + ) get.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] def getattribute(self, space, name): @@ -28,8 +28,8 @@ if name == '__class__': return self.w_selftype if self.w_type is None: - return space.call(object_getattribute(space), - space.newlist([w(self), w(name)])) + return space.call_function(object_getattribute(space), + w(self), w(name)) w_value = space.lookup_in_type_starting_at(self.w_type, self.w_starttype, @@ -47,6 +47,7 @@ if not o.match(space, space.w_AttributeError): raise return w_value + ###return space.call_function(w_get, w_self, self.w_type) return space.call(w_get, space.newlist([w_self, self.w_type])) getattribute.unwrap_spec = ['self', ObjSpace, str] From fijal at codespeak.net Tue Nov 20 19:03:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 19:03:11 +0100 (CET) Subject: [pypy-svn] r48862 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071120180311.9A22C8145@code0.codespeak.net> Author: fijal Date: Tue Nov 20 19:03:11 2007 New Revision: 48862 Modified: pypy/dist/pypy/module/__builtin__/descriptor.py Log: Kill XXX Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Nov 20 19:03:11 2007 @@ -4,7 +4,6 @@ Arguments from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError -# XXX from pypy.interpreter.callmethod import object_getattribute class W_Super(Wrappable): @@ -90,3 +89,4 @@ def meth(self, arg): super(C, self).meth(arg)""" ) + From cfbolz at codespeak.net Tue Nov 20 19:24:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 19:24:53 +0100 (CET) Subject: [pypy-svn] r48863 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c llsupport llvm/test Message-ID: <20071120182453.97904814B@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 19:24:52 2007 New Revision: 48863 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py Log: (cfbolz, rxe): make the wrapper unwrap tuples recursively, raise exceptions when they are encountered. Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Tue Nov 20 19:24:52 2007 @@ -207,9 +207,10 @@ return self.c_ext_module def get_entry_point(self): + from pypy.translator.llsupport import modwrapper assert self.c_ext_module - return getattr(self.c_ext_module, - self.entrypoint.func_name) + return modwrapper.wrapfn(getattr(self.c_ext_module, + self.entrypoint.func_name)) def cleanup(self): assert self.c_ext_module Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 19:24:52 2007 @@ -3,6 +3,7 @@ import py import ctypes from pypy.rpython.lltypesystem import lltype +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.rpython.lltypesystem.rstr import STR class CtypesModule: @@ -285,6 +286,29 @@ else: return self.TO_CTYPES[T] + +def unwrap(value): + import exceptions + if isinstance(value, dict): + # these mappings are a simple protocol to work over isolate + t = value["type"] + v = value["value"] + mapping = { + "exceptiontypename": ExceptionWrapper, + "r_uint": r_uint, + "r_longlong": r_longlong, + "r_ulonglong": r_ulonglong, + } + if t == "exceptiontypename": + exc_class = getattr(exceptions, v) + if exc_class is None: + exc_class = ExceptionWrapper(v) + raise exc_class() + if t == "tuple": + return StructTuple([unwrap(element) for element in v]) + value = mapping[t](v) + return value + def wrapfn(fn): def wrapped(*args): callargs = [] @@ -294,15 +318,20 @@ else: callargs.append(a) res = fn(*callargs) - if isinstance(res, dict): - # these mappings are a simple protocol to work over isolate - mapping = { - "exceptiontypename": ExceptionWrapper, - "tuple": StructTuple, - "r_uint": r_uint, - "r_longlong": r_longlong, - "r_ulonglong": r_ulonglong, - } - res = mapping[res["type"]](res["value"]) - return res + return unwrap(res) return wrapped + +class ExceptionWrapper: + def __init__(self, class_name): + self.class_name = class_name + + def __repr__(self): + return 'ExceptionWrapper(%s)' % repr(self.class_name) + +class StructTuple(tuple): + def __getattr__(self, name): + if name.startswith('item'): + i = int(name[len('item'):]) + return self[i] + else: + raise AttributeError, name Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llvm/test/runtest.py Tue Nov 20 19:24:52 2007 @@ -1,7 +1,6 @@ import py from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.translator.llvm.buildllvm import llvm_is_on_path, llvm_version, gcc_version from pypy.translator.llvm.genllvm import GenLLVM from pypy.annotation.model import lltype_to_annotation @@ -79,20 +78,6 @@ #______________________________________________________________________________ -class ExceptionWrapper: - def __init__(self, class_name): - self.class_name = class_name - - def __repr__(self): - return 'ExceptionWrapper(%s)' % repr(self.class_name) - -class StructTuple(tuple): - def __getattr__(self, name): - if name.startswith('item'): - i = int(name[len('item'):]) - return self[i] - else: - raise AttributeError, name def genllvm_compile(function, annotation, @@ -203,20 +188,11 @@ def interpret(self, fn, args, annotation=None): fn = self._compile(fn, args, annotation) - res = fn(*args) - if isinstance(res, ExceptionWrapper): - raise res - return res + return fn(*args) def interpret_raises(self, exception, fn, args): import exceptions # needed by eval - try: - self.interpret(fn, args) - except ExceptionWrapper, ex: - assert issubclass(eval(ex.class_name), exception) - return True - else: - assert False, 'function did raise no exception at all' + return py.test.raises(exception, self.interpret, fn, args) def float_eq(self, x, y): diff = abs(x-y) From pedronis at codespeak.net Tue Nov 20 19:28:26 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 20 Nov 2007 19:28:26 +0100 (CET) Subject: [pypy-svn] r48864 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071120182826.B82B78165@code0.codespeak.net> Author: pedronis Date: Tue Nov 20 19:28:24 2007 New Revision: 48864 Modified: pypy/dist/pypy/module/__builtin__/descriptor.py Log: this should really be a space.w_None Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Nov 20 19:28:24 2007 @@ -39,15 +39,14 @@ try: w_get = space.getattr(w_value, space.wrap('__get__')) if space.is_w(self.w_self, self.w_type): - w_self = None + w_self = space.w_None else: w_self = self.w_self except OperationError, o: if not o.match(space, space.w_AttributeError): raise return w_value - ###return space.call_function(w_get, w_self, self.w_type) - return space.call(w_get, space.newlist([w_self, self.w_type])) + return space.call_function(w_get, w_self, self.w_type) getattribute.unwrap_spec = ['self', ObjSpace, str] def descr_new_super(space, w_self, w_starttype, w_obj_or_type=None): From xoraxax at codespeak.net Tue Nov 20 19:31:21 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 20 Nov 2007 19:31:21 +0100 (CET) Subject: [pypy-svn] r48865 - in pypy/branch/kill-remaining-suggested-primitives: . pypy pypy/annotation pypy/annotation/test pypy/interpreter pypy/objspace/flow pypy/objspace/flow/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/module pypy/rpython/ootypesystem/test pypy/rpython/test pypy/tool pypy/tool/test pypy/translator pypy/translator/c pypy/translator/goal pypy/translator/jvm pypy/translator/stackless pypy/translator/stackless/test pypy/translator/tool Message-ID: <20071120183121.B29538188@code0.codespeak.net> Author: xoraxax Date: Tue Nov 20 19:31:20 2007 New Revision: 48865 Added: pypy/branch/kill-remaining-suggested-primitives/ - copied from r48795, pypy/dist/ pypy/branch/kill-remaining-suggested-primitives/pypy/ - copied from r48813, pypy/dist/pypy/ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/ll2ctypes.py - copied unchanged from r48844, pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/rffi.py - copied unchanged from r48844, pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/rcontrollerentry.py - copied unchanged from r48849, pypy/dist/pypy/rpython/rcontrollerentry.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_controllerentry.py - copied unchanged from r48849, pypy/dist/pypy/rpython/test/test_controllerentry.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_stack.py pypy/branch/kill-remaining-suggested-primitives/pypy/tool/descriptor.py pypy/branch/kill-remaining-suggested-primitives/pypy/tool/test/test_descriptor.py - copied, changed from r48813, pypy/dist/pypy/tool/test/test_instancemethod.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/jvm/genjvm.py - copied unchanged from r48840, pypy/dist/pypy/translator/jvm/genjvm.py Removed: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/module/ll_stack.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/module/ll_stackless.py pypy/branch/kill-remaining-suggested-primitives/pypy/tool/instancemethod.py pypy/branch/kill-remaining-suggested-primitives/pypy/tool/test/test_instancemethod.py Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/builtin.py pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/model.py pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/test/test_annrpython.py pypy/branch/kill-remaining-suggested-primitives/pypy/interpreter/eval.py pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/__init__.py pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/model.py pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_framestate.py pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_objspace.py pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/rstack.py pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/test/test_nonconst.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/controllerentry.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/extfunctable.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/llinterp.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/lloperation.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_bltann.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_ooann.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_oortype.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/rbuiltin.py pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_llann.py pypy/branch/kill-remaining-suggested-primitives/pypy/tool/udir.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/extfunc.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/exceptiontransform.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/geninterplevel.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/goal/targetlbench.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/frame.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/test/test_yield_current_frame_to_caller.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/transform.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/tool/make_dot.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/transform.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/translator.py Log: (armin, alexander): Removed the remaining suggested primitives, fiddled with controller logic and made it working in the end again. Checking it to run tests easily on remote machines ... Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/builtin.py Tue Nov 20 19:31:20 2007 @@ -18,7 +18,6 @@ from pypy.objspace.flow.model import Constant import pypy.rlib.rarithmetic import pypy.rlib.objectmodel -import pypy.rlib.rstack # convenience only! def immutablevalue(x): @@ -339,9 +338,6 @@ def llmemory_cast_int_to_adr(s): return SomeAddress() -def rstack_yield_current_frame_to_caller(): - return SomeExternalObject(pypy.rlib.rstack.frame_stack_top) - ##def rarith_ovfcheck(s_obj): ## if isinstance(s_obj, SomeInteger) and s_obj.unsigned: @@ -384,8 +380,6 @@ BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_ptr] = llmemory_cast_adr_to_ptr BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_int] = llmemory_cast_adr_to_int BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_int_to_adr] = llmemory_cast_int_to_adr -BUILTIN_ANALYZERS[pypy.rlib.rstack.yield_current_frame_to_caller] = ( - rstack_yield_current_frame_to_caller) BUILTIN_ANALYZERS[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( OSError_init) Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/model.py Tue Nov 20 19:31:20 2007 @@ -29,7 +29,8 @@ from types import BuiltinFunctionType, MethodType, FunctionType -import pypy.tool.instancemethod +import pypy +from pypy.tool import descriptor from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, base_int @@ -441,7 +442,7 @@ def __init__(self, analyser, s_self=None, methodname=None): if isinstance(analyser, MethodType): - analyser = pypy.tool.instancemethod.InstanceMethod( + analyser = descriptor.InstanceMethod( analyser.im_func, analyser.im_self, analyser.im_class) Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/test/test_annrpython.py Tue Nov 20 19:31:20 2007 @@ -16,7 +16,7 @@ from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong from pypy.rlib.rarithmetic import r_singlefloat from pypy.rlib import objectmodel -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.translator.test import snippet Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/interpreter/eval.py ============================================================================== --- pypy/dist/pypy/interpreter/eval.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/interpreter/eval.py Tue Nov 20 19:31:20 2007 @@ -4,7 +4,6 @@ """ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable -from pypy.rlib import rstack # for resume points class Code(Wrappable): Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/__init__.py ============================================================================== --- pypy/dist/pypy/objspace/flow/__init__.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/__init__.py Tue Nov 20 19:31:20 2007 @@ -1,2 +0,0 @@ -from pypy.objspace.flow.objspace import FlowObjSpace -Space = FlowObjSpace Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/model.py Tue Nov 20 19:31:20 2007 @@ -6,6 +6,7 @@ from __future__ import generators import py from pypy.tool.uid import uid, Hashable +from pypy.tool.descriptor import roproperty from pypy.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func """ @@ -30,15 +31,6 @@ __metaclass__ = type -class roproperty(object): - def __init__(self, getter): - self.getter = getter - def __get__(self, obj, cls=None): - if obj is None: - return self - else: - return self.getter(obj) - class FunctionGraph(object): __slots__ = ['startblock', 'returnblock', 'exceptblock', '__dict__'] Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_framestate.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_framestate.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_framestate.py Tue Nov 20 19:31:20 2007 @@ -4,7 +4,7 @@ from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import * from pypy.interpreter.pycode import PyCode -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace class TestFrameState: def setup_class(cls): Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/test/test_objspace.py Tue Nov 20 19:31:20 2007 @@ -4,7 +4,7 @@ from pypy.objspace.flow.model import flatten from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow import objspace from pypy import conftest Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/rstack.py ============================================================================== --- pypy/dist/pypy/rlib/rstack.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/rstack.py Tue Nov 20 19:31:20 2007 @@ -5,17 +5,35 @@ import inspect +from pypy.rlib.objectmodel import we_are_translated +from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.controllerentry import Controller, SomeControlledInstance + + def stack_unwind(): + if we_are_translated(): + from pypy.rpython.lltypesystem.lloperation import llop + return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") + def stack_capture(): + if we_are_translated(): + from pypy.rpython.lltypesystem.lloperation import llop + ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) + return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") + def stack_frames_depth(): - return len(inspect.stack()) + if we_are_translated(): + from pypy.rpython.lltypesystem.lloperation import llop + return llop.stack_frames_depth(lltype.Signed) + else: + return len(inspect.stack()) -def stack_too_big(): - return False +stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, _callable=lambda: 0) def stack_check(): if stack_too_big(): @@ -29,16 +47,62 @@ def yield_current_frame_to_caller(): raise NotImplementedError("only works in translated versions") + class frame_stack_top(object): def switch(self): raise NotImplementedError("only works in translated versions") -from pypy.rpython.extregistry import ExtRegistryEntry +class BoundSwitchOfFrameStackTop(object): pass +class BoundSwitchOfFrameStackTopController(Controller): + knowntype = BoundSwitchOfFrameStackTop + def call(self, real_object): + from pypy.rpython.lltypesystem.lloperation import llop + ptr = llop.stack_switch(OPAQUE_STATE_HEADER_PTR, real_object) + return frame_stack_top_controller.box(ptr) + + +class FrameStackTopController(Controller): + knowntype = frame_stack_top + can_be_None = True + + def is_true(self, real_object): + return bool(real_object) + + def get_switch(self, real_object): + return bound_switch_of_frame_stack_top_controller.box(real_object) + + def convert(self, obj): + assert obj is None + return lltype.nullptr(OPAQUE_STATE_HEADER_PTR.TO) + +frame_stack_top_controller = FrameStackTopController() +bound_switch_of_frame_stack_top_controller = BoundSwitchOfFrameStackTopController() +OPAQUE_STATE_HEADER = lltype.GcOpaqueType("OPAQUE_STATE_HEADER") +OPAQUE_STATE_HEADER._exttypeinfo = "Really bad hack - dont remove" +OPAQUE_STATE_HEADER_PTR = lltype.Ptr(OPAQUE_STATE_HEADER) + + + +class FrameStackTopReturningFnEntry(ExtRegistryEntry): + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + return SomeControlledInstance(annmodel.lltype_to_annotation(OPAQUE_STATE_HEADER_PTR), frame_stack_top_controller) + + +class YieldCurrentFrameToCallerFnEntry(FrameStackTopReturningFnEntry): + _about_ = yield_current_frame_to_caller + + def specialize_call(self, hop): + var = hop.genop("yield_current_frame_to_caller", [], hop.r_result.lowleveltype) + return var + def resume_point(label, *args, **kwds): pass + + class ResumePointFnEntry(ExtRegistryEntry): _about_ = resume_point @@ -83,17 +147,14 @@ r_arg = hop.rtyper.bindingrepr(v_arg) return hop.inputarg(r_arg, arg=index) -class ResumeStateCreateFnEntry(ExtRegistryEntry): +class ResumeStateCreateFnEntry(FrameStackTopReturningFnEntry): _about_ = resume_state_create def compute_result_annotation(self, s_prevstate, s_label, *args_s): - from pypy.annotation import model as annmodel - return annmodel.SomeExternalObject(frame_stack_top) + return FrameStackTopReturningFnEntry.compute_result_annotation(self) def specialize_call(self, hop): from pypy.rpython.lltypesystem import lltype - from pypy.rpython.rmodel import SimplePointerRepr - from pypy.translator.stackless.frame import STATE_HEADER assert hop.args_s[1].is_constant() c_label = hop.inputconst(lltype.Void, hop.args_s[1].const) @@ -142,4 +203,3 @@ return hop.genop('resume_state_invoke', [v_state, v_returning, v_raising], hop.r_result) - Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/test/test_nonconst.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_nonconst.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/test/test_nonconst.py Tue Nov 20 19:31:20 2007 @@ -4,7 +4,7 @@ from pypy.rlib.nonconst import NonConstant -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.annotation.annrpython import RPythonAnnotator from pypy.conftest import option from pypy.annotation.model import SomeInstance Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/controllerentry.py ============================================================================== --- pypy/dist/pypy/rpython/controllerentry.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/controllerentry.py Tue Nov 20 19:31:20 2007 @@ -1,5 +1,6 @@ from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype +from pypy.annotation.binaryop import _make_none_union, SomePBC # SomePBC needed by _make_none_union from pypy.annotation.bookkeeper import getbookkeeper from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.annlowlevel import cachedtype @@ -48,6 +49,7 @@ class Controller(object): __metaclass__ = cachedtype + can_be_None = False def _freeze_(self): return True @@ -210,6 +212,9 @@ self.controller = controller self.knowntype = controller.knowntype + def can_be_none(self): + return self.controller.can_be_None + def rtyper_makerepr(self, rtyper): from pypy.rpython.rcontrollerentry import ControlledInstanceRepr return ControlledInstanceRepr(rtyper, self.s_real_obj, self.controller) @@ -218,6 +223,7 @@ real_key = rtyper.makekey(self.s_real_obj) return self.__class__, real_key, self.controller +_make_none_union("SomeControlledInstance", "obj.s_real_obj, obj.controller", globals()) class __extend__(SomeControlledInstance): Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/extfunctable.py ============================================================================== --- pypy/dist/pypy/rpython/extfunctable.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/extfunctable.py Tue Nov 20 19:31:20 2007 @@ -163,18 +163,6 @@ ntpath.isabs = isabs # ___________________________________________________________ -# stackless -from pypy.rlib import rstack -declare(rstack.stack_frames_depth, int, 'll_stackless/stack_frames_depth') -declare(rstack.stack_too_big, bool, 'll_stack/too_big') -declare(rstack.stack_check, noneannotation, 'll_stack/check') -declare(rstack.stack_unwind, noneannotation, 'll_stack/unwind') -declare(rstack.stack_capture, rstack.frame_stack_top, 'll_stack/capture') -frametop_type_info = declaregcptrtype(rstack.frame_stack_top,'frame_stack_top', - switch = (rstack.frame_stack_top, - 'll_stackless/switch')) - -# ___________________________________________________________ # the exceptions that can be implicitely raised by some operations standardexceptions = { TypeError : True, Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/llinterp.py Tue Nov 20 19:31:20 2007 @@ -812,6 +812,17 @@ def op_yield_current_frame_to_caller(self): raise NotImplementedError("yield_current_frame_to_caller") + def op_stack_frames_depth(self): + return len(self.llinterpreter.frame_stack) + + def op_stack_switch(self, frametop): + raise NotImplementedError("stack_switch") + + def op_stack_unwind(self): + raise NotImplementedError("stack_unwind") + + def op_stack_capture(self): + raise NotImplementedError("stack_capture") # operations on pyobjects! for opname in lloperation.opimpls.keys(): Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/lloperation.py Tue Nov 20 19:31:20 2007 @@ -3,7 +3,7 @@ """ from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.objspace.flow.model import roproperty +from pypy.tool.descriptor import roproperty class LLOp(object): @@ -95,6 +95,12 @@ return '' % (getattr(self, 'opname', '?'),) +class _LLOP(object): + def _freeze_(self): + return True +llop = _LLOP() + + def enum_ops_without_sideeffects(raising_is_ok=False): """Enumerate operations that have no side-effects (see also enum_foldable_ops).""" @@ -425,6 +431,10 @@ 'resume_point': LLOp(canraise=(Exception,)), 'resume_state_create': LLOp(canraise=(MemoryError,), canunwindgc=True), 'resume_state_invoke': LLOp(canraise=(Exception, StackException)), + 'stack_frames_depth': LLOp(sideeffects=False, canraise=(StackException, )), + 'stack_switch': LLOp(canraise=(StackException, )), + 'stack_unwind': LLOp(canraise=(StackException, )), + 'stack_capture': LLOp(canraise=(StackException, )), # __________ misc operations __________ @@ -500,10 +510,6 @@ # Also export all operations in an attribute-based namespace. # Example usage from LL helpers: z = llop.int_add(Signed, x, y) -class LLOP(object): - def _freeze_(self): - return True -llop = LLOP() for opname, opdesc in LL_OPERATIONS.iteritems(): setattr(llop, opname, opdesc) del opname, opdesc Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_bltann.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_bltann.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_bltann.py Tue Nov 20 19:31:20 2007 @@ -4,7 +4,7 @@ import py from pypy.annotation import model as annmodel -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.annotation.annrpython import RPythonAnnotator import exceptions from pypy.rpython.ootypesystem.bltregistry import BasicExternal, ExternalType, MethodDesc, described Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_ooann.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_ooann.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_ooann.py Tue Nov 20 19:31:20 2007 @@ -1,7 +1,7 @@ import py from pypy.rpython.ootypesystem.ootype import * from pypy.annotation import model as annmodel -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.annotation.annrpython import RPythonAnnotator import exceptions from pypy.rpython.ootypesystem import ooregistry # side effects Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/test/test_oortype.py Tue Nov 20 19:31:20 2007 @@ -5,7 +5,7 @@ from pypy.rpython.ootypesystem.rlist import ListRepr from pypy.rpython.rint import signed_repr from pypy.annotation import model as annmodel -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.test_llinterp import interpret from pypy.rlib.objectmodel import r_dict Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/rbuiltin.py Tue Nov 20 19:31:20 2007 @@ -3,7 +3,7 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, rclass, llmemory from pypy.rpython import rint, raddress -from pypy.rlib import rarithmetic, rstack, objectmodel +from pypy.rlib import rarithmetic, objectmodel from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst from pypy.rpython.rrange import rtype_builtin_range, rtype_builtin_xrange @@ -273,10 +273,6 @@ hop.exception_cannot_occur() return hop.inputconst(lltype.Bool, True) -def rtype_yield_current_frame_to_caller(hop): - return hop.genop('yield_current_frame_to_caller', [], - resulttype=hop.r_result) - def rtype_hlinvoke(hop): _, s_repr = hop.r_s_popfirstarg() r_callable = s_repr.const @@ -516,8 +512,6 @@ BUILTIN_TYPER[lltype.runtime_type_info] = rtype_runtime_type_info BUILTIN_TYPER[rarithmetic.intmask] = rtype_intmask BUILTIN_TYPER[objectmodel.we_are_translated] = rtype_we_are_translated -BUILTIN_TYPER[rstack.yield_current_frame_to_caller] = ( - rtype_yield_current_frame_to_caller) BUILTIN_TYPER[objectmodel.hlinvoke] = rtype_hlinvoke Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llann.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_llann.py Tue Nov 20 19:31:20 2007 @@ -9,7 +9,7 @@ from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.test.test_llinterp import interpret -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.conftest import option # helpers Added: pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_stack.py ============================================================================== --- (empty file) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_stack.py Tue Nov 20 19:31:20 2007 @@ -0,0 +1,16 @@ + +from pypy.rpython.test.test_llinterp import interpret +from pypy.rlib.rstack import stack_frames_depth + + +def test_interp_c(): + def f(): + return stack_frames_depth() + + def g(): + return f() + res_f = interpret(f, []) + res_g = interpret(g, []) + assert res_f == 2 + assert res_g == 3 + Added: pypy/branch/kill-remaining-suggested-primitives/pypy/tool/descriptor.py ============================================================================== --- (empty file) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/tool/descriptor.py Tue Nov 20 19:31:20 2007 @@ -0,0 +1,40 @@ + +class roproperty(object): + def __init__(self, getter): + self.getter = getter + def __get__(self, obj, cls=None): + if obj is None: + return self + else: + return self.getter(obj) + + +class InstanceMethod(object): + "Like types.InstanceMethod, but with a reasonable (structural) equality." + + def __init__(self, im_func, im_self, im_class): + self.im_func = im_func + self.im_self = im_self + self.im_class = im_class + + def __call__(self, *args, **kwds): + firstarg = self.im_self + if firstarg is None: + if not args or not isinstance(args[0], self.im_class): + raise TypeError( + "must be called with %r instance as first argument" % ( + self.im_class,)) + firstarg = args[0] + args = args[1:] + return self.im_func(firstarg, *args, **kwds) + + def __eq__(self, other): + return isinstance(other, InstanceMethod) and ( + self.im_func == other.im_func and + self.im_self == other.im_self) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.im_func, self.im_self)) Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/tool/udir.py ============================================================================== --- pypy/dist/pypy/tool/udir.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/tool/udir.py Tue Nov 20 19:31:20 2007 @@ -9,4 +9,4 @@ from py.path import local -udir = local.make_numbered_dir(prefix='usession-', keep=3) +udir = local.make_numbered_dir(prefix='usession-', keep=1000000) Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/extfunc.py Tue Nov 20 19:31:20 2007 @@ -6,17 +6,12 @@ from pypy.rpython.lltypesystem import rstr from pypy.rpython.lltypesystem import rlist from pypy.rpython.module import ll_time, ll_os -from pypy.rpython.module import ll_stackless, ll_stack # table of functions hand-written in src/ll_*.h # Note about *.im_func: The annotator and the rtyper expect direct # references to functions, so we cannot insert classmethods here. EXTERNALS = { - ll_stackless.ll_stackless_switch: 'LL_stackless_switch', - ll_stackless.ll_stackless_stack_frames_depth: 'LL_stackless_stack_frames_depth', - ll_stack.ll_stack_unwind: 'LL_stack_unwind', - ll_stack.ll_stack_too_big: 'LL_stack_too_big', 'LL_flush_icache': 'LL_flush_icache', } Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/exceptiontransform.py Tue Nov 20 19:31:20 2007 @@ -55,6 +55,10 @@ self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) exc_data, null_type, null_value = self.setup_excdata() + runtime_error_def = translator.annotator.bookkeeper.getuniqueclassdef(RuntimeError) + runtime_error_ll_exc = edata.get_standard_ll_exc_instance(translator.rtyper, runtime_error_def) + runtime_error_ll_exc_type = runtime_error_ll_exc.typeptr + def rpyexc_occured(): exc_type = exc_data.exc_type return bool(exc_type) @@ -81,6 +85,9 @@ exc_data.exc_type = etype exc_data.exc_value = evalue + def rpyexc_raise_runtime_error(): + rpyexc_raise(runtime_error_ll_exc_type, runtime_error_ll_exc) + self.rpyexc_occured_ptr = self.build_func( "RPyExceptionOccurred", rpyexc_occured, @@ -114,6 +121,11 @@ lltype.Void, jitcallkind='rpyexc_raise') # for the JIT + self.rpyexc_raise_runtime_error_ptr = self.build_func( + "RPyRaiseRuntimeError", + rpyexc_raise_runtime_error, + [], lltype.Void) + self.mixlevelannotator.finish() self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() @@ -148,6 +160,7 @@ n_need_exc_matching_blocks = 0 n_gen_exc_checks = 0 for block in list(graph.iterblocks()): + self.replace_stack_unwind(block) need_exc_matching, gen_exc_checks = self.transform_block(graph, block) n_need_exc_matching_blocks += need_exc_matching n_gen_exc_checks += gen_exc_checks @@ -156,6 +169,16 @@ removenoops.remove_superfluous_keep_alive(graph) return n_need_exc_matching_blocks, n_gen_exc_checks + def replace_stack_unwind(self, block): + for i in range(len(block.operations)): + if block.operations[i].opname == 'stack_unwind': + # if there are stack_unwind ops left, + # the graph was not stackless-transformed + # so we need to raise a RuntimeError in any + # case + block.operations[i].opname = "direct_call" + block.operations[i].args = [self.rpyexc_raise_runtime_error_ptr] + def transform_block(self, graph, block): need_exc_matching = False n_gen_exc_checks = 0 Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/geninterplevel.py ============================================================================== --- pypy/dist/pypy/translator/geninterplevel.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/geninterplevel.py Tue Nov 20 19:31:20 2007 @@ -57,7 +57,7 @@ from pypy.translator.backendopt.ssa import SSI_to_SSA from pypy.translator.translator import TranslationContext -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.tool.sourcetools import render_docstr, NiceCompile Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/goal/targetlbench.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetlbench.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/goal/targetlbench.py Tue Nov 20 19:31:20 2007 @@ -5,27 +5,26 @@ TP = lltype.GcArray(lltype.Signed) -def f(x): +def longername(a, b, size): if 1: - a = lltype.malloc(TP, x) - for i in range(x): - a[i] = i - b = lltype.malloc(TP, x, zero=False) - for j in range(1000): - #for i in range(x): - # b[i] = a[i] - baseofs = itemoffsetof(TP, 0) - onesize = sizeof(TP.OF) - size = baseofs + onesize*(x - 1) - raw_memcopy(cast_ptr_to_adr(b)+baseofs, cast_ptr_to_adr(a)+baseofs, size) + baseofs = itemoffsetof(TP, 0) + onesize = sizeof(TP.OF) + size = baseofs + onesize*(size - 1) + raw_memcopy(cast_ptr_to_adr(b)+baseofs, cast_ptr_to_adr(a)+baseofs, size) else: a = [] for i in range(x): a.append(i) return 0 +longername.dont_inline = True def entry_point(argv): - print f(int(argv[1])) + size = int(argv[1]) + a = lltype.malloc(TP, size) + b = lltype.malloc(TP, size, zero=False) + for i in range(size): + a[i] = i + print longername(a, b, size) return 0 # _____ Define and setup target ___ Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/frame.py ============================================================================== --- pypy/dist/pypy/translator/stackless/frame.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/frame.py Tue Nov 20 19:31:20 2007 @@ -6,6 +6,7 @@ from pypy.tool.sourcetools import compile2 from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.rlib import rstack # ____________________________________________________________ # generic data types @@ -69,8 +70,7 @@ null_state = lltype.nullptr(STATE_HEADER) -OPAQUE_STATE_HEADER_PTR = lltype.Ptr( - extfunctable.frametop_type_info.get_lltype()) +OPAQUE_STATE_HEADER_PTR = rstack.OPAQUE_STATE_HEADER_PTR def make_state_header_type(name, *fields): Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/test/test_yield_current_frame_to_caller.py ============================================================================== --- pypy/dist/pypy/translator/stackless/test/test_yield_current_frame_to_caller.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/test/test_yield_current_frame_to_caller.py Tue Nov 20 19:31:20 2007 @@ -64,3 +64,25 @@ res = run_stackless_function(f) assert res == 1234567 + + +def test_frame_none_mix(): + def h(flag): + if flag: + c = g() + else: + c = None + return c + def f(): + return bool(h(False)) * 2 + bool(h(True)) + def g(): + d = rstack.yield_current_frame_to_caller() + return d + + data = llinterp_stackless_function(f) + assert data == 1 + + res = run_stackless_function(f) + assert res == 1 + + Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/transform.py Tue Nov 20 19:31:20 2007 @@ -12,7 +12,6 @@ from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rbuiltin import gen_cast from pypy.rpython.rtyper import LowLevelOpList -from pypy.rpython.module import ll_stackless, ll_stack from pypy.rlib.objectmodel import ComputedIntSymbolic from pypy.translator.backendopt import graphanalyze @@ -250,13 +249,9 @@ self.stackless_gc = stackless_gc def operation_is_true(self, op): - if op.opname == 'yield_current_frame_to_caller': - return True - elif op.opname == 'resume_point': - return True - elif op.opname == 'resume_state_invoke': - return True - elif op.opname == 'resume_state_create': + if op.opname in ('yield_current_frame_to_caller', 'resume_point', + 'resume_state_invoke', 'resume_state_create', 'stack_frames_depth', + 'stack_switch', 'stack_unwind', 'stack_capture'): return True if self.stackless_gc: if op.opname in ('malloc', 'malloc_varsize'): @@ -265,11 +260,6 @@ return LL_OPERATIONS[op.opname].canunwindgc return False - def analyze_external_call(self, op): - callable = op.args[0].value._obj._callable - return callable in [ll_stack.ll_stack_unwind, ll_stack.ll_stack_capture, - ll_stackless.ll_stackless_stack_frames_depth, - ll_stackless.ll_stackless_switch] def vars_to_save(block): lastresult = block.operations[-1].result @@ -373,23 +363,20 @@ } s_StatePtr = annmodel.SomePtr(frame.OPAQUE_STATE_HEADER_PTR) - self.suggested_primitives = { - ll_stackless.ll_stackless_stack_frames_depth: - mixlevelannotator.constfunc( + + self.operation_replacement = { + 'yield_current_frame_to_caller': mixlevelannotator.constfunc( + code.yield_current_frame_to_caller, [], s_StatePtr), + 'stack_frames_depth': mixlevelannotator.constfunc( code.stack_frames_depth, [], annmodel.SomeInteger()), - ll_stackless.ll_stackless_switch: - mixlevelannotator.constfunc( + 'stack_switch': mixlevelannotator.constfunc( code.ll_frame_switch, [s_StatePtr], s_StatePtr), - ll_stack.ll_stack_unwind: - mixlevelannotator.constfunc( + 'stack_unwind': mixlevelannotator.constfunc( code.ll_stack_unwind, [], annmodel.s_None), - ll_stack.ll_stack_capture: - mixlevelannotator.constfunc( + 'stack_capture': mixlevelannotator.constfunc( code.ll_stack_capture, [], s_StatePtr), - } + } - self.yield_current_frame_to_caller_ptr = mixlevelannotator.constfunc( - code.yield_current_frame_to_caller, [], s_StatePtr) s_hdrptr = annmodel.SomePtr(lltype.Ptr(STATE_HEADER)) # order really matters on 64 bits machines on which @@ -798,7 +785,7 @@ i = 0 def replace_with_call(fnptr): - args = [fnptr] + op.args[1:] + args = [fnptr] + op.args newop = model.SpaceOperation('direct_call', args, op.result) block.operations[i] = newop return newop @@ -806,8 +793,9 @@ while i < len(block.operations): stackless_op = False op = block.operations[i] - if op.opname == 'yield_current_frame_to_caller': - op = replace_with_call(self.yield_current_frame_to_caller_ptr) + + if op.opname in self.operation_replacement: + op = replace_with_call(self.operation_replacement[op.opname]) stackless_op = True if op.opname == 'resume_state_create': @@ -824,13 +812,6 @@ i = 0 continue - # trap calls to stackless-related suggested primitives - if op.opname == 'direct_call': - func = getattr(op.args[0].value._obj, '_callable', None) - if func in self.suggested_primitives: - op = replace_with_call(self.suggested_primitives[func]) - stackless_op = True - if not stackless_op and not self.analyzer.analyze(op): i += 1 continue Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/tool/make_dot.py ============================================================================== --- pypy/dist/pypy/translator/tool/make_dot.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/tool/make_dot.py Tue Nov 20 19:31:20 2007 @@ -6,7 +6,7 @@ import autopath, os import inspect, linecache from pypy.objspace.flow.model import * -from pypy.objspace.flow import Space +from pypy.objspace.flow.objspace import FlowObjSpace as Space from pypy.tool.udir import udir from py.process import cmdexec from pypy.interpreter.pytraceback import offset2lineno Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/transform.py ============================================================================== --- pypy/dist/pypy/translator/transform.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/transform.py Tue Nov 20 19:31:20 2007 @@ -239,10 +239,10 @@ def insert_ll_stackcheck(translator): from pypy.translator.backendopt.support import find_calls_from - from pypy.rpython.module.ll_stack import ll_stack_check + from pypy.rlib.rstack import stack_check from pypy.tool.algo.graphlib import Edge, make_edge_dict, break_cycles rtyper = translator.rtyper - graph = rtyper.annotate_helper(ll_stack_check, []) + graph = rtyper.annotate_helper(stack_check, []) rtyper.specialize_more_blocks() stack_check_ptr = rtyper.getcallable(graph) stack_check_ptr_const = Constant(stack_check_ptr, lltype.typeOf(stack_check_ptr)) Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/translator.py ============================================================================== --- pypy/dist/pypy/translator/translator.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/translator.py Tue Nov 20 19:31:20 2007 @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import * from pypy.translator import simplify -from pypy.objspace.flow import FlowObjSpace +from pypy.objspace.flow.objspace import FlowObjSpace from pypy.tool.ansi_print import ansi_log from pypy.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription From cfbolz at codespeak.net Tue Nov 20 19:38:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 19:38:42 +0100 (CET) Subject: [pypy-svn] r48866 - pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport Message-ID: <20071120183842.4C80D814F@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 19:38:40 2007 New Revision: 48866 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: remove this, it's not needed any more Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 19:38:40 2007 @@ -30,8 +30,6 @@ except AttributeError: pass # only if there is Boehm -GC_OFFSET = getattr(_c, "pypy_g__size_of_gc_header", 0) - startup_code = _c.ctypes_RPython_StartupCode startup_code.argtypes = [] startup_code.restype = ctypes.c_int From fijal at codespeak.net Tue Nov 20 19:46:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 19:46:30 +0100 (CET) Subject: [pypy-svn] r48867 - pypy/dist/pypy/interpreter Message-ID: <20071120184630.D6AFC818A@code0.codespeak.net> Author: fijal Date: Tue Nov 20 19:46:30 2007 New Revision: 48867 Modified: pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/typedef.py Log: Missing attributes to make StaticMethod a first-class member Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Tue Nov 20 19:46:30 2007 @@ -461,6 +461,9 @@ """staticmethod(x).__get__(obj[, type]) -> x""" return self.w_function + def descr_staticmethod__new__(space, w_type, w_function): + return space.wrap(StaticMethod(w_function)) + class BuiltinFunction(Function): def __init__(self, func): Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Tue Nov 20 19:46:30 2007 @@ -677,7 +677,22 @@ ) StaticMethod.typedef = TypeDef("staticmethod", + __doc__ = """staticmethod(function) -> static method + +Convert a function to be a static method. + +A static method does not receive an implicit first argument. +To declare a static method, use this idiom: + + class C: + def f(arg1, arg2, ...): ... + f = staticmethod(f) + +It can be called either on the class (e.g. C.f()) or on an instance +(e.g. C().f()). The instance is ignored except for its class.""", __get__ = interp2app(StaticMethod.descr_staticmethod_get), + __new__ = interp2app(StaticMethod.descr_staticmethod__new__.im_func, + unwrap_spec = [ObjSpace, W_Root, W_Root]), ) def always_none(self, obj): From fijal at codespeak.net Tue Nov 20 19:47:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 20 Nov 2007 19:47:22 +0100 (CET) Subject: [pypy-svn] r48868 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071120184722.8DF4E818E@code0.codespeak.net> Author: fijal Date: Tue Nov 20 19:47:22 2007 New Revision: 48868 Modified: pypy/dist/pypy/module/__builtin__/__init__.py pypy/dist/pypy/module/__builtin__/app_descriptor.py pypy/dist/pypy/module/__builtin__/descriptor.py pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Log: Move staticmethod and classmethod to rpython Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Tue Nov 20 19:47:22 2007 @@ -56,8 +56,6 @@ 'dir' : 'app_inspect.dir', 'property' : 'app_descriptor.property', - 'staticmethod' : 'app_descriptor.staticmethod', - 'classmethod' : 'app_descriptor.classmethod', 'complex' : 'app_complex.complex', @@ -127,6 +125,8 @@ 'all' : 'functional.all', 'any' : 'functional.any', 'super' : 'descriptor.W_Super', + 'staticmethod' : 'descriptor.StaticMethod', + 'classmethod' : 'descriptor.W_ClassMethod', } def pick_builtin(self, w_globals): Modified: pypy/dist/pypy/module/__builtin__/app_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/app_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/app_descriptor.py Tue Nov 20 19:47:22 2007 @@ -5,65 +5,6 @@ # Descriptor code, shamelessly stolen from Raymond Hettinger: # http://users.rcn.com/python/download/Descriptor.htm - -# XXX there is an interp-level pypy.interpreter.function.StaticMethod -# XXX because __new__ needs to be a StaticMethod early. -class staticmethod(object): - """staticmethod(function) -> static method - -Convert a function to be a static method. - -A static method does not receive an implicit first argument. -To declare a static method, use this idiom: - - class C: - def f(arg1, arg2, ...): ... - f = staticmethod(f) - -It can be called either on the class (e.g. C.f()) or on an instance -(e.g. C().f()). The instance is ignored except for its class.""" - __slots__ = ['_f'] - - def __init__(self, f): - self._f = f - - def __get__(self, obj, objtype=None): - return self._f - - -class classmethod(object): - """classmethod(function) -> class method - -Convert a function to be a class method. - -A class method receives the class as implicit first argument, -just like an instance method receives the instance. -To declare a class method, use this idiom: - - class C: - def f(cls, arg1, arg2, ...): ... - f = classmethod(f) - -It can be called either on the class (e.g. C.f()) or on an instance -(e.g. C().f()). The instance is ignored except for its class. -If a class method is called for a derived class, the derived class -object is passed as the implied first argument.""" - __slots__ = ['_f'] - - def __init__(self, f): - if not callable(f): - raise TypeError, "'%s' object is not callable" % type(f).__name__ - self._f = f - - def __get__(self, obj, klass=None): - if klass is None: - klass = type(obj) - return MethodType(self._f, klass) - -def dummy(): pass -MethodType = type(dummy.__get__(42)) -del dummy - # It's difficult to have a class that has both a docstring and a slot called # '__doc__', but not impossible... class docstring(object): Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Nov 20 19:47:22 2007 @@ -5,6 +5,7 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError from pypy.interpreter.callmethod import object_getattribute +from pypy.interpreter.function import StaticMethod, Method class W_Super(Wrappable): def __init__(self, space, w_selftype, w_starttype, w_type, w_self): @@ -89,3 +90,42 @@ super(C, self).meth(arg)""" ) +class W_ClassMethod(Wrappable): + def __init__(self, w_function): + self.w_function = w_function + + def new(space, w_type, w_function): + if not space.is_true(space.callable(w_function)): + name = space.getattr(space.type(w_function), space.wrap('__name__')) + raise OperationError(space.w_TypeError, space.wrap( + "'%s' object is not callable" % name)) + return W_ClassMethod(w_function) + + def get(self, space, w_obj, w_klass=None): + if space.is_w(w_klass, space.w_None): + w_klass = space.type(w_obj) + return space.wrap(Method(space, self.w_function, w_klass, space.w_None)) + +W_ClassMethod.typedef = TypeDef( + 'classmethod', + __new__ = interp2app(W_ClassMethod.new.im_func, + unwrap_spec=[ObjSpace, W_Root, W_Root]), + __get__ = interp2app(W_ClassMethod.get, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __doc__ = """classmethod(function) -> class method + +Convert a function to be a class method. + +A class method receives the class as implicit first argument, +just like an instance method receives the instance. +To declare a class method, use this idiom: + + class C: + def f(cls, arg1, arg2, ...): ... + f = classmethod(f) + +It can be called either on the class (e.g. C.f()) or on an instance +(e.g. C().f()). The instance is ignored except for its class. +If a class method is called for a derived class, the derived class +object is passed as the implied first argument.""", +) Modified: pypy/dist/pypy/module/__builtin__/test/test_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Tue Nov 20 19:47:22 2007 @@ -193,3 +193,34 @@ raises(TypeError, "super(D, C())") raises(TypeError, "super(D).__get__(12)") raises(TypeError, "super(D).__get__(C())") + + def test_classmethods_various(self): + class C(object): + def foo(*a): return a + goo = classmethod(foo) + c = C() + assert C.goo(1) == (C, 1) + assert c.goo(1) == (C, 1) + + assert c.foo(1) == (c, 1) + class D(C): + pass + d = D() + assert D.goo(1) == (D, 1) + assert d.goo(1) == (D, 1) + assert d.foo(1) == (d, 1) + assert D.foo(d, 1) == (d, 1) + def f(cls, arg): return (cls, arg) + ff = classmethod(f) + assert ff.__get__(0, int)(42) == (int, 42) + assert ff.__get__(0)(42) == (int, 42) + + assert C.goo.im_self is C + assert D.goo.im_self is D + assert super(D,D).goo.im_self is D + assert super(D,d).goo.im_self is D + assert super(D,D).goo() == (D,) + assert super(D,d).goo() == (D,) + + raises(TypeError, "classmethod(1).__get__(1)") + From arigo at codespeak.net Tue Nov 20 19:57:36 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Nov 2007 19:57:36 +0100 (CET) Subject: [pypy-svn] r48869 - in pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm: . module Message-ID: <20071120185736.3F62581A7@code0.codespeak.net> Author: arigo Date: Tue Nov 20 19:57:35 2007 New Revision: 48869 Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/externs2ll.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/extfuncnode.py pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/module/genexterns.c Log: (rxe, arigo) Kill unneeded code in llvm. Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/externs2ll.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/externs2ll.py Tue Nov 20 19:57:35 2007 @@ -9,46 +9,8 @@ from pypy.translator.llvm.buildllvm import llvm_gcc_version from pypy.tool.udir import udir -from pypy.rpython.module import ll_stack - -ll_stack_too_big = """ - -define internal ccc i1 @LL_stack_too_big_() { - %result = call ccc i32 @LL_stack_too_big() - %tmp = trunc i32 %result to i1 - ret i1 %tmp -} - -""" - -# table of functions hand-written in src/ll_*.h -# Note about *.im_func: The annotator and the rtyper expect direct -# references to functions, so we cannot insert classmethods here. - -EXTERNALS = { - ll_stack.ll_stack_unwind: 'LL_stack_unwind', - ll_stack.ll_stack_too_big: 'LL_stack_too_big_', - } def predeclare_stuff(c_db): - modules = {} - def module_name(c_name): - frags = c_name[3:].split('_') - if frags[0] == '': - return '_' + frags[1] - else: - return frags[0] - - for func, funcobj in c_db.externalfuncs.items(): - c_name = EXTERNALS[func] - # construct a define LL_NEED_ to make it possible to isolate in-development externals and headers - modname = module_name(c_name) - if modname not in modules: - modules[modname] = True - yield 'LL_NEED_%s' % modname.upper(), 1 - funcptr = funcobj._as_ptr() - yield c_name, funcptr - exctransformer = c_db.exctransformer yield ('_rpyexc_occured_ptr', exctransformer._rpyexc_occured_ptr.value) yield ('rpyexc_fetch_type_ptr', exctransformer.rpyexc_fetch_type_ptr.value) @@ -228,7 +190,6 @@ predeclarefn("__ENTRY_POINT__", entrynode.get_ref()) ccode.append('#define ENTRY_POINT_DEFINED 1\n\n') - LL_stack_too_big_ = False for c_name, obj in extern_decls: if isinstance(obj, lltype.LowLevelType): s = "#define %s struct %s\n%s;\n" % (c_name, c_name, c_name) @@ -243,8 +204,6 @@ pass elif type(c_name) is str and type(obj) is int: - if c_name == 'LL_NEED_STACK': - LL_stack_too_big_ = True ccode.append("#define\t%s\t%d\n" % (c_name, obj)) else: @@ -269,9 +228,4 @@ # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) llcode = get_ll("".join(ccode), function_names, default_cconv) - - # XXX enormous temporary hack - if LL_stack_too_big_: - llcode += ll_stack_too_big - return llcode Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/extfuncnode.py Tue Nov 20 19:57:35 2007 @@ -1,13 +1,9 @@ from pypy.translator.llvm.node import FuncNode -from pypy.translator.llvm.externs2ll import EXTERNALS from pypy.rpython.lltypesystem import lltype class ExternalFuncNode(FuncNode): - def __init__(self, db, value, mapname=None): - if mapname is None: - name = value._name - else: - name = EXTERNALS[mapname] + def __init__(self, db, value): + name = value._name self.db = db self.value = value self.name = "@" + name Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/module/genexterns.c Tue Nov 20 19:57:35 2007 @@ -2,24 +2,6 @@ char *LLVM_RPython_StartupCode(void); -#define RPyRaiseSimpleException(exctype, errormsg) raise##exctype(errormsg); - -// XXX abort() this is just to make tests pass. actually it is a million times -// better than it was since it used to basically be a nooop. - -// all of these will go away at some point - -#define FAKE_ERROR(name) \ - int raisePyExc_##name(char *x) { \ - abort(); \ - } - -#ifdef LL_NEED_STACK - FAKE_ERROR(RuntimeError); - #include "src/thread.h" - #include "src/stack.h" -#endif - // raw malloc code char *raw_malloc(long size) { From rxe at codespeak.net Tue Nov 20 20:19:31 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 20:19:31 +0100 (CET) Subject: [pypy-svn] r48870 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c/src llsupport Message-ID: <20071120191931.358418160@code0.codespeak.net> Author: rxe Date: Tue Nov 20 20:19:30 2007 New Revision: 48870 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/src/mem.h pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe, pedronis watching) malloc counter support Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/src/mem.h (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/src/mem.h Tue Nov 20 20:19:30 2007 @@ -69,15 +69,10 @@ #else /*COUNT_OP_MALLOCS*/ /*------------------------------------------------------------*/ -static int count_mallocs=0, count_frees=0; +int pypy_g__count_mallocs=0, pypy_g__count_frees=0; -#define COUNT_MALLOC count_mallocs++ -#define COUNT_FREE count_frees++ - -PyObject* malloc_counters(PyObject* self, PyObject* args) -{ - return Py_BuildValue("ii", count_mallocs, count_frees); -} +#define COUNT_MALLOC pypy_g__count_mallocs++ +#define COUNT_FREE pypy_g__count_frees++ /*------------------------------------------------------------*/ #endif /*COUNT_OP_MALLOCS*/ Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 20:19:30 2007 @@ -30,6 +30,10 @@ except AttributeError: pass # only if there is Boehm +def malloc_counters(): + return (ctypes.c_int.in_dll(_c, "pypy_g__count_frees").value, + ctypes.c_int.in_dll(_c, "pypy_g__count_mallocs").value) + startup_code = _c.ctypes_RPython_StartupCode startup_code.argtypes = [] startup_code.restype = ctypes.c_int From rxe at codespeak.net Tue Nov 20 20:22:22 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 20 Nov 2007 20:22:22 +0100 (CET) Subject: [pypy-svn] r48871 - pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test Message-ID: <20071120192222.9EB5680B4@code0.codespeak.net> Author: rxe Date: Tue Nov 20 20:22:22 2007 New Revision: 48871 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_boehm.py Log: (cfbolz, rxe) remove pyobj test Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_boehm.py Tue Nov 20 20:22:22 2007 @@ -195,31 +195,6 @@ res = fn() assert res == 10 - # this test shows if we have a problem with refcounting PyObject - def test_refcount_pyobj(self): - from pypy.rpython.lltypesystem.lloperation import llop - def prob_with_pyobj(b): - return 3, b - def collect(): - llop.gc__collect(lltype.Void) - f = self.getcompiled(prob_with_pyobj, [object]) - c = self.getcompiled(collect, []) - from sys import getrefcount as g - obj = None - before = g(obj) - f(obj) - f(obj) - f(obj) - f(obj) - f(obj) - c() - c() - c() - c() - c() - after = g(obj) - assert abs(before - after) < 5 - def test_zero_malloc(self): T = lltype.GcStruct("C", ('x', lltype.Signed)) def fixed_size(): From arigo at codespeak.net Tue Nov 20 20:30:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Nov 2007 20:30:50 +0100 (CET) Subject: [pypy-svn] r48872 - in pypy/dist/pypy/rpython: module tool Message-ID: <20071120193050.5103B8142@code0.codespeak.net> Author: arigo Date: Tue Nov 20 20:30:49 2007 New Revision: 48872 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/rpython/tool/rffi_platform.py Log: Fix translator/c/test/test_extfunc:test_largefile. Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Tue Nov 20 20:30:49 2007 @@ -145,6 +145,8 @@ from pypy.rpython.tool import rffi_platform as platform class CConfig: + # This must be set to 64 on some systems to enable large file support. + _header_ = '#define _FILE_OFFSET_BITS 64' _includes_ = INCLUDES STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) config = platform.configure(CConfig) Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Tue Nov 20 20:30:49 2007 @@ -85,11 +85,13 @@ def write_header(self): f = self.f CConfig = self.config - print >> f, C_HEADER - print >> f + # NB: the _header_ must be printed before everything else, + # because it might contain #defines that need to appear before + # any system #include. print >> f, getattr(CConfig, '_header_', '') # optional for path in getattr(CConfig, '_includes_', ()): # optional print >> f, '#include <%s>' % (path,) + print >> f, C_HEADER print >> f def write_entry(self, key, entry): From jacob at codespeak.net Tue Nov 20 20:37:42 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Tue, 20 Nov 2007 20:37:42 +0100 (CET) Subject: [pypy-svn] r48873 - in pypy/branch/dist-future-fixing/pypy/interpreter: . pyparser Message-ID: <20071120193742.676048142@code0.codespeak.net> Author: jacob Date: Tue Nov 20 20:37:41 2007 New Revision: 48873 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py Log: Missed checkin. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py Tue Nov 20 20:37:41 2007 @@ -5,7 +5,7 @@ from codeop import PyCF_DONT_IMPLY_DEDENT from pypy.interpreter.error import OperationError -class AbstractCompiler: +class AbstractCompiler(object): """Abstract base class for a bytecode compiler.""" # The idea is to grow more methods here over the time, @@ -78,42 +78,53 @@ # faked compiler import warnings -from pypy.tool import stdlib___future__ -compiler_flags = 0 -compiler_features = {} -for fname in stdlib___future__.all_feature_names: - flag = getattr(stdlib___future__, fname).compiler_flag - compiler_flags |= flag - compiler_features[fname] = flag -allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT - -def get_flag_names(space, flags): - if flags & ~allowed_flags: - raise OperationError(space.w_ValueError, - space.wrap("compile(): unrecognized flags")) - flag_names = [] - for name, value in compiler_features.items(): - if flags & value: - flag_names.append( name ) - return flag_names + +## from pypy.tool import stdlib___future__ +## compiler_flags = 0 +## compiler_features = {} +## for fname in stdlib___future__.all_feature_names: +## flag = getattr(stdlib___future__, fname).compiler_flag +## compiler_flags |= flag +## compiler_features[fname] = flag +## allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT + +## def get_flag_names(space, flags): +## if flags & ~allowed_flags: +## raise OperationError(space.w_ValueError, +## space.wrap("compile(): unrecognized flags")) +## flag_names = [] +## for name, value in compiler_features.items(): +## if flags & value: +## flag_names.append( name ) +## return flag_names class PyCodeCompiler(AbstractCompiler): """Base class for compilers producing PyCode objects.""" def getcodeflags(self, code): + """Return the __future__ compiler flags that were used to compile + the given code object.""" from pypy.interpreter.pycode import PyCode if isinstance(code, PyCode): - return code.co_flags & compiler_flags + return code.co_flags & self.compiler_flags else: return 0 +from pypy.interpreter.pyparser.future import futureFlags class CPythonCompiler(PyCodeCompiler): """Faked implementation of a compiler, using the underlying compile().""" + def __init__(self, space): + self.space = space + self.w_compile_hook = space.w_None + + self.compiler_flags = futureFlags.allowed_flags + def compile(self, source, filename, mode, flags): - flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) + from pypy.tool import stdlib___future__ + #flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) space = self.space try: old = self.setup_warn_explicit(warnings) @@ -191,6 +202,7 @@ ######## + class PythonAstCompiler(PyCodeCompiler): """Uses the stdlib's python implementation of compiler @@ -207,6 +219,8 @@ self.parser = make_pyparser(self.grammar_version) self.additional_rules = {} + self.compiler_flags = futureFlags.allowed_flags + def compile(self, source, filename, mode, flags): from pyparser.error import SyntaxError from pypy.interpreter import astcompiler @@ -218,13 +232,16 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.function import Function - flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) + from pypy.interpreter.pyparser.future import getFutures + +## flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) space = self.space try: builder = AstBuilder(self.parser, self.grammar_version, space=space) for rulename, buildfunc in self.additional_rules.iteritems(): assert isinstance(buildfunc, Function) builder.user_build_rules[rulename] = buildfunc + flags |= getFutures(source) self.parser.parse_source(source, mode, builder, flags) ast_tree = builder.rule_stack[-1] encoding = builder.source_encoding @@ -244,7 +261,7 @@ raise try: astcompiler.misc.set_filename(filename, ast_tree) - flag_names = get_flag_names(space, flags) + flag_names = futureFlags.get_flag_names(space, flags) if mode == 'exec': codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names) elif mode == 'single': @@ -257,7 +274,7 @@ e.wrap_info(space, filename)) except (ValueError, TypeError), e: raise OperationError(space.w_SystemError, space.wrap(str(e))) - assert isinstance(c,PyCode) + assert isinstance(c, PyCode) return c Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py Tue Nov 20 20:37:41 2007 @@ -4,13 +4,13 @@ helper functions are provided that use the grammar to parse using file_input, single_input and eval_input targets """ -import sys -import os -from pypy.interpreter.error import OperationError, debug_print +#import sys +#import os +#from pypy.interpreter.error import OperationError, debug_print from pypy.interpreter import gateway from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration -from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT +#from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT # XXX seems dead #import pypy.interpreter.pyparser.pysymbol as pysymbol import pypy.interpreter.pyparser.pytoken as pytoken @@ -18,8 +18,8 @@ from pypy.interpreter.pyparser.ebnflexer import GrammarSource from pypy.interpreter.pyparser.ebnfgrammar import GRAMMAR_GRAMMAR import pypy.interpreter.pyparser.grammar as grammar -from pypy.interpreter.pyparser.pythonutil import build_parser_for_version, build_parser - +from pypy.interpreter.pyparser.pythonutil import build_parser_for_version +#from pypy.interpreter.pyparser.pythonutil import build_parser # try: from pypy.interpreter.pyparser import symbol # except ImportError: From cfbolz at codespeak.net Tue Nov 20 20:48:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 20 Nov 2007 20:48:40 +0100 (CET) Subject: [pypy-svn] r48874 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c c/test llsupport Message-ID: <20071120194840.6FADD8138@code0.codespeak.net> Author: cfbolz Date: Tue Nov 20 20:48:40 2007 New Revision: 48874 Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py (contents, props changed) Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (cfbolz, rxe): start writing genc tests for the modwrapper Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/gc.py Tue Nov 20 20:48:40 2007 @@ -313,7 +313,7 @@ return '%s = %s; /* for moving GCs */' % (args[1], args[0]) def common_gcheader_definition(self): - return defnode.db.gctransformer.gc_fields() + return self.db.gctransformer.gc_fields() def common_gcheader_initdata(self, defnode): o = top_container(defnode.obj) Added: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py ============================================================================== --- (empty file) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py Tue Nov 20 20:48:40 2007 @@ -0,0 +1,104 @@ +import py +from pypy.translator.c.test.test_typed import CompilationTestCase + +class WrapperTests(object): + def test_return_none(self): + def fn(): + return None + fn = self.getcompiled(fn, []) + assert fn() is None + + def test_return_int(self): + def fn(): + return 1 + fn = self.getcompiled(fn, []) + assert fn() == 1 + + def test_return_float(self): + def fn(): + return 42.1 + fn = self.getcompiled(fn, []) + assert fn() == 42.1 + + def test_return_true(self): + def fn(): + return True + fn = self.getcompiled(fn, []) + assert fn() + + def test_return_string(self): + def fn(): + return "hello!" + fn = self.getcompiled(fn, []) + assert fn() == "hello!" + + def test_raises_builtin_exception(self): + def fn(): + raise ValueError + fn = self.getcompiled(fn, []) + py.test.raises(ValueError, fn) + + def test_raises_custom_exception(self): + from pypy.translator.llsupport import modwrapper + class MyException(Exception): + pass + def fn(): + raise MyException + fn = self.getcompiled(fn, []) + excinfo = py.test.raises(modwrapper.ExceptionWrapper, fn) + assert excinfo.value.class_name == "MyException" + + def test_return_list_of_strings(self): + def f(): + return ['abc', 'def'] + fn = self.getcompiled(f, []) + assert fn() == ['abc', 'def'] + + def test_return_list_of_bools(self): + def fn(): + return [True, True, False] + fn = self.getcompiled(fn, []) + assert fn() == [True, True, False] + + def test_return_tuple_of_list_of_strings(self): + def fn(): + return ['abc', 'def'], ['abc', 'def'] + fn = self.getcompiled(fn, []) + assert fn() == (['abc', 'def'], ['abc', 'def']) + + def test_argument_1int(self): + def fn(x): + return x + 42 + fn = self.getcompiled(fn, [int]) + assert fn(42) == 84 + assert fn(2) == 44 + + def test_argument_2int(self): + def fn(x, y): + return x + 42 * y + fn = self.getcompiled(fn, [int, int]) + assert fn(42, 0) == 42 + assert fn(2, 1) == 44 + assert fn(6, 2) == 90 + + + +class TestWrapperRefcounting(CompilationTestCase, WrapperTests): + pass + +from pypy.translator.c.test.test_boehm import AbstractGCTestClass + +class TestWrapperMarknSweep(AbstractGCTestClass, WrapperTests): + gcpolicy = "marksweep" + + +class TestWrapperBoehm(AbstractGCTestClass, WrapperTests): + gcpolicy = "boehm" + + +class TestWrapperSemispace(AbstractGCTestClass, WrapperTests): + gcpolicy = "semispace" + + +class TestWrapperGeneration(AbstractGCTestClass, WrapperTests): + gcpolicy = "generation" Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Tue Nov 20 20:48:40 2007 @@ -3,6 +3,7 @@ import py import ctypes from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import llmemory from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.rpython.lltypesystem.rstr import STR @@ -156,11 +157,12 @@ lltype.SingleFloat: "ctypes.c_float", lltype.Float: "ctypes.c_double", lltype.Char: "ctypes.c_char", - lltype.Signed: "ctypes.c_int", - lltype.Unsigned: "ctypes.c_uint", + lltype.Signed: "ctypes.c_long", + lltype.Unsigned: "ctypes.c_ulong", lltype.SignedLongLong: "ctypes.c_longlong", lltype.UnsignedLongLong: "ctypes.c_ulonglong", lltype.Void: None, + llmemory.Address: "ctypes.c_long", lltype.UniChar: "ctypes.c_uint", } @@ -302,9 +304,9 @@ "r_ulonglong": r_ulonglong, } if t == "exceptiontypename": - exc_class = getattr(exceptions, v) + exc_class = getattr(exceptions, v, None) if exc_class is None: - exc_class = ExceptionWrapper(v) + raise ExceptionWrapper(v) raise exc_class() if t == "tuple": return StructTuple([unwrap(element) for element in v]) From arigo at codespeak.net Tue Nov 20 21:04:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Nov 2007 21:04:38 +0100 (CET) Subject: [pypy-svn] r48875 - pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/test Message-ID: <20071120200438.9508B814A@code0.codespeak.net> Author: arigo Date: Tue Nov 20 21:04:37 2007 New Revision: 48875 Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/test/test_stackless.py Log: It's not really possible to test stack_too_big() directly. We have other tests in test_typed that checks that we get a RuntimeError in case of infinite recursion, so that will have to be good enough. The issue is that recursive functions automatically contain a stack_too_big() check, which might be triggered before our explicit one. Modified: pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/test/test_stackless.py ============================================================================== --- pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/test/test_stackless.py (original) +++ pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/test/test_stackless.py Tue Nov 20 21:04:37 2007 @@ -135,30 +135,6 @@ assert res == 10010255 - def test_stack_too_big(self): - def f1(): - return stack_too_big() - def f2(): - return lst[1]() - def f3(): - return lst[2]() - def f4(): - return lst[3]() - def f5(): - return lst[4]() - lst = [None,f1,f2,f3,f4,f5] - - def f(n): - if lst[5](): - return n - return f(n)+1 - - def fn(): - return f(0) - res = self.wrap_stackless_function(fn) - assert res > 500 - - def test_stack_unwind(self): def f(): stack_unwind() From antocuni at codespeak.net Tue Nov 20 21:14:31 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 21:14:31 +0100 (CET) Subject: [pypy-svn] r48876 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071120201431.462B68138@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 21:14:29 2007 New Revision: 48876 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: another (little) task Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Tue Nov 20 21:14:29 2007 @@ -65,6 +65,8 @@ - remove prebuilt costate vars (might happen before) + - think about the __name__ of MixedModules, because *.py makes CLR unhappy + Modules +++++++ From arigo at codespeak.net Tue Nov 20 21:15:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Nov 2007 21:15:18 +0100 (CET) Subject: [pypy-svn] r48877 - pypy/branch/kill-remaining-suggested-primitives/pypy/tool Message-ID: <20071120201518.202768139@code0.codespeak.net> Author: arigo Date: Tue Nov 20 21:15:15 2007 New Revision: 48877 Added: pypy/branch/kill-remaining-suggested-primitives/pypy/tool/udir.py - copied unchanged from r48875, pypy/dist/pypy/tool/udir.py Log: Revert unintended change. From arigo at codespeak.net Tue Nov 20 21:25:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Nov 2007 21:25:12 +0100 (CET) Subject: [pypy-svn] r48878 - in pypy: branch/kill-remaining-suggested-primitives dist/pypy/annotation dist/pypy/interpreter dist/pypy/objspace/flow dist/pypy/rlib dist/pypy/rlib/test dist/pypy/rpython dist/pypy/rpython/lltypesystem dist/pypy/rpython/module dist/pypy/rpython/ootypesystem dist/pypy/rpython/test dist/pypy/tool dist/pypy/translator dist/pypy/translator/c dist/pypy/translator/goal dist/pypy/translator/llvm dist/pypy/translator/stackless dist/pypy/translator/tool Message-ID: <20071120202512.413638139@code0.codespeak.net> Author: arigo Date: Tue Nov 20 21:25:11 2007 New Revision: 48878 Added: pypy/dist/pypy/annotation/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/annotation/ pypy/dist/pypy/interpreter/eval.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/interpreter/eval.py pypy/dist/pypy/objspace/flow/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/objspace/flow/ pypy/dist/pypy/rlib/rstack.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/rstack.py pypy/dist/pypy/rlib/test/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rlib/test/ pypy/dist/pypy/rpython/controllerentry.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/controllerentry.py pypy/dist/pypy/rpython/extfunctable.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/extfunctable.py pypy/dist/pypy/rpython/llinterp.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/ootypesystem/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/ootypesystem/ pypy/dist/pypy/rpython/rbuiltin.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/test/test_llann.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_llann.py pypy/dist/pypy/rpython/test/test_stack.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/rpython/test/test_stack.py pypy/dist/pypy/tool/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/tool/ pypy/dist/pypy/translator/c/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/c/ pypy/dist/pypy/translator/exceptiontransform.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/exceptiontransform.py pypy/dist/pypy/translator/geninterplevel.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/geninterplevel.py pypy/dist/pypy/translator/goal/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/goal/ pypy/dist/pypy/translator/llvm/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/llvm/ pypy/dist/pypy/translator/stackless/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/stackless/ pypy/dist/pypy/translator/tool/ - copied from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/tool/ pypy/dist/pypy/translator/transform.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/transform.py pypy/dist/pypy/translator/translator.py - copied unchanged from r48877, pypy/branch/kill-remaining-suggested-primitives/pypy/translator/translator.py Removed: pypy/branch/kill-remaining-suggested-primitives/ pypy/dist/pypy/rpython/module/ll_stack.py pypy/dist/pypy/rpython/module/ll_stackless.py Log: (alexander, armin) Removed the remaining suggested primitives - the stack and stackless manipulation. We had to resolve circular imports by removing "import objspace" from pypy/objspace/flow/__init__.py. This required some other imports to be fixed. Improved the controllerentry logic for rstack.py. In translator/c/test/: it's not really possible to test stack_too_big() directly. We have other tests in test_typed that checks that we get a RuntimeError in case of infinite recursion, so that will have to be good enough. The issue is that recursive functions automatically contain a stack_too_big() check, which might be triggered before our explicit one. From ac at codespeak.net Tue Nov 20 22:03:49 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Tue, 20 Nov 2007 22:03:49 +0100 (CET) Subject: [pypy-svn] r48879 - in pypy/branch/dist-future-fixing/pypy/interpreter: pyparser pyparser/test test Message-ID: <20071120210349.8B349814A@code0.codespeak.net> Author: ac Date: Tue Nov 20 22:03:48 2007 New Revision: 48879 Removed: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_astbuilder_future.py Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnflexer.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py pypy/branch/dist-future-fixing/pypy/interpreter/test/test_compiler.py Log: (jacob, arre): More work on making from __future__ support more sane. Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py Tue Nov 20 22:03:48 2007 @@ -764,7 +764,6 @@ else_ = atoms[6] builder.push(ast.While(test, body, else_, atoms[0].lineno)) - def build_with_stmt(builder, nb): """with_stmt: 'with' test [ NAME expr ] ':' suite""" @@ -783,7 +782,6 @@ body = atoms[5] builder.push(ast.With(test, body, var, atoms[0].lineno)) - def build_import_name(builder, nb): """import_name: 'import' dotted_as_names @@ -838,7 +836,6 @@ import_as_name: NAME [NAME NAME] """ atoms = get_atoms(builder, nb) - index = 1 incr, from_name = parse_dotted_names(atoms[index:], builder) index += (incr + 1) # skip 'import' @@ -882,42 +879,6 @@ builder.push(ast.From(from_name, names, atoms[0].lineno)) -def build_future_import_feature(builder, nb): - """ - future_import_feature: NAME [('as'|NAME) NAME] - - Enables python language future imports. Called once per feature imported, - no matter how you got to this one particular feature. - """ - - atoms = peek_atoms(builder, nb) - - feature_name = atoms[0].get_value() - assert type(feature_name) is str - space = builder.space - feature_code = space.unwrap(space.appexec([space.wrap(feature_name)], - """(feature): - import __future__ as f - feature = getattr(f, feature, None) - return feature and feature.compiler_flag or 0 - """)) - - # We will call a method on the parser (the method exists only in unit - # tests). - if feature_code == consts.CO_FUTURE_WITH_STATEMENT: - rules = """ - compound_stmt: (if_stmt | while_stmt | for_stmt | try_stmt | - funcdef | classdef | with_stmt) - with_stmt: 'with' test [ 'as' expr ] ':' suite - """ - builder.insert_grammar_rule(rules, { - 'with_stmt': build_with_stmt}) - - # We need to keep the rule on the stack so we can share atoms - # with a later rule - return True - - def build_yield_stmt(builder, nb): atoms = get_atoms(builder, nb) builder.push(ast.Yield(atoms[1], atoms[0].lineno)) @@ -1089,6 +1050,7 @@ 'exprlist' : build_exprlist, 'decorator' : build_decorator, 'eval_input' : build_eval_input, + 'with_stmt' : build_with_stmt, } @@ -1110,11 +1072,6 @@ ## self.with_enabled = False self.build_rules = ASTRULES_Template self.user_build_rules = {} - if grammar_version >= "2.5": - self.build_rules.update({ - 'future_import_feature': build_future_import_feature, - 'import_from_future': build_import_from, - }) ## def enable_with(self): ## if self.with_enabled: @@ -1157,7 +1114,9 @@ self.push(astnode) else: builder_func = self.build_rules.get(rulename, None) - if not builder_func or builder_func(self, 1): + if builder_func: + builder_func(self, 1) + else: self.push_rule(rule.codename, 1, source) else: self.push_rule(rule.codename, 1, source) @@ -1177,7 +1136,9 @@ self.push(astnode) else: builder_func = self.build_rules.get(rulename, None) - if not builder_func or builder_func(self, elts_number): + if builder_func: + builder_func(self, elts_number) + else: self.push_rule(rule.codename, elts_number, source) else: self.push_rule(rule.codename, elts_number, source) @@ -1228,14 +1189,6 @@ else: return None - def insert_grammar_rule(self, rule, buildfuncs): - """Inserts new grammar rules for the builder - This allows to change the rules during the parsing - """ - self.parser.insert_rule(rule) - self.build_rules.update(buildfuncs) - - def show_stack(before, after): """debugging helper function""" size1 = len(before) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfgrammar.py Tue Nov 20 22:03:48 2007 @@ -27,7 +27,7 @@ group: '(' alternative ')' star? """ p = GRAMMAR_GRAMMAR - p.add_token(Token('EOF','EOF')) + p.add_token(Token('EOF')) # star: '*' | '+' star = p.Alternative_n( "star", [p.Token_n('TOK_STAR', '*'), p.Token_n('TOK_ADD', '+')] ) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnflexer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnflexer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnflexer.py Tue Nov 20 22:03:48 2007 @@ -136,7 +136,7 @@ end = len(self.input) pos = self.skip_empty_lines(inp,pos,end) if pos==end: - return Token(_p, _p.EOF, None) + return Token(_p.EOF, None) # at this point nextchar is not a white space nor \n nextchr = inp[pos] @@ -148,22 +148,22 @@ self.pos = npos _endpos = npos - 1 assert _endpos>=0 - return Token(_p, _p.TOK_STRING, inp[pos+1:_endpos]) + return Token(_p.TOK_STRING, inp[pos+1:_endpos]) else: npos = match_symbol( inp, pos, end) if npos!=pos: self.pos = npos if npos!=end and inp[npos]==":": self.pos += 1 - return Token(_p, _p.TOK_SYMDEF, inp[pos:npos]) + return Token(_p.TOK_SYMDEF, inp[pos:npos]) else: - return Token(_p, _p.TOK_SYMBOL, inp[pos:npos]) + return Token(_p.TOK_SYMBOL, inp[pos:npos]) # we still have pos!=end here chr = inp[pos] if chr in "[]()*+|": self.pos = pos+1 - return Token(_p, _p.tok_values[chr], chr) + return Token(_p.tok_values[chr], chr) self.RaiseError( "Unknown token" ) def peek(self): Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py Tue Nov 20 22:03:48 2007 @@ -41,7 +41,7 @@ class NameToken(Token): """A token that is not a keyword""" def __init__(self, parser, keywords=None): - Token.__init__(self, parser, parser.tokens['NAME']) + Token.__init__(self, parser.tokens['NAME']) self.keywords = keywords def match(self, source, builder, level=0): @@ -105,7 +105,7 @@ self.current_rule_name = "" self.tokens = {} self.keywords = [] - NAME = dest_parser.add_token(Token(dest_parser, 'NAME')) + NAME = dest_parser.add_token(Token('NAME')) # NAME = dest_parser.tokens['NAME'] self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords) @@ -158,7 +158,7 @@ """Returns a new or existing Token""" if codename in self.tokens: return self.tokens[codename] - token = self.tokens[codename] = Token(self.parser, codename, None) + token = self.tokens[codename] = Token(codename, None) return token def get_symbolcode(self, name): @@ -273,12 +273,12 @@ if value in self.parser.tok_values: # punctuation tokencode = self.parser.tok_values[value] - tok = Token(self.parser, tokencode, None) + tok = Token(tokencode, None) else: if not is_py_name(value): raise RuntimeError("Unknown STRING value ('%s')" % value) # assume a keyword - tok = Token(self.parser, self.parser.tokens['NAME'], value) + tok = Token(self.parser.tokens['NAME'], value) if value not in self.keywords: self.keywords.append(value) self.rule_stack.append(tok) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py Tue Nov 20 22:03:48 2007 @@ -276,7 +276,7 @@ # and that we have a hacked __future__ module. from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=False) -if config.objspace.pyversion == '2.4': +if config.objspace.pyversion == '2.4' and False: futureFlags = FutureFlags((2, 4, 4, 'final', 0)) else: futureFlags = FutureFlags((2, 5, 0, 'final', 0)) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py Tue Nov 20 22:03:48 2007 @@ -655,9 +655,8 @@ class Token(GrammarElement): """Represents a Token in a grammar rule (a lexer token)""" - def __init__(self, parser, codename, value=None): + def __init__(self, codename, value=None): GrammarElement.__init__(self, codename) - self.parser = parser self.value = value self.first_set = [self] # self.first_set = {self: 1} @@ -693,11 +692,10 @@ return 0 def display(self, level=0): - name = self.parser.symbol_repr( self.codename ) if self.value is None: - return "<%s>" % name + return "<%s>" % self.codename else: - return "<%s>=='%s'" % (name, self.value) + return "<%s>=='%s'" % (self.codename, self.value) def match_token(self, builder, other): """convenience '==' implementation, this is *not* a *real* equality test @@ -738,7 +736,7 @@ return False -EmptyToken = Token(None, -1, None) +EmptyToken = Token(-1, None) class Parser(object): def __init__(self): @@ -875,9 +873,9 @@ # XXX What is the significance of the name_id? Needs to be found # out for full refactoring of this code. - t = Token(self, name, value) + t = Token(name, value) name_id = self.add_token(t) - return Token(self, name_id, value) + return Token(name_id, value) # Debugging functions def show_rules(self, name): Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py Tue Nov 20 22:03:48 2007 @@ -121,7 +121,7 @@ endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch - tok = Token(parser, parser.tokens['STRING'], contstr + line[:end]) + tok = Token(parser.tokens['STRING'], contstr + line[:end]) token_list.append((tok, line, lnum, pos)) last_comment = '' # token_list.append((STRING, contstr + line[:end], @@ -130,7 +130,7 @@ contline = None elif (needcont and not line.endswith('\\\n') and not line.endswith('\\\r\n')): - tok = Token(parser, parser.tokens['ERRORTOKEN'], contstr + line) + tok = Token(parser.tokens['ERRORTOKEN'], contstr + line) token_list.append((tok, line, lnum, pos)) last_comment = '' # token_list.append((ERRORTOKEN, contstr + line, @@ -161,10 +161,10 @@ if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': - tok = Token(parser, parser.tokens['COMMENT'], line[pos:]) + tok = Token(parser.tokens['COMMENT'], line[pos:]) last_comment = line[pos:] else: - tok = Token(parser, parser.tokens['NL'], line[pos:]) + tok = Token(parser.tokens['NL'], line[pos:]) last_comment = '' # XXX Skip NL and COMMENT Tokens # token_list.append((tok, line, lnum, pos)) @@ -172,12 +172,12 @@ if column > indents[-1]: # count indents or dedents indents.append(column) - tok = Token(parser, parser.tokens['INDENT'], line[:pos]) + tok = Token(parser.tokens['INDENT'], line[:pos]) token_list.append((tok, line, lnum, pos)) last_comment = '' while column < indents[-1]: indents = indents[:-1] - tok = Token(parser, parser.tokens['DEDENT'], '') + tok = Token(parser.tokens['DEDENT'], '') token_list.append((tok, line, lnum, pos)) last_comment = '' else: # continued statement @@ -204,22 +204,22 @@ token, initial = line[start:end], line[start] if initial in numchars or \ (initial == '.' and token != '.'): # ordinary number - tok = Token(parser, parser.tokens['NUMBER'], token) + tok = Token(parser.tokens['NUMBER'], token) token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial in '\r\n': if parenlev > 0: - tok = Token(parser, parser.tokens['NL'], token) + tok = Token(parser.tokens['NL'], token) last_comment = '' # XXX Skip NL else: - tok = Token(parser, parser.tokens['NEWLINE'], token) + tok = Token(parser.tokens['NEWLINE'], token) # XXX YUCK ! tok.value = last_comment token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial == '#': - tok = Token(parser, parser.tokens['COMMENT'], token) + tok = Token(parser.tokens['COMMENT'], token) last_comment = token # XXX Skip # token_list.append((tok, line, lnum, pos)) # token_list.append((COMMENT, token, spos, epos, line)) @@ -229,7 +229,7 @@ if endmatch >= 0: # all on one line pos = endmatch token = line[start:pos] - tok = Token(parser, parser.tokens['STRING'], token) + tok = Token(parser.tokens['STRING'], token) token_list.append((tok, line, lnum, pos)) last_comment = '' else: @@ -246,11 +246,11 @@ contline = line break else: # ordinary string - tok = Token(parser, parser.tokens['STRING'], token) + tok = Token(parser.tokens['STRING'], token) token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial in namechars: # ordinary name - tok = Token(parser, parser.tokens['NAME'], token) + tok = Token(parser.tokens['NAME'], token) token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial == '\\': # continued stmt @@ -266,9 +266,9 @@ (lnum-1, 0), token_list) if token in parser.tok_values: punct = parser.tok_values[token] - tok = Token(parser, punct, None) + tok = Token(punct, None) else: - tok = Token(parser, parser.tokens['OP'], token) + tok = Token(parser.tokens['OP'], token) token_list.append((tok, line, lnum, pos)) last_comment = '' else: @@ -279,7 +279,7 @@ raise TokenError( 'EOL while scanning single-quoted string', line, x(lnum, start), token_list) - tok = Token(parser, parser.tokens['ERRORTOKEN'], line[pos]) + tok = Token(parser.tokens['ERRORTOKEN'], line[pos]) token_list.append((tok, line, lnum, pos)) last_comment = '' pos = pos + 1 @@ -289,17 +289,17 @@ if (token_list and token_list[-1][0].codename != parser.tokens['NEWLINE']): token_list.append( - (Token(parser, parser.tokens['NEWLINE'], ''), '\n', lnum, 0)) + (Token(parser.tokens['NEWLINE'], ''), '\n', lnum, 0)) for indent in indents[1:]: # pop remaining indent levels - tok = Token(parser, parser.tokens['DEDENT'], '') + tok = Token(parser.tokens['DEDENT'], '') token_list.append((tok, line, lnum, pos)) #if token_list and token_list[-1][0].codename != pytoken.NEWLINE: token_list.append( - (Token(parser, parser.tokens['NEWLINE'], ''), '\n', lnum, 0)) + (Token(parser.tokens['NEWLINE'], ''), '\n', lnum, 0)) token_list.append( - (Token(parser, parser.tokens['ENDMARKER'], '',), line, lnum, pos)) + (Token(parser.tokens['ENDMARKER'], '',), line, lnum, pos)) #for t in token_list: # print '%20s %-25s %d' % (pytoken.tok_name.get(t[0].codename, '?'), t[0], t[-2]) #print '----------------------------------------- pyparser/pythonlexer.py' Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pytoken.py Tue Nov 20 22:03:48 2007 @@ -17,62 +17,62 @@ # global tok_rpunct # For compatibility, this produces the same constant values as Python 2.4. from grammar import Token - parser.add_token(Token(parser, 'ENDMARKER' )) - parser.add_token(Token(parser, 'NAME' )) - parser.add_token(Token(parser, 'NUMBER' )) - parser.add_token(Token(parser, 'STRING' )) - parser.add_token(Token(parser, 'NEWLINE' )) - parser.add_token(Token(parser, 'INDENT' )) - parser.add_token(Token(parser, 'DEDENT' )) - parser.add_token(Token(parser, 'LPAR', "(" )) - parser.add_token(Token(parser, 'RPAR', ")" )) - parser.add_token(Token(parser, 'LSQB', "[" )) - parser.add_token(Token(parser, 'RSQB', "]" )) - parser.add_token(Token(parser, 'COLON', ":" )) - parser.add_token(Token(parser, 'COMMA', "," )) - parser.add_token(Token(parser, 'SEMI', ";" )) - parser.add_token(Token(parser, 'PLUS', "+" )) - parser.add_token(Token(parser, 'MINUS', "-" )) - parser.add_token(Token(parser, 'STAR', "*" )) - parser.add_token(Token(parser, 'SLASH', "/" )) - parser.add_token(Token(parser, 'VBAR', "|" )) - parser.add_token(Token(parser, 'AMPER', "&" )) - parser.add_token(Token(parser, 'LESS', "<" )) - parser.add_token(Token(parser, 'GREATER', ">" )) - parser.add_token(Token(parser, 'EQUAL', "=" )) - parser.add_token(Token(parser, 'DOT', "." )) - parser.add_token(Token(parser, 'PERCENT', "%" )) - parser.add_token(Token(parser, 'BACKQUOTE', "`" )) - parser.add_token(Token(parser, 'LBRACE', "{" )) - parser.add_token(Token(parser, 'RBRACE', "}" )) - parser.add_token(Token(parser, 'EQEQUAL', "==" )) - ne = parser.add_token(Token(parser, 'NOTEQUAL', "!=" )) + parser.add_token(Token('ENDMARKER' )) + parser.add_token(Token('NAME' )) + parser.add_token(Token('NUMBER' )) + parser.add_token(Token('STRING' )) + parser.add_token(Token('NEWLINE' )) + parser.add_token(Token('INDENT' )) + parser.add_token(Token('DEDENT' )) + parser.add_token(Token('LPAR', "(" )) + parser.add_token(Token('RPAR', ")" )) + parser.add_token(Token('LSQB', "[" )) + parser.add_token(Token('RSQB', "]" )) + parser.add_token(Token('COLON', ":" )) + parser.add_token(Token('COMMA', "," )) + parser.add_token(Token('SEMI', ";" )) + parser.add_token(Token('PLUS', "+" )) + parser.add_token(Token('MINUS', "-" )) + parser.add_token(Token('STAR', "*" )) + parser.add_token(Token('SLASH', "/" )) + parser.add_token(Token('VBAR', "|" )) + parser.add_token(Token('AMPER', "&" )) + parser.add_token(Token('LESS', "<" )) + parser.add_token(Token('GREATER', ">" )) + parser.add_token(Token('EQUAL', "=" )) + parser.add_token(Token('DOT', "." )) + parser.add_token(Token('PERCENT', "%" )) + parser.add_token(Token('BACKQUOTE', "`" )) + parser.add_token(Token('LBRACE', "{" )) + parser.add_token(Token('RBRACE', "}" )) + parser.add_token(Token('EQEQUAL', "==" )) + ne = parser.add_token(Token('NOTEQUAL', "!=" )) parser.tok_values["<>"] = ne - parser.add_token(Token(parser, 'LESSEQUAL', "<=" )) - parser.add_token(Token(parser, 'GREATEREQUAL', ">=" )) - parser.add_token(Token(parser, 'TILDE', "~" )) - parser.add_token(Token(parser, 'CIRCUMFLEX', "^" )) - parser.add_token(Token(parser, 'LEFTSHIFT', "<<" )) - parser.add_token(Token(parser, 'RIGHTSHIFT', ">>" )) - parser.add_token(Token(parser, 'DOUBLESTAR', "**" )) - parser.add_token(Token(parser, 'PLUSEQUAL', "+=" )) - parser.add_token(Token(parser, 'MINEQUAL', "-=" )) - parser.add_token(Token(parser, 'STAREQUAL', "*=" )) - parser.add_token(Token(parser, 'SLASHEQUAL', "/=" )) - parser.add_token(Token(parser, 'PERCENTEQUAL', "%=" )) - parser.add_token(Token(parser, 'AMPEREQUAL', "&=" )) - parser.add_token(Token(parser, 'VBAREQUAL', "|=" )) - parser.add_token(Token(parser, 'CIRCUMFLEXEQUAL', "^=" )) - parser.add_token(Token(parser, 'LEFTSHIFTEQUAL', "<<=" )) - parser.add_token(Token(parser, 'RIGHTSHIFTEQUAL', ">>=" )) - parser.add_token(Token(parser, 'DOUBLESTAREQUAL', "**=" )) - parser.add_token(Token(parser, 'DOUBLESLASH', "//" )) - parser.add_token(Token(parser, 'DOUBLESLASHEQUAL',"//=" )) - parser.add_token(Token(parser, 'AT', "@" )) - parser.add_token(Token(parser, 'OP' )) - parser.add_token(Token(parser, 'ERRORTOKEN' )) + parser.add_token(Token('LESSEQUAL', "<=" )) + parser.add_token(Token('GREATEREQUAL', ">=" )) + parser.add_token(Token('TILDE', "~" )) + parser.add_token(Token('CIRCUMFLEX', "^" )) + parser.add_token(Token('LEFTSHIFT', "<<" )) + parser.add_token(Token('RIGHTSHIFT', ">>" )) + parser.add_token(Token('DOUBLESTAR', "**" )) + parser.add_token(Token('PLUSEQUAL', "+=" )) + parser.add_token(Token('MINEQUAL', "-=" )) + parser.add_token(Token('STAREQUAL', "*=" )) + parser.add_token(Token('SLASHEQUAL', "/=" )) + parser.add_token(Token('PERCENTEQUAL', "%=" )) + parser.add_token(Token('AMPEREQUAL', "&=" )) + parser.add_token(Token('VBAREQUAL', "|=" )) + parser.add_token(Token('CIRCUMFLEXEQUAL', "^=" )) + parser.add_token(Token('LEFTSHIFTEQUAL', "<<=" )) + parser.add_token(Token('RIGHTSHIFTEQUAL', ">>=" )) + parser.add_token(Token('DOUBLESTAREQUAL', "**=" )) + parser.add_token(Token('DOUBLESLASH', "//" )) + parser.add_token(Token('DOUBLESLASHEQUAL',"//=" )) + parser.add_token(Token('AT', "@" )) + parser.add_token(Token('OP' )) + parser.add_token(Token('ERRORTOKEN' )) # extra PyPy-specific tokens - parser.add_token(Token(parser, "COMMENT" )) - parser.add_token(Token(parser, "NL" )) + parser.add_token(Token("COMMENT" )) + parser.add_token(Token("NL" )) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_lookahead.py Tue Nov 20 22:03:48 2007 @@ -86,7 +86,7 @@ p = self.parser LOW = p.tokens['LOW'] CAP = p.tokens['CAP'] - for s in [Token(p, LOW, 'low'), EmptyToken, Token(p, CAP, 'cap')]: + for s in [Token(LOW, 'low'), EmptyToken, Token(CAP, 'cap')]: assert s in self.A.first_set assert s in self.B.first_set assert s in self.C.first_set Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_parser.py Tue Nov 20 22:03:48 2007 @@ -7,7 +7,7 @@ def test_symbols(): p = Parser() x1 = p.add_symbol('sym') - x2 = p.add_token(Token(p, 'tok')) + x2 = p.add_token(Token('tok')) x3 = p.add_anon_symbol(':sym') x4 = p.add_anon_symbol(':sym1') # test basic numbering assumption @@ -20,7 +20,7 @@ assert x3 < 0 y1 = p.add_symbol('sym') assert y1 == x1 - y2 = p.add_token(Token(p, 'tok')) + y2 = p.add_token(Token('tok')) assert y2 == x2 y3 = p.add_symbol(':sym') assert y3 == x3 @@ -50,11 +50,7 @@ def __init__(self, *args, **kw): self.trace = [] - self.exclude_rules = [ - 'dotted_name', 'dotted_as_name', 'dotted_as_names', - 'import_stmt', 'small_stmt', 'simple_stmt', 'stmt', - 'single_input', 'file_input', 'future_import_list', - 'import_from_future', 'future_import_as_names'] + self.exclude_rules = [] def __getitem__(self, attr): if attr in self.exclude_rules: @@ -76,7 +72,7 @@ self.build_rules = RuleTracer() -class TestFuture(object): +class XTestFuture(object): _grammar_ver = '2.5a' Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py Tue Nov 20 22:03:48 2007 @@ -20,7 +20,7 @@ """returns list of parsed tokens""" lexer = Source( P, source.splitlines(True)) tokens = [] - last_token = Token( P, NULLTOKEN, None) + last_token = Token(NULLTOKEN, None) while last_token.codename != ENDMARKER: last_token = lexer.next() tokens.append(last_token) @@ -58,24 +58,24 @@ s = """['a' ]""" tokens = parse_source(s) - assert tokens[:4] == [Token(P, LSQB, None), Token(P, STRING, "'a'"), - Token(P, RSQB, None), Token(P, NEWLINE, '')] + assert tokens[:4] == [Token(LSQB, None), Token(STRING, "'a'"), + Token(RSQB, None), Token(NEWLINE, '')] def test_numbers(): """make sure all kind of numbers are correctly parsed""" for number in NUMBERS: - assert parse_source(number)[0] == Token(P, NUMBER, number) + assert parse_source(number)[0] == Token(NUMBER, number) neg = '-%s' % number - assert parse_source(neg)[:2] == [Token(P, MINUS, None), - Token(P, NUMBER, number)] + assert parse_source(neg)[:2] == [Token(MINUS, None), + Token(NUMBER, number)] for number in BAD_NUMBERS: - assert parse_source(number)[0] != Token(P, NUMBER, number) + assert parse_source(number)[0] != Token(NUMBER, number) def test_hex_number(): """basic pasrse""" tokens = parse_source("a = 0x12L") - assert tokens[:4] == [Token(P, NAME, 'a'), Token(P, EQUAL, None), - Token(P, NUMBER, '0x12L'), Token(P, NEWLINE, '')] + assert tokens[:4] == [Token(NAME, 'a'), Token(EQUAL, None), + Token(NUMBER, '0x12L'), Token(NEWLINE, '')] def test_punct(): """make sure each punctuation is correctly parsed""" Modified: pypy/branch/dist-future-fixing/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/test/test_compiler.py Tue Nov 20 22:03:48 2007 @@ -588,7 +588,7 @@ source2 = "with = 3" - code = self.compiler.compile(source, '', 'exec', 0) + code = self.compiler.compile(source2, '', 'exec', 0) assert isinstance(code, PyCode) assert code.co_filename == '' From antocuni at codespeak.net Tue Nov 20 22:21:45 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 20 Nov 2007 22:21:45 +0100 (CET) Subject: [pypy-svn] r48880 - in pypy/dist/pypy/translator: . cli Message-ID: <20071120212145.766A28110@code0.codespeak.net> Author: antocuni Date: Tue Nov 20 22:21:44 2007 New Revision: 48880 Modified: pypy/dist/pypy/translator/cli/record.py pypy/dist/pypy/translator/driver.py Log: automatically use ledit to start pypy-{cli,jvm} if available Modified: pypy/dist/pypy/translator/cli/record.py ============================================================================== --- pypy/dist/pypy/translator/cli/record.py (original) +++ pypy/dist/pypy/translator/cli/record.py Tue Nov 20 22:21:44 2007 @@ -100,21 +100,25 @@ self.ilasm.end_function() def _getHashCode(self): - # return the hash of the first field. XXX: it can lead to a bad distribution record_type = self.cts.lltype_to_cts(self.record) self.ilasm.begin_function('GetHashCode', [], 'int32', False, 'virtual', 'instance', 'default') gethash = 'int32 [pypylib]pypy.runtime.Utils::GetHashCode<%s>(!!0)' + + self.ilasm.opcode('ldc.i4.0') # initial hash if self.record._fields: - f_name, (FIELD_TYPE, default) = self.record._fields.iteritems().next() - if FIELD_TYPE is ootype.Void: - self.ilasm.opcode('ldc.i4.0') - else: - f_name = self.cts.escape_name(f_name) - f_type = self.cts.lltype_to_cts(FIELD_TYPE) - self.ilasm.opcode('ldarg.0') - self.ilasm.get_field((f_type, record_type.classname(), f_name)) - self.ilasm.call(gethash % f_type) - else: - self.ilasm.opcode('ldc.i4.0') + for f_name, (FIELD_TYPE, default) in self.record._fields.iteritems(): + if FIELD_TYPE is ootype.Void: + continue + else: + # compute the hash for this field + f_name = self.cts.escape_name(f_name) + f_type = self.cts.lltype_to_cts(FIELD_TYPE) + self.ilasm.opcode('ldarg.0') + self.ilasm.get_field((f_type, record_type.classname(), f_name)) + self.ilasm.call(gethash % f_type) + + # xor with the previous value + self.ilasm.opcode('xor') + self.ilasm.opcode('ret') self.ilasm.end_function() Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Nov 20 22:21:44 2007 @@ -665,8 +665,9 @@ newexename = basename f = file(newexename, 'w') f.write("""#!/bin/bash +LEDIT=`which ledit 2>/dev/null` if [ `uname -o` = 'Cygwin' ]; then MONO=; else MONO=mono; fi -$MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH +$LEDIT $MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() os.chmod(newexename, 0755) @@ -734,7 +735,8 @@ newexename = basename f = file(newexename, 'w') f.write("""#!/bin/bash -java -jar $0.jar "$@" +LEDIT=`which ledit 2>/dev/null` +$LEDIT java -jar $0.jar "$@" """) f.close() os.chmod(newexename, 0755) From xoraxax at codespeak.net Wed Nov 21 01:13:28 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 21 Nov 2007 01:13:28 +0100 (CET) Subject: [pypy-svn] r48883 - pypy/dist/pypy/translator/c/src Message-ID: <20071121001328.AADCA8107@code0.codespeak.net> Author: xoraxax Date: Wed Nov 21 01:13:27 2007 New Revision: 48883 Modified: pypy/dist/pypy/translator/c/src/stack.h Log: Remove LL_stack_unwind from stack.h. Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Wed Nov 21 01:13:27 2007 @@ -20,11 +20,6 @@ # endif #endif -void LL_stack_unwind(void) -{ - RPyRaiseSimpleException(PyExc_RuntimeError, "Recursion limit exceeded"); -} - long PYPY_NOINLINE _LL_stack_growing_direction(char *parent) { char local; From cfbolz at codespeak.net Wed Nov 21 10:20:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 10:20:35 +0100 (CET) Subject: [pypy-svn] r48884 - in pypy/branch/ropes-unicode/pypy/objspace/std: . test Message-ID: <20071121092035.4E4B180ED@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 10:20:34 2007 New Revision: 48884 Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Log: fix rope translation failures. add tests for the code that tries to quickly encode/decode ropes. Modified: pypy/branch/ropes-unicode/pypy/objspace/std/rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/rope.py Wed Nov 21 10:20:34 2007 @@ -233,7 +233,7 @@ def getrope(self, index): ch = ord(self.u[index]) if ch < 256: - return LiteralStringNode.PREBUILT[ord(self.s[index])] + return LiteralStringNode.PREBUILT[ch] if len(self.u) == 1: return self return LiteralUnicodeNode(unichr(ch)) @@ -606,7 +606,7 @@ unichunk.append(unichr(c)) i += 1 if unichunk: - nodelist.append(LiteralUnicodeNode("".join(unichunk))) + nodelist.append(LiteralUnicodeNode(u"".join(unichunk))) strchunk = [] while i < length: c = ord(charlist[i]) @@ -1379,8 +1379,8 @@ str_decode_utf8(rope.right)) elif isinstance(rope, LiteralStringNode): try: - result, consumed = str_decode_utf_8(rope.s, len(rope.s), False, - "strict") + result, consumed = str_decode_utf_8(rope.s, len(rope.s), "strict", + False) except UnicodeDecodeError: return None if consumed < len(rope.s): @@ -1388,7 +1388,7 @@ return rope_from_unicode(result) s = rope.flatten_string() try: - result, consumed = str_decode_utf_8(s, len(s), True) + result, consumed = str_decode_utf_8(s, len(s), "strict", True) return rope_from_unicode(result) except UnicodeDecodeError: pass @@ -1410,11 +1410,8 @@ return BinaryConcatNode(unicode_encode_utf8(rope.left), unicode_encode_utf8(rope.right)) elif isinstance(rope, LiteralUnicodeNode): - try: - return LiteralStringNode( - unicode_encode_utf_8(rope.u, len(rope.u), "strict")) - except UnicodeDecodeError: - return None + return LiteralStringNode( + unicode_encode_utf_8(rope.u, len(rope.u), "strict")) elif isinstance(rope, LiteralStringNode): return LiteralStringNode(_str_encode_utf_8(rope.s)) @@ -1428,7 +1425,8 @@ if (ch < 0x80): # Encode ASCII result.append(chr(ch)) + continue # Encode Latin-1 result.append(chr((0xc0 | (ch >> 6)))) result.append(chr((0x80 | (ch & 0x3f)))) - return LiteralStringNode("".join(s)) + return "".join(result) Modified: pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Wed Nov 21 10:20:34 2007 @@ -39,7 +39,9 @@ result = rope.str_decode_utf8(node) if result is not None: return W_RopeUnicodeObject(result) - return unicode_from_encoded_object(space, w_str, encoding, errors) + result = unicode_from_encoded_object(space, w_str, encoding, errors) + assert isinstance(result, W_RopeUnicodeObject) + return result def encode_unicode(space, w_unistr, encoding, errors): from pypy.objspace.std.unicodetype import getdefaultencoding, \ Modified: pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py (original) +++ pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py Wed Nov 21 10:20:34 2007 @@ -770,3 +770,69 @@ assert node.length() == 90 assert not node.is_ascii() assert not node.is_bytestring() + +def test_encode(): + node = LiteralStringNode("abc") + assert unicode_encode_ascii(node) is node + assert unicode_encode_latin1(node) is node + assert unicode_encode_utf8(node) is node + node = LiteralStringNode("abc\xff") + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is node + assert unicode_encode_utf8(node).s == 'abc\xc3\xbf' + node = LiteralUnicodeNode(u"\uffffab") + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is None + assert unicode_encode_utf8(node).s == '\xef\xbf\xbfab' + node = BinaryConcatNode(LiteralStringNode("abc"), + LiteralUnicodeNode(u"\uffffab")) + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is None + res = unicode_encode_utf8(node) + assert res.left is node.left + assert res.right.s == '\xef\xbf\xbfab' + +def test_decode(): + node = LiteralStringNode("abc") + assert str_decode_ascii(node) is node + assert str_decode_latin1(node) is node + assert str_decode_utf8(node) is node + node = LiteralStringNode("abc\xff") + assert str_decode_ascii(node) is None + assert str_decode_latin1(node) is node + +def test_decode_utf8(): + # bad data + node = LiteralStringNode("\xd7\x50") + assert str_decode_utf8(node) is None + node = LiteralStringNode("\xf0\x90\x91") + assert str_decode_utf8(node) is None + + # correct data in one node + node = LiteralStringNode('\xef\xbf\xbfab') + assert str_decode_utf8(node).u == u"\uffffab" + + # binary node, left node can be decoded + node = BinaryConcatNode(LiteralStringNode('\xef\xbf\xbfab'), + LiteralStringNode('\xef\xbf\xbfab')) + res = str_decode_utf8(node) + assert res.left.u == u"\uffffab" + assert res.right.u == u"\uffffab" + + # binary node, left node alone cannot be decoded + node = BinaryConcatNode(LiteralStringNode('\xef'), + LiteralStringNode('\xbf\xbfab')) + res = str_decode_utf8(node) + assert res.u == u"\uffffab" + + # binary node, left node cannot be decoded, bad data + node = BinaryConcatNode(LiteralStringNode("\xf0\x90"), + LiteralStringNode("\x91")) + assert str_decode_utf8(node) is None + + # binary node, incomplete data + node = BinaryConcatNode(LiteralStringNode('ab\xef'), + LiteralStringNode('\xbf')) + + res = str_decode_utf8(node) + assert res is None From cfbolz at codespeak.net Wed Nov 21 10:22:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 10:22:35 +0100 (CET) Subject: [pypy-svn] r48885 - in pypy/dist/pypy/module/_codecs: . test Message-ID: <20071121092235.AEA8F811B@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 10:22:35 2007 New Revision: 48885 Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py pypy/dist/pypy/module/_codecs/test/test_codecs.py Log: make the codecs module not segfault anymore Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/interp_codecs.py Wed Nov 21 10:22:35 2007 @@ -131,7 +131,7 @@ lookup_error.unwrap_spec = [ObjSpace, str] -def encode(space, w_obj, encoding=NoneNotWrapped, errors='strict'): +def encode(space, w_obj, w_encoding=NoneNotWrapped, errors='strict'): """encode(obj, [encoding[,errors]]) -> object Encodes obj using the codec registered for encoding. encoding defaults @@ -142,17 +142,19 @@ codecs.register_error that can handle ValueErrors. """ #import pdb; pdb.set_trace() - if encoding is None: + if w_encoding is None: encoding = space.sys.defaultencoding + else: + encoding = space.str_w(w_encoding) w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0)) if space.is_true(w_encoder): w_res = space.call_function(w_encoder, w_obj, space.wrap(errors)) return space.getitem(w_res, space.wrap(0)) else: assert 0, "XXX, what to do here?" -encode.unwrap_spec = [ObjSpace, W_Root, str, str] +encode.unwrap_spec = [ObjSpace, W_Root, W_Root, str] -def decode(space, w_obj, encoding=NoneNotWrapped, errors='strict'): +def decode(space, w_obj, w_encoding=NoneNotWrapped, errors='strict'): """decode(obj, [encoding[,errors]]) -> object Decodes obj using the codec registered for encoding. encoding defaults @@ -162,8 +164,10 @@ as well as any other name registerd with codecs.register_error that is able to handle ValueErrors. """ - if encoding is None: + if w_encoding is None: encoding = space.sys.defaultencoding + else: + encoding = space.str_w(w_encoding) w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1)) if space.is_true(w_decoder): w_res = space.call_function(w_decoder, w_obj, space.wrap(errors)) @@ -175,7 +179,7 @@ return space.getitem(w_res, space.wrap(0)) else: assert 0, "XXX, what to do here?" -decode.unwrap_spec = [ObjSpace, W_Root, str, str] +decode.unwrap_spec = [ObjSpace, W_Root, W_Root, str] def register_error(space, errors, w_handler): """register_error(errors, handler) Modified: pypy/dist/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/test/test_codecs.py Wed Nov 21 10:22:35 2007 @@ -278,3 +278,16 @@ _codecs.register(search_function) raises(TypeError, "hello".decode, "test.mytestenc") raises(TypeError, u"hello".encode, "test.mytestenc") + + def test_cpytest_decode(self): + import codecs + print 1 + assert codecs.decode('\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc' + print 2 + raises(TypeError, codecs.decode) + print 3 + assert codecs.decode('abc') == u'abc' + print 4 + raises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii') + + From cfbolz at codespeak.net Wed Nov 21 10:33:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 10:33:05 +0100 (CET) Subject: [pypy-svn] r48886 - pypy/branch/ropes-unicode/pypy/config Message-ID: <20071121093305.C73878115@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 10:33:04 2007 New Revision: 48886 Modified: pypy/branch/ropes-unicode/pypy/config/pypyoption.py Log: fix typo Modified: pypy/branch/ropes-unicode/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/config/pypyoption.py (original) +++ pypy/branch/ropes-unicode/pypy/config/pypyoption.py Wed Nov 21 10:33:04 2007 @@ -153,7 +153,7 @@ suggests=[("objspace.std.withprebuiltchar", True), ("objspace.std.sharesmallstr", True)]), - BoolOption("withropeunicod", "use ropes for the unicode implementation", + BoolOption("withropeunicode", "use ropes for the unicode implementation", default=False, requires=[("objspace.std.withrope", True)]), From antocuni at codespeak.net Wed Nov 21 10:37:20 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Nov 2007 10:37:20 +0100 (CET) Subject: [pypy-svn] r48887 - pypy/dist/pypy/translator Message-ID: <20071121093720.C9D398125@code0.codespeak.net> Author: antocuni Date: Wed Nov 21 10:37:20 2007 New Revision: 48887 Modified: pypy/dist/pypy/translator/driver.py Log: use type -p instead of which because BSD's which is broken (thanks xoraxax) Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Wed Nov 21 10:37:20 2007 @@ -665,7 +665,7 @@ newexename = basename f = file(newexename, 'w') f.write("""#!/bin/bash -LEDIT=`which ledit 2>/dev/null` +LEDIT=`type -p ledit` if [ `uname -o` = 'Cygwin' ]; then MONO=; else MONO=mono; fi $LEDIT $MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) @@ -735,7 +735,7 @@ newexename = basename f = file(newexename, 'w') f.write("""#!/bin/bash -LEDIT=`which ledit 2>/dev/null` +LEDIT=`type -p ledit` $LEDIT java -jar $0.jar "$@" """) f.close() From pedronis at codespeak.net Wed Nov 21 11:09:38 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 21 Nov 2007 11:09:38 +0100 (CET) Subject: [pypy-svn] r48888 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20071121100938.238DA8138@code0.codespeak.net> Author: pedronis Date: Wed Nov 21 11:09:37 2007 New Revision: 48888 Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Log: make sure the struct is initialized properly for the sake of the c backend for the i386 tests Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Wed Nov 21 11:09:37 2007 @@ -1191,7 +1191,7 @@ T = lltype.Struct('T', ('x', lltype.Signed)) A = lltype.Array(T) S = lltype.GcStruct('S', ('a', A)) - s = lltype.malloc(S, 3) + s = lltype.malloc(S, 3, zero=True) s.a[2].x = 42 def f(n): s1 = hint(s, variable=True) From antocuni at codespeak.net Wed Nov 21 11:20:55 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Nov 2007 11:20:55 +0100 (CET) Subject: [pypy-svn] r48889 - pypy/dist/pypy/doc Message-ID: <20071121102055.CAF07813C@code0.codespeak.net> Author: antocuni Date: Wed Nov 21 11:20:54 2007 New Revision: 48889 Modified: pypy/dist/pypy/doc/_ref.txt pypy/dist/pypy/doc/translation.txt Log: fix doc tests Modified: pypy/dist/pypy/doc/_ref.txt ============================================================================== --- pypy/dist/pypy/doc/_ref.txt (original) +++ pypy/dist/pypy/doc/_ref.txt Wed Nov 21 11:20:54 2007 @@ -70,7 +70,6 @@ .. _`objspace/cpy/`: ../../pypy/objspace/cpy .. _`objspace/dump.py`: ../../pypy/objspace/dump.py .. _`objspace/flow/`: ../../pypy/objspace/flow -.. _`objspace/logic.py`: ../../pypy/objspace/logic.py .. _`objspace/std/`: .. _`pypy/objspace/std`: ../../pypy/objspace/std .. _`objspace/taint.py`: ../../pypy/objspace/taint.py @@ -123,6 +122,5 @@ .. _`translator/js/`: ../../pypy/translator/js .. _`translator/jvm/`: ../../pypy/translator/jvm .. _`translator/llvm/`: ../../pypy/translator/llvm -.. _`translator/squeak/`: ../../pypy/translator/squeak .. _`translator/stackless/`: ../../pypy/translator/stackless .. _`translator/tool/`: ../../pypy/translator/tool \ No newline at end of file Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Wed Nov 21 11:20:54 2007 @@ -817,6 +817,9 @@ .. _`whatis document`: js/whatis.html .. _`using document`: js/using.html +.. _`Python again`: +.. _`Python back-end`: + The Interpreter-Level backend ----------------------------- From antocuni at codespeak.net Wed Nov 21 11:50:18 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Nov 2007 11:50:18 +0100 (CET) Subject: [pypy-svn] r48890 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/ootypesystem translator Message-ID: <20071121105018.9B18A8140@code0.codespeak.net> Author: antocuni Date: Wed Nov 21 11:50:17 2007 New Revision: 48890 Modified: pypy/dist/pypy/rpython/exceptiondata.py pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py pypy/dist/pypy/rpython/ootypesystem/exceptiondata.py pypy/dist/pypy/translator/exceptiontransform.py Log: fix exceptiontransformer to work with ootype Modified: pypy/dist/pypy/rpython/exceptiondata.py ============================================================================== --- pypy/dist/pypy/rpython/exceptiondata.py (original) +++ pypy/dist/pypy/rpython/exceptiondata.py Wed Nov 21 11:50:17 2007 @@ -36,3 +36,9 @@ helper_fn = rtyper.annotate_helper_fn(ll_raise_OSError, [annmodel.SomeInteger()]) return helper_fn + def get_standard_ll_exc_instance(self, rtyper, clsdef): + rclass = rtyper.type_system.rclass + r_inst = rclass.getinstancerepr(rtyper, clsdef) + example = r_inst.get_reusable_prebuilt_instance() + example = self.cast_exception(self.lltype_of_exception_value, example) + return example Modified: pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py Wed Nov 21 11:50:17 2007 @@ -105,8 +105,5 @@ helper_fn = rtyper.annotate_helper_fn(ll_pyexcclass2exc, [s_pyobj]) return helper_fn - def get_standard_ll_exc_instance(self, rtyper, clsdef): - r_inst = rclass.getinstancerepr(rtyper, clsdef) - example = r_inst.get_reusable_prebuilt_instance() - example = rclass.ll_cast_to_object(example) - return example + def cast_exception(self, TYPE, value): + return rclass.ll_cast_to_object(value) Modified: pypy/dist/pypy/rpython/ootypesystem/exceptiondata.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/exceptiondata.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/exceptiondata.py Wed Nov 21 11:50:17 2007 @@ -56,9 +56,7 @@ is_standard = (cls.__module__ == 'exceptions' and not clsdef.attrs) if is_standard: - r_inst = rclass.getinstancerepr(rtyper, clsdef) - example = r_inst.get_reusable_prebuilt_instance() - example = ootype.ooupcast(self.lltype_of_exception_value, example) + example = self.get_standard_ll_exc_instance(rtyper, clsdef) table[cls] = example r_inst = rclass.getinstancerepr(rtyper, None) r_inst.setup() @@ -99,3 +97,6 @@ self._TYPE = FakeCallableType self._callable = fn return fake_callable(pyexcclass2exc) + + def cast_exception(self, TYPE, value): + return ootype.ooupcast(TYPE, value) Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Wed Nov 21 11:50:17 2007 @@ -55,9 +55,10 @@ self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) exc_data, null_type, null_value = self.setup_excdata() + rclass = translator.rtyper.type_system.rclass runtime_error_def = translator.annotator.bookkeeper.getuniqueclassdef(RuntimeError) runtime_error_ll_exc = edata.get_standard_ll_exc_instance(translator.rtyper, runtime_error_def) - runtime_error_ll_exc_type = runtime_error_ll_exc.typeptr + runtime_error_ll_exc_type = rclass.ll_inst_type(runtime_error_ll_exc) def rpyexc_occured(): exc_type = exc_data.exc_type From fijal at codespeak.net Wed Nov 21 11:57:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Nov 2007 11:57:03 +0100 (CET) Subject: [pypy-svn] r48891 - pypy/dist/pypy/module/sys Message-ID: <20071121105703.1507D8112@code0.codespeak.net> Author: fijal Date: Wed Nov 21 11:57:02 2007 New Revision: 48891 Modified: pypy/dist/pypy/module/sys/app.py Log: Move it back to app-level, seems to be not working under llinterp Modified: pypy/dist/pypy/module/sys/app.py ============================================================================== --- pypy/dist/pypy/module/sys/app.py (original) +++ pypy/dist/pypy/module/sys/app.py Wed Nov 21 11:57:02 2007 @@ -1,3 +1,4 @@ +# NOT_RPYTHON """ The 'sys' module. """ From xoraxax at codespeak.net Wed Nov 21 12:04:40 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 21 Nov 2007 12:04:40 +0100 (CET) Subject: [pypy-svn] r48892 - pypy/dist/pypy/objspace/std Message-ID: <20071121110440.0887C811B@code0.codespeak.net> Author: xoraxax Date: Wed Nov 21 12:04:39 2007 New Revision: 48892 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dicttype.py Log: Add a dict.pop MM for the dictmultiobject to avoid using the slow app-level version from dicttype. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Nov 21 12:04:39 2007 @@ -1235,6 +1235,21 @@ def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): return w_dict.get(w_lookup, w_default) +def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): + defaults = space.unpackiterable(w_defaults) + len_defaults = len(defaults) + if len_defaults > 1: + raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) + w_item = w_dict.implementation.get(w_key) + if w_item is None: + if len_defaults > 0: + return defaults[0] + else: + raise OperationError(space.w_KeyError, w_key) + else: + w_dict.implementation.delitem(w_key) + return w_item + app = gateway.applevel(''' def dictrepr(currently_in_repr, d): # Now we only handle one implementation of dicts, this one. Modified: pypy/dist/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/dist/pypy/objspace/std/dicttype.py (original) +++ pypy/dist/pypy/objspace/std/dicttype.py Wed Nov 21 12:04:39 2007 @@ -52,6 +52,8 @@ #dict_str = StdObjSpace.str # default application-level implementations for some operations +# most of these (notably not popitem and update*) are overwritten +# in dictmultiobject # gateway is imported in the stdtypedef module app = gateway.applevel(''' @@ -64,9 +66,10 @@ d[k] = v def update(d, *args, **kwargs): - if len(args) == 1: + len_args = len(args) + if len_args == 1: update1(d, args[0]) - elif len(args) > 1: + elif len_args > 1: raise TypeError("update takes at most 1 (non-keyword) argument") if kwargs: update1(d, kwargs) From cfbolz at codespeak.net Wed Nov 21 12:09:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 12:09:27 +0100 (CET) Subject: [pypy-svn] r48893 - pypy/dist/pypy/module/_codecs Message-ID: <20071121110927.02C6D8142@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 12:09:27 2007 New Revision: 48893 Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py Log: fix CPython codecs tests Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/interp_codecs.py Wed Nov 21 12:09:27 2007 @@ -218,7 +218,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) assert hasattr(runicode, rname) - def wrap_decoder(space, string, errors="strict", w_final=True): + def wrap_decoder(space, string, errors="strict", w_final=False): final = space.is_true(w_final) state = space.fromcache(CodecState) func = getattr(runicode, rname) From pedronis at codespeak.net Wed Nov 21 12:15:25 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 21 Nov 2007 12:15:25 +0100 (CET) Subject: [pypy-svn] r48894 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071121111525.C4DD1813C@code0.codespeak.net> Author: pedronis Date: Wed Nov 21 12:15:25 2007 New Revision: 48894 Modified: pypy/dist/pypy/translator/backendopt/all.py pypy/dist/pypy/translator/backendopt/support.py pypy/dist/pypy/translator/backendopt/test/test_all.py Log: (cfbolz, pedronis) fix for test_idempotent rerun noops removal at the end of all opts Modified: pypy/dist/pypy/translator/backendopt/all.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/all.py (original) +++ pypy/dist/pypy/translator/backendopt/all.py Wed Nov 21 12:15:25 2007 @@ -54,15 +54,18 @@ check_virtual_methods() # remove obvious no-ops - for graph in graphs: - removenoops.remove_same_as(graph) - simplify.eliminate_empty_blocks(graph) - simplify.transform_dead_op_vars(graph, translator) - removenoops.remove_duplicate_casts(graph, translator) + def remove_obvious_noops(): + for graph in graphs: + removenoops.remove_same_as(graph) + simplify.eliminate_empty_blocks(graph) + simplify.transform_dead_op_vars(graph, translator) + removenoops.remove_duplicate_casts(graph, translator) - if config.print_statistics: - print "after no-op removal:" - print_statistics(translator.graphs[0], translator) + if config.print_statistics: + print "after no-op removal:" + print_statistics(translator.graphs[0], translator) + + remove_obvious_noops() if config.inline or config.mallocs: heuristic = get_function(config.inline_heuristic) @@ -129,6 +132,8 @@ from pypy.translator.backendopt import coalloc coalloc.malloc_to_coalloc(translator) + remove_obvious_noops() + for graph in graphs: checkgraph(graph) Modified: pypy/dist/pypy/translator/backendopt/support.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/support.py (original) +++ pypy/dist/pypy/translator/backendopt/support.py Wed Nov 21 12:15:25 2007 @@ -185,9 +185,12 @@ def md5digest(translator): import md5 - m = md5.new() - for op in all_operations(translator.graphs): - m.update(op.opname + str(op.result)) - for a in op.args: - m.update(str(a)) - return m.digest()[:] + graph2digest = {} + for graph in translator.graphs: + m = md5.new() + for op in graph_operations(graph): + m.update(op.opname + str(op.result)) + for a in op.args: + m.update(str(a)) + graph2digest[graph.name] = m.digest() + return graph2digest Modified: pypy/dist/pypy/translator/backendopt/test/test_all.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_all.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_all.py Wed Nov 21 12:15:25 2007 @@ -134,20 +134,31 @@ c = [i for i in range(n2)] return 33 + big() + g(10) - t = self.translateopt(idempotent, [int, int], raisingop2direct_call=True, - constfold=False) + t = self.translateopt(idempotent, [int, int], + raisingop2direct_call=True, + constfold=False) + #backend_optimizations(t, raisingop2direct_call=True, + # inline_threshold=0, constfold=False) + digest1 = md5digest(t) digest2 = md5digest(t) - assert digest1 == digest2 + def compare(digest1, digest2): + diffs = [] + assert digest1.keys() == digest2.keys() + for name in digest1: + if digest1[name] != digest2[name]: + diffs.append(name) + assert not diffs + + compare(digest1, digest2) #XXX Inlining and constfold are currently non-idempotent. # Maybe they just renames variables but the graph changes in some way. backend_optimizations(t, raisingop2direct_call=True, inline_threshold=0, constfold=False) digest3 = md5digest(t) - assert digest1 == digest3 - + compare(digest1, digest3) def test_bug_inlined_if(self): def f(x, flag): From cfbolz at codespeak.net Wed Nov 21 12:37:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 12:37:43 +0100 (CET) Subject: [pypy-svn] r48895 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071121113743.5C6FD8125@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 12:37:41 2007 New Revision: 48895 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Wed Nov 21 12:37:41 2007 @@ -11,17 +11,33 @@ Wednesday after the planning session: - Start slowly thinking about 1.1 plans +Thursday afternoon: + - discuss callback mechanism for rffi/lltype/ll2ctypes + +Friday: + - breakday + +Saturday: + - discuss approaches for extension modules Translation toolchain +++++++++++++++++++++ - - fix the nightly CPython test runs (Samuele) + - fix the nightly CPython test runs DONE + + - kill remaining suggested_primitives DONE + + - remove the symbol table from the C backend DONE - - find out why our memcpy behaves so erratically (Armin, Alexander, quickly) + - fix windows compilation (christian) + + - find out why our memcpy behaves so erratically + GCC behaves very erratically, so we won't work on this for now - finish rctypes removal - - kill remaining suggested_primitives (Armin, Alexander) + - remove support for suggested_primitives and the extfunctable (alexander, + carl friedrich) - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) @@ -32,19 +48,16 @@ - clean up the tangle of including headers in the C backend - - remove the use of pyobjs in the test wrapping functions (Carl Friedrich, - Richard) - - - remove the symbol table from the C backend + - remove the use of pyobjs in the test wrapping functions IN-PROGRESS + (richard, samuele) - kill half concrete wrapper - - decide how to implement constants in rffi + - implement global variables in rffi (anto, maciek) - - implement callback mechanism for rffi/lltype/ll2ctypes - - - refactor OO external objects (Maciek, Anto) + - refactor OO external objects. discussion happened, the outcome was that it + doesn't make sense, because javascript and cli are too different - review pdbplus, especially the graph commands, also in the light of https://codespeak.net/issue/pypy-dev/issue303 and the fact that we @@ -61,7 +74,8 @@ - fix the thunk space - review the things implemented at applevel whether they are - performance-critical + performance-critical IN-PROGRESS: moved super, classmethod, statimethod to + interplevel, some dict methods - remove prebuilt costate vars (might happen before) From cfbolz at codespeak.net Wed Nov 21 12:38:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 12:38:38 +0100 (CET) Subject: [pypy-svn] r48896 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071121113838.8F97D8129@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 12:38:37 2007 New Revision: 48896 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: the discussion is moved, armin is still sleeping Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Wed Nov 21 12:38:37 2007 @@ -8,7 +8,7 @@ - Review the cpython regression test suite (skips/failures) - think about/remove orphaned parts -Wednesday after the planning session: +Wednesday afternoon: - Start slowly thinking about 1.1 plans Thursday afternoon: From tismer at codespeak.net Wed Nov 21 13:27:42 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Wed, 21 Nov 2007 13:27:42 +0100 (CET) Subject: [pypy-svn] r48897 - pypy/dist/pypy/rpython/module Message-ID: <20071121122742.09CE9811A@code0.codespeak.net> Author: tismer Date: Wed Nov 21 13:27:40 2007 New Revision: 48897 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: fixing includes, to make windows build, again Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Nov 21 13:27:40 2007 @@ -138,7 +138,7 @@ if sys.platform.startswith('win'): _name_struct_stat = '_stati64' - INCLUDES = [] + INCLUDES = ['sys/types.h', 'sys/stat.h'] else: _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] From cfbolz at codespeak.net Wed Nov 21 14:02:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 14:02:12 +0100 (CET) Subject: [pypy-svn] r48898 - in pypy/dist/pypy/rpython: . test Message-ID: <20071121130212.896158135@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 14:02:11 2007 New Revision: 48898 Modified: pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/test/test_rstr.py Log: add support for "str in str" Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Wed Nov 21 14:02:11 2007 @@ -371,6 +371,15 @@ return hop.genop('int_gt', [vres, hop.inputconst(Signed, 0)], resulttype=Bool) + def rtype_contains((r_str1, r_str2), hop): + v_str1, v_str2 = hop.inputargs(r_str1.repr, r_str2.repr) + v_end = hop.gendirectcall(r_str1.ll.ll_strlen, v_str1) + vres = hop.gendirectcall(r_str1.ll.ll_find, v_str1, v_str2, + hop.inputconst(Signed, 0), v_end) + return hop.genop('int_ne', [vres, hop.inputconst(Signed, -1)], + resulttype=Bool) + + class __extend__(pairtype(AbstractStringRepr, AbstractCharRepr), pairtype(AbstractUnicodeRepr, AbstractUniCharRepr)): def rtype_contains((r_str, r_chr), hop): Modified: pypy/dist/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rstr.py (original) +++ pypy/dist/pypy/rpython/test/test_rstr.py Wed Nov 21 14:02:11 2007 @@ -239,6 +239,18 @@ res = self.interpret(fn, [i,j]) assert res == fn(i, j) + + def test_contains_str(self): + const = self.const + def fn(i, j): + s1 = [const('one two three'), const('abc abcdab abcdabcdabde')] + s2 = [const('one'), const('two'), const('abcdab'), const('one tou'), const('abcdefgh'), const('fortytwo'), const('')] + return s2[j] in s1[i] + for i in range(2): + for j in range(7): + res = self.interpret(fn, [i,j]) + assert res == fn(i, j) + def test_find_with_start(self): const = self.const def fn(i): From xoraxax at codespeak.net Wed Nov 21 14:34:41 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 21 Nov 2007 14:34:41 +0100 (CET) Subject: [pypy-svn] r48899 - in pypy/dist/pypy: annotation jit/hintannotator rlib rpython rpython/lltypesystem rpython/module translator translator/backendopt translator/c translator/c/test translator/llvm translator/oosupport translator/stackless Message-ID: <20071121133441.410EA8142@code0.codespeak.net> Author: xoraxax Date: Wed Nov 21 14:34:39 2007 New Revision: 48899 Removed: pypy/dist/pypy/rpython/extfunctable.py Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/rlib/rstack.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/exceptiondata.py pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/extfuncregistry.py pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/module/support.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rexternalobj.py pypy/dist/pypy/rpython/rspecialcase.py pypy/dist/pypy/translator/backendopt/inline.py pypy/dist/pypy/translator/backendopt/removenoops.py pypy/dist/pypy/translator/backendopt/support.py pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_stackless.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/typedefnode.py pypy/dist/pypy/translator/oosupport/function.py pypy/dist/pypy/translator/oosupport/metavm.py pypy/dist/pypy/translator/simplify.py pypy/dist/pypy/translator/stackless/code.py pypy/dist/pypy/translator/stackless/frame.py Log: (cfbolz, xoraxax): Remove extfunctable and suggested_primitive support. Refactored exttypeinfo-usage of rstack to use a hint on the opaquetype. Ignored a weird hack in the extfunctable regarding ntpath.isabs because there is str in str support in rpython now. Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Wed Nov 21 14:34:39 2007 @@ -662,25 +662,3 @@ BUILTIN_ANALYZERS[llmemory.offsetof] = offsetof -#_________________________________ -# external functions - - -from pypy.rpython import extfunctable - -def update_exttables(): - - # import annotation information for external functions - # from the extfunctable.table into our own annotation specific table - for func, extfuncinfo in extfunctable.table.iteritems(): - BUILTIN_ANALYZERS[func] = extfuncinfo.annotation - - # import annotation information for external types - # from the extfunctable.typetable into our own annotation specific table - for typ, exttypeinfo in extfunctable.typetable.iteritems(): - EXTERNAL_TYPE_ANALYZERS[typ] = exttypeinfo.get_annotations() - -# Note: calls to declare() may occur after builtin.py is first imported. -# We must track future changes to the extfunctables. -extfunctable.table_callbacks.append(update_exttables) -update_exttables() Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Wed Nov 21 14:34:39 2007 @@ -460,9 +460,7 @@ class SomeExternalObject(SomeObject): """Stands for an object of 'external' type. External types have a Repr - controlled by pypy.rpython.extregistry; or they come from the (obsolete) - table created by pypy.rpython.extfunctable.declaretype() and represent - simple types with some methods that need direct back-end support.""" + controlled by pypy.rpython.extregistry.""" def __init__(self, knowntype): self.knowntype = knowntype Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Wed Nov 21 14:34:39 2007 @@ -634,21 +634,8 @@ return self.s_result class __extend__(SomeExternalObject): - # XXX kill with extfunctable.py - def find_method(obj, name): - "Look for a special-case implementation for the named method." - type_analyser = builtin.EXTERNAL_TYPE_ANALYZERS[obj.knowntype] - if name in type_analyser: - analyser = type_analyser[name] - return SomeBuiltin(analyser, obj, name) - return SomeObject.find_method(obj, name) - def getattr(p, s_attr): if s_attr.is_constant() and isinstance(s_attr.const, str): - # XXX kill with extfunctable.py - if p.knowntype in builtin.EXTERNAL_TYPE_ANALYZERS: - return SomeObject.getattr(p, s_attr) - attr = s_attr.const entry = extregistry.lookup_type(p.knowntype) s_value = entry.get_field_annotation(p.knowntype, attr) Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Wed Nov 21 14:34:39 2007 @@ -437,9 +437,6 @@ *args_hs) except NotImplementedError: pass - # don't try to annotate suggested_primitive graphs - if getattr(getattr(fnobj, '_callable', None), 'suggested_primitive', False): - return variableoftype(lltype.typeOf(fnobj).RESULT) # normal call if not hasattr(fnobj, 'graph'): Modified: pypy/dist/pypy/rlib/rstack.py ============================================================================== --- pypy/dist/pypy/rlib/rstack.py (original) +++ pypy/dist/pypy/rlib/rstack.py Wed Nov 21 14:34:39 2007 @@ -78,8 +78,7 @@ frame_stack_top_controller = FrameStackTopController() bound_switch_of_frame_stack_top_controller = BoundSwitchOfFrameStackTopController() -OPAQUE_STATE_HEADER = lltype.GcOpaqueType("OPAQUE_STATE_HEADER") -OPAQUE_STATE_HEADER._exttypeinfo = "Really bad hack - dont remove" +OPAQUE_STATE_HEADER = lltype.GcOpaqueType("OPAQUE_STATE_HEADER", hints={"render_structure": True}) OPAQUE_STATE_HEADER_PTR = lltype.Ptr(OPAQUE_STATE_HEADER) Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Nov 21 14:34:39 2007 @@ -8,7 +8,7 @@ from pypy.annotation.policy import AnnotatorPolicy, Sig from pypy.annotation.specialize import flatten_star_args from pypy.rpython.lltypesystem import lltype -from pypy.rpython import extfunctable, extregistry +from pypy.rpython import extregistry from pypy.objspace.flow.model import Constant class KeyComp(object): @@ -71,25 +71,6 @@ return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, {}) default_specialize = staticmethod(default_specialize) - def override__init_opaque_object(pol, s_opaqueptr, s_value): - assert isinstance(s_opaqueptr, annmodel.SomePtr) - assert isinstance(s_opaqueptr.ll_ptrtype.TO, lltype.OpaqueType) - assert isinstance(s_value, annmodel.SomeExternalObject) - exttypeinfo = extfunctable.typetable[s_value.knowntype] - assert s_opaqueptr.ll_ptrtype.TO._exttypeinfo == exttypeinfo - return annmodel.SomeExternalObject(exttypeinfo.typ) - - def override__from_opaque_object(pol, s_opaqueptr): - assert isinstance(s_opaqueptr, annmodel.SomePtr) - assert isinstance(s_opaqueptr.ll_ptrtype.TO, lltype.OpaqueType) - exttypeinfo = s_opaqueptr.ll_ptrtype.TO._exttypeinfo - return annmodel.SomeExternalObject(exttypeinfo.typ) - - def override__to_opaque_object(pol, s_value): - assert isinstance(s_value, annmodel.SomeExternalObject) - exttypeinfo = extfunctable.typetable[s_value.knowntype] - return annmodel.SomePtr(lltype.Ptr(exttypeinfo.get_lltype())) - def specialize__ts(pol, funcdesc, args_s, ref): ts = pol.rtyper.type_system ref = ref.split('.') Modified: pypy/dist/pypy/rpython/exceptiondata.py ============================================================================== --- pypy/dist/pypy/rpython/exceptiondata.py (original) +++ pypy/dist/pypy/rpython/exceptiondata.py Wed Nov 21 14:34:39 2007 @@ -1,7 +1,26 @@ from pypy.rpython import rclass -from pypy.rpython.extfunctable import standardexceptions from pypy.annotation import model as annmodel + +# the exceptions that can be implicitely raised by some operations +standardexceptions = { + TypeError : True, + OverflowError : True, + ValueError : True, + ZeroDivisionError: True, + MemoryError : True, + IOError : True, + OSError : True, + StopIteration : True, + KeyError : True, + IndexError : True, + AssertionError : True, + RuntimeError : True, + UnicodeDecodeError: True, + UnicodeEncodeError: True, + } + + class AbstractExceptionData: """Public information for the code generators to help with exceptions.""" standardexceptions = standardexceptions @@ -42,3 +61,5 @@ example = r_inst.get_reusable_prebuilt_instance() example = self.cast_exception(self.lltype_of_exception_value, example) return example + + Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Wed Nov 21 14:34:39 2007 @@ -271,8 +271,6 @@ def is_external(func): if hasattr(func, 'value'): func = func.value - if getattr(func._callable, 'suggested_primitive', False): - return True if hasattr(func, '_external_name'): return True return False Modified: pypy/dist/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extfuncregistry.py (original) +++ pypy/dist/pypy/rpython/extfuncregistry.py Wed Nov 21 14:34:39 2007 @@ -1,6 +1,4 @@ # this registry uses the new interface for external functions -# all the above declarations in extfunctable should be moved here -# at some point. from extfunc import register_external Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Wed Nov 21 14:34:39 2007 @@ -615,8 +615,6 @@ def perform_call(self, f, ARGS, args): fobj = self.llinterpreter.typer.type_system.deref(f) has_callable = getattr(fobj, '_callable', None) is not None - if has_callable and getattr(fobj._callable, 'suggested_primitive', False): - return self.invoke_callable_with_pyexceptions(f, *args) if hasattr(fobj, 'graph'): graph = fobj.graph else: Modified: pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/exceptiondata.py Wed Nov 21 14:34:39 2007 @@ -4,7 +4,6 @@ Array, malloc, Ptr, PyObject, pyobjectptr, \ FuncType, functionptr, Signed from pypy.rpython.exceptiondata import AbstractExceptionData -from pypy.rpython.extfunctable import standardexceptions from pypy.annotation.classdef import FORCE_ATTRIBUTES_INTO_CLASSES class ExceptionData(AbstractExceptionData): Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Nov 21 14:34:39 2007 @@ -455,6 +455,8 @@ _gckind = 'raw' def __init__(self, tag, hints={}): + """ if hints['render_structure'] is set, the type is internal and not considered + to come from somewhere else (it should be rendered as a structure) """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) Modified: pypy/dist/pypy/rpython/module/support.py ============================================================================== --- pypy/dist/pypy/rpython/module/support.py (original) +++ pypy/dist/pypy/rpython/module/support.py Wed Nov 21 14:34:39 2007 @@ -1,6 +1,5 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.rpython import extfunctable from pypy.rpython.lltypesystem.lltype import \ GcStruct, Signed, Array, Char, Ptr, malloc, GcArray from pypy.rpython.rlist import ll_append @@ -86,19 +85,4 @@ dstchars[i] = srcchars[i] i += 1 -def init_opaque_object(opaqueptr, value): - "NOT_RPYTHON" - opaqueptr._obj.externalobj = value -init_opaque_object._annspecialcase_ = "override:init_opaque_object" - -def from_opaque_object(opaqueptr): - "NOT_RPYTHON" - return opaqueptr._obj.externalobj -from_opaque_object._annspecialcase_ = "override:from_opaque_object" - -def to_opaque_object(value): - "NOT_RPYTHON" - exttypeinfo = extfunctable.typetable[value.__class__] - return lltype.opaqueptr(exttypeinfo.get_lltype(), 'opaque', - externalobj=value) -to_opaque_object._annspecialcase_ = "override:to_opaque_object" + Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Wed Nov 21 14:34:39 2007 @@ -515,53 +515,6 @@ BUILTIN_TYPER[objectmodel.hlinvoke] = rtype_hlinvoke -from pypy.rpython import extfunctable - -def rnormalize(rtyper, r): - # this replaces char_repr with string_repr, because so far we have - # no external function expecting a char, but only external functions - # that happily crash if passed a char instead of a string - if r == rtyper.type_system.rstr.char_repr: - r = rtyper.type_system.rstr.string_repr - return r - -def make_rtype_extfunc(extfuncinfo): - if extfuncinfo.ll_annotable: - def rtype_extfunc(hop): - ll_function = extfuncinfo.get_ll_function(hop.rtyper.type_system) - vars = hop.inputargs(*[rnormalize(hop.rtyper, r) - for r in hop.args_r]) - hop.exception_is_here() - return hop.gendirectcall(ll_function, *vars) - else: - def rtype_extfunc(hop): - ll_function = extfuncinfo.get_ll_function(hop.rtyper.type_system) - resulttype = hop.r_result - vars = hop.inputargs(*[rnormalize(hop.rtyper, r) - for r in hop.args_r]) - hop.exception_is_here() - return hop.llops.genexternalcall(ll_function.__name__, vars, resulttype=resulttype, - _callable = ll_function) - - if extfuncinfo.func is not None: - rtype_extfunc = sourcetools.func_with_new_name(rtype_extfunc, - "rtype_extfunc_%s" % extfuncinfo.func.__name__) - return rtype_extfunc - - -def update_exttable(): - """import rtyping information for external functions - from the extfunctable.table into our own specific table - """ - for func, extfuncinfo in extfunctable.table.iteritems(): - if func not in BUILTIN_TYPER: - BUILTIN_TYPER[func] = make_rtype_extfunc(extfuncinfo) - -# Note: calls to declare() may occur after rbuiltin.py is first imported. -# We must track future changes to the extfunctable. -extfunctable.table_callbacks.append(update_exttable) -update_exttable() - # _________________________________________________________________ # memory addresses Modified: pypy/dist/pypy/rpython/rexternalobj.py ============================================================================== --- pypy/dist/pypy/rpython/rexternalobj.py (original) +++ pypy/dist/pypy/rpython/rexternalobj.py Wed Nov 21 14:34:39 2007 @@ -2,9 +2,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.rpython.rmodel import Repr, HalfConcreteWrapper -from pypy.rpython.extfunctable import typetable from pypy.rpython import rbuiltin -from pypy.rpython.module.support import init_opaque_object from pypy.objspace.flow.model import Constant, Variable from pypy.rpython import extregistry from pypy.annotation.signature import annotation @@ -15,13 +13,9 @@ class __extend__(annmodel.SomeExternalObject): def rtyper_makerepr(self, rtyper): - # XXX kill with extfunctable.py - if self.knowntype in typetable: - return ExternalObjRepr(self.knowntype) - else: - # delegate to the get_repr() of the extregistrered Entry class - entry = extregistry.lookup_type(self.knowntype) - return entry.get_repr(rtyper, self) + # delegate to the get_repr() of the extregistrered Entry class + entry = extregistry.lookup_type(self.knowntype) + return entry.get_repr(rtyper, self) def rtyper_makekey(self): # grab all attributes of the SomeExternalObject for the key @@ -31,41 +25,4 @@ if 'const_box' in attrs: del attrs['const_box'] return self.__class__, attrs - -class ExternalObjRepr(Repr): - """Repr for the (obsolecent) extfunctable.declaretype() case. - If you use the extregistry instead you get to pick your own Repr. - """ - def __init__(self, knowntype): - self.exttypeinfo = typetable[knowntype] - TYPE = self.exttypeinfo.get_lltype() - self.lowleveltype = lltype.Ptr(TYPE) - self.instance_cache = {} - # The set of methods supported depends on 'knowntype', so we - # cannot have rtype_method_xxx() methods directly on the - # ExternalObjRepr class. But we can store them in 'self' now. - for name, extfuncinfo in self.exttypeinfo.methods.items(): - methodname = 'rtype_method_' + name - bltintyper = rbuiltin.make_rtype_extfunc(extfuncinfo) - setattr(self, methodname, bltintyper) - - def convert_const(self, value): - T = self.exttypeinfo.get_lltype() - if value is None: - return lltype.nullptr(T) - if not isinstance(value, self.exttypeinfo.typ): - raise TyperError("expected a %r: %r" % (self.exttypeinfo.typ, - value)) - key = Constant(value) - try: - p = self.instance_cache[key] - except KeyError: - p = lltype.malloc(T) - init_opaque_object(p.obj, value) - self.instance_cache[key] = p - return p - - def rtype_is_true(self, hop): - vlist = hop.inputargs(self) - return hop.genop('ptr_nonzero', vlist, resulttype=lltype.Bool) Modified: pypy/dist/pypy/rpython/rspecialcase.py ============================================================================== --- pypy/dist/pypy/rpython/rspecialcase.py (original) +++ pypy/dist/pypy/rpython/rspecialcase.py Wed Nov 21 14:34:39 2007 @@ -30,18 +30,6 @@ v, = hop.inputargs(hop.args_r[0]) return v -def rtype_override_init_opaque_object(hop): - return hop.genop('init_opaque_object_should_never_be_seen_by_the_backend', - [], resulttype=hop.r_result) - -def rtype_override_from_opaque_object(hop): - return hop.genop('from_opaque_object_should_never_be_seen_by_the_backend', - [], resulttype=hop.r_result) - -def rtype_override_to_opaque_object(hop): - return hop.genop('to_opaque_object_should_never_be_seen_by_the_backend', - [], resulttype=hop.r_result) - def rtype_override_yield_current_frame_to_caller(hop): return hop.genop('yield_current_frame_to_caller', [], resulttype=hop.r_result) Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Wed Nov 21 14:34:39 2007 @@ -634,9 +634,6 @@ graph = getattr(funcobj, 'graph', None) if graph is not None: if getattr(getattr(funcobj, '_callable', None), - 'suggested_primitive', False): - continue - if getattr(getattr(funcobj, '_callable', None), 'dont_inline', False): continue result.append((parentgraph, graph)) @@ -669,9 +666,6 @@ graph = getattr(funcobj, 'graph', None) if graph is not None: if getattr(getattr(funcobj, '_callable', None), - 'suggested_primitive', False): - continue - if getattr(getattr(funcobj, '_callable', None), 'dont_inline', False): continue if candidate(graph): Modified: pypy/dist/pypy/translator/backendopt/removenoops.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/removenoops.py (original) +++ pypy/dist/pypy/translator/backendopt/removenoops.py Wed Nov 21 14:34:39 2007 @@ -111,25 +111,4 @@ else: used[op.args[0]] = True -##def rename_extfunc_calls(translator): -## from pypy.rpython.extfunctable import table as extfunctable -## def visit(block): -## if isinstance(block, Block): -## for op in block.operations: -## if op.opname != 'direct_call': -## continue -## functionref = op.args[0] -## if not isinstance(functionref, Constant): -## continue -## _callable = functionref.value._obj._callable -## for func, extfuncinfo in extfunctable.iteritems(): # precompute a dict? -## if _callable is not extfuncinfo.ll_function or not extfuncinfo.backend_functiontemplate: -## continue -## language, functionname = extfuncinfo.backend_functiontemplate.split(':') -## if language is 'C': -## old_name = functionref.value._obj._name[:] -## functionref.value._obj._name = functionname -## #print 'rename_extfunc_calls: %s -> %s' % (old_name, functionref.value._obj._name) -## break -## for func, graph in translator.flowgraphs.iteritems(): -## traverse(visit, graph) + Modified: pypy/dist/pypy/translator/backendopt/support.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/support.py (original) +++ pypy/dist/pypy/translator/backendopt/support.py Wed Nov 21 14:34:39 2007 @@ -108,8 +108,6 @@ return splitlink def find_calls_from(translator, graph): - if getattr(getattr(graph, "func", None), "suggested_primitive", False): - return for block in graph.iterblocks(): for op in block.operations: if op.opname == "direct_call": Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Wed Nov 21 14:34:39 2007 @@ -87,7 +87,7 @@ node = BareBoneArrayDefNode(self, T, varlength) else: node = ArrayDefNode(self, T, varlength) - elif isinstance(T, OpaqueType) and hasattr(T, '_exttypeinfo'): + elif isinstance(T, OpaqueType) and T.hints.get("render_structure", False): node = ExtTypeOpaqueDefNode(self, T) elif T == WeakRef: REALT = self.gcpolicy.get_real_weakref_type() @@ -134,8 +134,7 @@ elif isinstance(T, OpaqueType): if T == RuntimeTypeInfo: return self.gcpolicy.rtti_type() - elif hasattr(T, '_exttypeinfo'): - # for external types (pypy.rpython.extfunctable.declaretype()) + elif T.hints.get("render_structure", False): node = self.gettypedefnode(T, varlength=varlength) if who_asks is not None: who_asks.dependencies[node] = True Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Wed Nov 21 14:34:39 2007 @@ -433,7 +433,7 @@ class ExtTypeOpaqueDefNode: - "For OpaqueTypes created by pypy.rpython.extfunctable.ExtTypeInfo." + """For OpaqueTypes created with the hint render_structure.""" typetag = 'struct' def __init__(self, db, T): @@ -867,7 +867,7 @@ def opaquenode_factory(db, T, obj): if T == RuntimeTypeInfo: return db.gcpolicy.rtti_node_factory()(db, T, obj) - if hasattr(T, '_exttypeinfo'): + if T.hints.get("render_structure", False): return ExtType_OpaqueNode(db, T, obj) raise Exception("don't know about %r" % (T,)) Modified: pypy/dist/pypy/translator/c/test/test_stackless.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_stackless.py (original) +++ pypy/dist/pypy/translator/c/test/test_stackless.py Wed Nov 21 14:34:39 2007 @@ -17,9 +17,6 @@ def setup_class(cls): import py if cls.gcpolicy in (None, "ref"): - # to re-enable this, remove the two characters 'gc' in the - # declaregcptrtype(rstack.frame_stack_top,...) call in - # rpython/extfunctable. Doing so breaks translator/stackless/. import py py.test.skip("stackless + refcounting doesn't work any more for now") elif cls.gcpolicy == "boehm": Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Nov 21 14:34:39 2007 @@ -72,9 +72,7 @@ def create_constant_node(self, type_, value): node = None if isinstance(type_, lltype.FuncType): - if getattr(value._callable, "suggested_primitive", False): - node = ExternalFuncNode(self, value, value._callable) - elif hasattr(value, '_external_name'): + if hasattr(value, '_external_name'): node = ExternalFuncNode(self, value, value._external_name) elif getattr(value, 'external', None) == 'C': node = ExternalFuncNode(self, value) @@ -102,7 +100,7 @@ node = ArrayNode(self, value) elif isinstance(type_, lltype.OpaqueType): - if hasattr(type_, '_exttypeinfo'): + if type_.hints.get('render_structure', False): node = ExtOpaqueNode(self, value) else: node = OpaqueNode(self, value) Modified: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/typedefnode.py (original) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Wed Nov 21 14:34:39 2007 @@ -145,7 +145,7 @@ return ArrayTypeNode(db, TYPE) elif isinstance(TYPE, lltype.OpaqueType): - if hasattr(TYPE, '_exttypeinfo'): + if TYPE.hints.get("render_structure", False): return ExtOpaqueTypeNode(db, TYPE) else: return OpaqueTypeNode(db, TYPE) Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Wed Nov 21 14:34:39 2007 @@ -109,9 +109,6 @@ if self.db.graph_name(self.graph) is not None and not self.is_method: return # already rendered - if getattr(self.graph.func, 'suggested_primitive', False): - assert False, 'Cannot render a suggested_primitive' - self.ilasm = ilasm self.generator = self._create_generator(self.ilasm) graph = self.graph Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Wed Nov 21 14:34:39 2007 @@ -410,12 +410,6 @@ graph = callee.graph except AttributeError: return callee._name.rsplit('.', 1) - else: - if getattr(graph.func, 'suggested_primitive', False): - _, module = graph.func.__module__.rsplit('.', 1) - return module, graph.func.func_name - else: - return None def render(self, generator, op): callee = op.args[0].value Modified: pypy/dist/pypy/translator/simplify.py ============================================================================== --- pypy/dist/pypy/translator/simplify.py (original) +++ pypy/dist/pypy/translator/simplify.py Wed Nov 21 14:34:39 2007 @@ -34,9 +34,6 @@ funcobj = get_funcobj(f) try: callable = funcobj._callable - # external function calls don't have a real graph - if getattr(callable, "suggested_primitive", False): - return None except (AttributeError, KeyError, AssertionError): return None try: Modified: pypy/dist/pypy/translator/stackless/code.py ============================================================================== --- pypy/dist/pypy/translator/stackless/code.py (original) +++ pypy/dist/pypy/translator/stackless/code.py Wed Nov 21 14:34:39 2007 @@ -1,7 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, lloperation from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import rarithmetic -from pypy.rpython import extfunctable from pypy.translator.stackless import frame from pypy.translator.stackless.frame import STATE_HEADER, SAVED_REFERENCE, STORAGE_TYPES_AND_FIELDS Modified: pypy/dist/pypy/translator/stackless/frame.py ============================================================================== --- pypy/dist/pypy/translator/stackless/frame.py (original) +++ pypy/dist/pypy/translator/stackless/frame.py Wed Nov 21 14:34:39 2007 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython import extfunctable from pypy.rpython.typesystem import getfunctionptr from pypy.rpython.annlowlevel import annotate_lowlevel_helper from pypy.objspace.flow.model import FunctionGraph From cfbolz at codespeak.net Wed Nov 21 14:49:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 14:49:09 +0100 (CET) Subject: [pypy-svn] r48900 - pypy/dist/pypy/annotation Message-ID: <20071121134909.26B0A81A0@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 14:49:08 2007 New Revision: 48900 Modified: pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/signature.py Log: (xoraxax, cfbolz, pedronis): kill some code that is dead now that the extfunctable is gone Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Wed Nov 21 14:49:08 2007 @@ -405,8 +405,6 @@ elif extregistry.is_registered(x, self.policy): entry = extregistry.lookup(x, self.policy) result = entry.compute_annotation_bk(self) - elif tp in EXTERNAL_TYPE_ANALYZERS: - result = SomeExternalObject(tp) elif isinstance(x, lltype._ptr): result = SomePtr(lltype.typeOf(x)) elif isinstance(x, llmemory.fakeaddress): @@ -771,7 +769,6 @@ def delayed_imports(): # import ordering hack - global BUILTIN_ANALYZERS, EXTERNAL_TYPE_ANALYZERS + global BUILTIN_ANALYZERS from pypy.annotation.builtin import BUILTIN_ANALYZERS - from pypy.annotation.builtin import EXTERNAL_TYPE_ANALYZERS Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Wed Nov 21 14:49:08 2007 @@ -361,7 +361,6 @@ # collect all functions import __builtin__, exceptions BUILTIN_ANALYZERS = {} -EXTERNAL_TYPE_ANALYZERS = {} for name, value in globals().items(): if name.startswith('builtin_'): original = getattr(__builtin__, name[8:]) Modified: pypy/dist/pypy/annotation/signature.py ============================================================================== --- pypy/dist/pypy/annotation/signature.py (original) +++ pypy/dist/pypy/annotation/signature.py Wed Nov 21 14:49:08 2007 @@ -66,7 +66,6 @@ def annotationoftype(t, bookkeeper=False): from pypy.annotation.builtin import BUILTIN_ANALYZERS - from pypy.annotation.builtin import EXTERNAL_TYPE_ANALYZERS from pypy.rpython import extregistry """The most precise SomeValue instance that contains all @@ -89,8 +88,6 @@ # can't do tuple elif t is types.NoneType: return s_None - elif t in EXTERNAL_TYPE_ANALYZERS: - return SomeExternalObject(t) elif bookkeeper and extregistry.is_registered_type(t, bookkeeper.policy): entry = extregistry.lookup_type(t, bookkeeper.policy) return entry.compute_annotation_bk(bookkeeper) From rxe at codespeak.net Wed Nov 21 14:53:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 21 Nov 2007 14:53:03 +0100 (CET) Subject: [pypy-svn] r48901 - in pypy/branch/new-genc-tests-wrapper/pypy/translator: c c/test llsupport Message-ID: <20071121135303.3D11F81BF@code0.codespeak.net> Author: rxe Date: Wed Nov 21 14:53:02 2007 New Revision: 48901 Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Log: (pedronis, rxe) add a way to augment the entrypoint for mod wrapping with a mixed level annotator. this approach should be extended for returning values also. Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/genc.py Wed Nov 21 14:53:02 2007 @@ -36,6 +36,9 @@ libraries = [] self.libraries = libraries + def augment_entrypoint(self): + pass + def build_database(self, pyobj_options=None): translator = self.translator @@ -53,6 +56,8 @@ else: stacklesstransformer = None + self.augment_entrypoint() + db = LowLevelDatabase(translator, standalone=self.standalone, gcpolicyclass=gcpolicyclass, stacklesstransformer=stacklesstransformer, @@ -108,7 +113,7 @@ if db is None: db = self.build_database() - self.db = db + self.db = db pf = self.getentrypointptr() pfname = db.get(pf) @@ -171,6 +176,10 @@ self.graph_entrypoint = bk.getdesc(self.entrypoint).getuniquegraph() return getfunctionptr(self.graph_entrypoint) + def augment_entrypoint(self): + from pypy.translator.llsupport import modwrapper + self.entrypoint = modwrapper.augment_entrypoint(self.translator, + self.entrypoint) def compile(self): assert self.c_source_filename assert not self._compiled Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/c/test/test_modwrapper.py Wed Nov 21 14:53:02 2007 @@ -81,6 +81,12 @@ assert fn(2, 1) == 44 assert fn(6, 2) == 90 + def test_argument_string(self): + def fn(s): + return len(s) + fn = self.getcompiled(fn, [str]) + assert fn('aaa') == 3 + class TestWrapperRefcounting(CompilationTestCase, WrapperTests): Modified: pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/translator/llsupport/modwrapper.py Wed Nov 21 14:53:02 2007 @@ -1,11 +1,15 @@ " THIS IS ONLY FOR TESTING " import py +import inspect import ctypes from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import llmemory from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.rpython.lltypesystem.rstr import STR +from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.annotation import model as annmodel +from pypy.rpython.lltypesystem.rffi import charp2str, CCHARP class CtypesModule: """ use ctypes to create a temporary module """ @@ -58,19 +62,7 @@ return res def from_str(arg): - class Chars(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("data", ctypes.c_byte * len(arg))] - class STR(ctypes.Structure): - _fields_ = get_gc_header() + [ - ("hash", ctypes.c_int), - ("chars", Chars)] - s = STR() - s.hash = 0 - s.chars.size = len(arg) - for ii in range(len(arg)): - s.chars.data[ii] = ord(arg[ii]) - return ctypes.addressof(s) + return arg def to_r_uint(res): return {'type':'r_uint', 'value':long(res)} @@ -123,7 +115,7 @@ def array_to_list(res, C_TYPE, action, size=-1): if res: if size == -1: - size = ctypes.cast(res, ctypes.POINTER(ctypes.c_int)).contents.value + size = ctypes.cast(res + gc_header_offset(), ctypes.POINTER(ctypes.c_int)).contents.value class Array(ctypes.Structure): _fields_ = get_gc_header() + [ ("size", ctypes.c_int), @@ -139,6 +131,12 @@ size = ctypes.cast(addr_str, ctypes.POINTER(ctypes.c_int)).contents.value - 1 name = ctypes.string_at(addr_str+4, size) return name + +def gc_header_offset(): + class GcHeader(ctypes.Structure): + _fields_ = get_gc_header() + return ctypes.sizeof(GcHeader) + """ epilog = """ @@ -228,8 +226,9 @@ elif A is lltype.Float: action = 'ctypes.c_double' - elif isinstance(A, lltype.Ptr) and A.TO is STR: - action = 'from_str' + elif isinstance(A, lltype.Ptr) and A == CCHARP: + action = 'identity' + else: assert A in self.TO_CTYPES action = 'identity' @@ -339,3 +338,35 @@ return self[i] else: raise AttributeError, name + +def augment_entrypoint(translator, entrypoint): + bk = translator.annotator.bookkeeper + graph_entrypoint = bk.getdesc(entrypoint).getuniquegraph() + args, varargs, kwds, _ = inspect.getargspec(entrypoint) + assert varargs is None and kwds is None + args_s = [translator.annotator.binding(v) for v in graph_entrypoint.getargs()] + s_result = translator.annotator.binding(graph_entrypoint.getreturnvar()) + converted = [] + converted_s = [] + for v, s_v in zip(args, args_s): + if isinstance(s_v, annmodel.SomeString): + v = "charp2str(%s)" % v + s_v = annmodel.SomePtr(CCHARP) + converted.append(v) + converted_s.append(s_v) + code = """ +def new_entrypoint(%s): + return entrypoint(%s) +""" % (", ".join(args), ", ".join(converted)) + d = dict(entrypoint=entrypoint, charp2str=charp2str) + exec code in d + + mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) + new_entrypoint = d['new_entrypoint'] + + mixlevelannotator.getgraph(new_entrypoint, converted_s, s_result) + mixlevelannotator.finish() + + return new_entrypoint + + From antocuni at codespeak.net Wed Nov 21 15:29:48 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Nov 2007 15:29:48 +0100 (CET) Subject: [pypy-svn] r48902 - in pypy/dist/pypy: interpreter translator/cli/src Message-ID: <20071121142948.0608181BD@code0.codespeak.net> Author: antocuni Date: Wed Nov 21 15:29:47 2007 New Revision: 48902 Modified: pypy/dist/pypy/interpreter/mixedmodule.py pypy/dist/pypy/translator/cli/src/ll_os.cs Log: don't use '*.py' as __file__ for mixed modules, because it hurt's .NET's System.IO.GetFullPath. Modified: pypy/dist/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/dist/pypy/interpreter/mixedmodule.py (original) +++ pypy/dist/pypy/interpreter/mixedmodule.py Wed Nov 21 15:29:47 2007 @@ -126,7 +126,7 @@ assert os.path.basename(fname).startswith('__init__.py') # make it clear that it's not really the interp-level module # at this path that we are seeing, but an app-level version of it - fname = os.path.join(os.path.dirname(fname), '*.py') + fname = os.path.dirname(fname) cls._fname = fname return space.wrap(fname) Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_os.cs Wed Nov 21 15:29:47 2007 @@ -524,6 +524,11 @@ { } + public static string posix__getfullpathname(string path) + { + return System.IO.Path.GetFullPath(path); + } + public static Record_Signed_Signed ll_os_pipe() { Helpers.raise_OSError(Errno.EPERM); // this is only a stub From fijal at codespeak.net Wed Nov 21 15:35:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Nov 2007 15:35:32 +0100 (CET) Subject: [pypy-svn] r48903 - pypy/dist/pypy/translator/c Message-ID: <20071121143532.14B118129@code0.codespeak.net> Author: fijal Date: Wed Nov 21 15:35:31 2007 New Revision: 48903 Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py Log: (fijal, antocuni) Simplify the C backend by accident (we were trying to understand what's going on) Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Wed Nov 21 15:35:31 2007 @@ -126,11 +126,12 @@ db = self.build_database() pf = self.getentrypointptr() pfname = db.get(pf) - + extra_info = extra_information(db) if self.modulename is None: self.modulename = uniquemodulename('testing') modulename = self.modulename targetdir = udir.ensure(modulename, dir=1) + self.targetdir = targetdir defines = defines.copy() if self.config.translation.countmallocs: @@ -149,7 +150,8 @@ defines = defines, exports = self.exports, symboltable = self.symboltable, - libraries = self.libraries) + libraries = self.libraries, + extra_info = extra_info) else: if self.config.translation.instrument: defines['INSTRUMENT'] = 1 @@ -159,7 +161,8 @@ cfile, extra, include_dirs, library_dirs = \ gen_source_standalone(db, modulename, targetdir, entrypointname = pfname, - defines = defines) + defines = defines, + extra_info = extra_info) self.c_source_filename = py.path.local(cfile) self.extrafiles = extra self.include_dirs = include_dirs.keys() @@ -388,15 +391,6 @@ # graph.optimize() # self.funcnodes = graph.ordered_funcnodes() - def write_extra_sources(self, sources): - basename = 'additional_node.c' - retval = [] - for source in sorted(sources.keys()): - f = self.makefile(self.uniquecname(basename)) - for include in sources[source]: - print >>f, "#include <%s>" % (include,) - print >>f, str(source) - def uniquecname(self, name): assert name.endswith('.c') return self.namespace.uniquename(name[:-2]) + '.c' @@ -633,24 +627,17 @@ include_dirs = {} library_dirs = {} for node in database.globalcontainers(): - if hasattr(node, 'includes'): - for include in node.includes: - includes[include] = True - if hasattr(node, 'sources'): - for source in node.sources: - sources[source] = getattr(node, 'includes', []) - if hasattr(node, 'include_dirs'): - for include_dir in node.include_dirs: - include_dirs[include_dir] = True - if hasattr(node, 'library_dirs'): - for library_dir in node.library_dirs: - library_dirs[library_dir] = True + for attrname in ['includes', 'sources', 'include_dirs', 'library_dirs']: + if hasattr(node, attrname): + for elem in getattr(node, attrname): + locals()[attrname][elem] = True includes = includes.keys() includes.sort() - return includes, sources, include_dirs, library_dirs + return {'includes':includes, 'sources':sources, + 'include_dirs':include_dirs, 'library_dirs':library_dirs} def gen_source_standalone(database, modulename, targetdir, - entrypointname, defines={}): + entrypointname, defines={}, extra_info={}): assert database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -678,9 +665,10 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes, sources, include_dirs, library_dirs = extra_information(database) - for include in includes: + for include in extra_info.get('includes', []): print >> fi, '#include <%s>' % (include,) + for source in extra_info.get('sources', []): + print >> f, source fi.close() preimplementationlines = list( @@ -694,7 +682,6 @@ sg.set_strategy(targetdir) database.prepare_inline_helpers() sg.gen_readable_parts_of_source(f) - sg.write_extra_sources(sources) # 3) start-up code print >> f @@ -708,11 +695,12 @@ print >>fi, "#define INSTRUMENT_NCOUNTER %d" % n fi.close() - return filename, sg.getextrafiles(), include_dirs, library_dirs + return filename, sg.getextrafiles(), extra_info.get('include_dirs', None),\ + extra_info.get('library_dirs', None) def gen_source(database, modulename, targetdir, defines={}, exports={}, - symboltable=None, libraries=[]): + symboltable=None, libraries=[], extra_info={}): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -738,9 +726,10 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes, sources, include_dirs, library_dirs = extra_information(database) - for include in includes: + for include in extra_info.get('includes', []): print >> fi, '#include <%s>' % (include,) + for source in extra_info.get('sources', []): + print >> f, source fi.close() if database.translator is None or database.translator.rtyper is None: @@ -756,7 +745,6 @@ sg = SourceGenerator(database, preimplementationlines) sg.set_strategy(targetdir) sg.gen_readable_parts_of_source(f) - sg.write_extra_sources(sources) # # Debugging info @@ -867,10 +855,13 @@ # pypy_include_dir = autopath.this_dir f = targetdir.join('setup.py').open('w') + include_dirs = extra_info.get('include_dirs', []) + library_dirs = extra_info.get('library_dirs', []) f.write(SETUP_PY % locals()) f.close() - return filename, sg.getextrafiles(), include_dirs, library_dirs + return filename, sg.getextrafiles(), include_dirs,\ + library_dirs SETUP_PY = ''' Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Wed Nov 21 15:35:31 2007 @@ -702,12 +702,9 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - if hasattr(obj, 'libraries'): - self.libraries = obj.libraries - if hasattr(obj, 'include_dirs'): - self.include_dirs = obj.include_dirs - if hasattr(obj, 'library_dirs'): - self.library_dirs = obj.library_dirs + for attrname in 'libraries', 'include_dirs', 'includes', 'sources', 'library_dirs': + if hasattr(obj, attrname): + setattr(self, attrname, getattr(obj, attrname)) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) From fijal at codespeak.net Wed Nov 21 15:40:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Nov 2007 15:40:44 +0100 (CET) Subject: [pypy-svn] r48904 - in pypy/dist/pypy: module/fcntl module/rctime rlib rpython/lltypesystem rpython/lltypesystem/module rpython/lltypesystem/test rpython/module translator/tool Message-ID: <20071121144044.7C2C28129@code0.codespeak.net> Author: fijal Date: Wed Nov 21 15:40:43 2007 New Revision: 48904 Added: pypy/dist/pypy/rlib/rposix.py (contents, props changed) Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py pypy/dist/pypy/module/rctime/interp_time.py pypy/dist/pypy/rlib/_rsocket_rffi.py pypy/dist/pypy/rlib/rmmap.py pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/ll_os_environ.py pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/rpython/module/ll_strtod.py pypy/dist/pypy/rpython/module/ll_time.py pypy/dist/pypy/translator/tool/cbuild.py Log: (fijal, antocuni) * Untangle a bit cexternvariable * Move rffi.get/set_errno to rlib/rposix Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/interp_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/interp_fcntl.py Wed Nov 21 15:40:43 2007 @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace +from pypy.rlib import rposix import sys class CConfig: @@ -67,7 +68,7 @@ c_flock = external('flock', [rffi.INT, rffi.INT], rffi.INT) def _get_error_msg(): - errno = rffi.get_errno() + errno = rposix.get_errno() return rffi.charp2str(strerror(errno)) def _get_module_object(space, obj_name): Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Wed Nov 21 15:40:43 2007 @@ -4,6 +4,7 @@ from pypy.interpreter.baseobjspace import W_Root, ObjSpace from pypy.rpython.lltypesystem import lltype from pypy.rlib.rarithmetic import ovfcheck_float_to_int +from pypy.rlib import rposix import math import os import sys @@ -135,7 +136,7 @@ return timezone, daylight, tzname, altzone def _get_error_msg(): - errno = rffi.get_errno() + errno = rposix.get_errno() return os.strerror(errno) def sleep(secs): Modified: pypy/dist/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/dist/pypy/rlib/_rsocket_rffi.py Wed Nov 21 15:40:43 2007 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rffi import CCHARP -from pypy.rpython.lltypesystem.rffi import get_errno as geterrno +from pypy.rlib.rposix import get_errno as geterrno from pypy.rlib.rarithmetic import intmask, r_uint import os Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Wed Nov 21 15:40:43 2007 @@ -1,6 +1,7 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import rffi, lltype, llmemory +from pypy.rlib import rposix import sys import os @@ -111,7 +112,7 @@ _get_page_size = external('getpagesize', [], rffi.INT) def _get_error_no(): - return rffi.get_errno() + return rposix.get_errno() elif _MS_WINDOWS: Added: pypy/dist/pypy/rlib/rposix.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rposix.py Wed Nov 21 15:40:43 2007 @@ -0,0 +1,20 @@ +from pypy.rpython.lltypesystem.rffi import CConstant, CExternVariable +from pypy.rpython.lltypesystem import lltype, ll2ctypes + +class CConstantErrno(CConstant): + # these accessors are used when calling get_errno() or set_errno() + # on top of CPython + def __getitem__(self, index): + assert index == 0 + try: + return ll2ctypes.TLS.errno + except AttributeError: + raise ValueError("no C function call occurred so far, " + "errno is undefined") + def __setitem__(self, index, value): + assert index == 0 + ll2ctypes.TLS.errno = value + +get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', CConstantErrno, + includes=['errno.h']) + Modified: pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py Wed Nov 21 15:40:43 2007 @@ -14,6 +14,7 @@ from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.annotation import model as annmodel +from pypy.translator.tool.cbuild import cache_c_module def uaddressof(obj): @@ -511,6 +512,16 @@ # ____________________________________________ + +def compile_c_snippet(name, source): + from pypy.tool.udir import udir + cname = udir.join(name + '.c') + f = cname.open('w') + f.write(source) + f.write('\n') + f.close() + return cache_c_module([cname], name) + def get_ctypes_callable(funcptr, calling_conv): if not ctypes: raise ImportError("ctypes is needed to use ll2ctypes") @@ -523,13 +534,16 @@ except AttributeError: pass - if getattr(funcptr._obj, 'sources', None): - # give up - for tests with an inlined bit of C code - raise NotImplementedError("cannot call a C function defined in " - "a custom C source snippet") + sources = getattr(funcptr._obj, 'sources', None) + if sources: + assert len(sources) == 1 + dllname = compile_c_snippet(funcptr._obj._name, sources[0]) + libraries = [dllname] + else: + libraries = getattr(funcptr._obj, 'libraries', None) + FUNCTYPE = lltype.typeOf(funcptr).TO funcname = funcptr._obj._name - libraries = getattr(funcptr._obj, 'libraries', None) if not libraries: cfunc = get_on_lib(standard_c_lib, funcname) # XXX magic: on Windows try to load the function from 'kernel32' too @@ -688,7 +702,7 @@ # should have in C. We have to save it away from one external C function # call to the next. Otherwise a non-zero value left behind will confuse # CPython itself a bit later, and/or CPython will stamp on it before we -# try to inspect it via rffi.get_errno(). +# try to inspect it via rposix.get_errno(). TLS = tlsobject() # helpers to save/restore the C-level errno -- platform-specific because Modified: pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py Wed Nov 21 15:40:43 2007 @@ -3,6 +3,7 @@ import py from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name +from pypy.rlib import rposix math_frexp = rffi.llexternal('frexp', [rffi.DOUBLE, rffi.INTP], rffi.DOUBLE, sandboxsafe=True) @@ -40,11 +41,11 @@ return r def _error_reset(): - rffi.set_errno(0) + rposix.set_errno(0) ERANGE = errno.ERANGE def _check_error(x): - errno = rffi.get_errno() + errno = rposix.get_errno() if errno: if errno == ERANGE: if not x: Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Nov 21 15:40:43 2007 @@ -242,40 +242,51 @@ def COpaquePtr(*args, **kwds): return lltype.Ptr(COpaque(*args, **kwds)) -def CExternVariable(TYPE, name, _CConstantClass=CConstant): +def CExternVariable(TYPE, name, _CConstantClass=CConstant, includes=[], include_dirs=[]): """Return a pair of functions - a getter and a setter - to access the given global C variable. """ - # XXX THIS IS ONLY A QUICK HACK TO MAKE IT WORK - # In general, we need to re-think a few things to be more consistent, - # e.g. what if a CStruct, COpaque or CExternVariable requires - # some #include... - assert not isinstance(TYPE, lltype.ContainerType) - CTYPE = lltype.FixedSizeArray(TYPE, 1) - c_variable_ref = _CConstantClass('(&%s)' % (name,), lltype.Ptr(CTYPE)) - def getter(): - return c_variable_ref[0] - def setter(newvalue): - c_variable_ref[0] = newvalue - return (func_with_new_name(getter, '%s_getter' % (name,)), - func_with_new_name(setter, '%s_setter' % (name,))) - - -class CConstantErrno(CConstant): - # these accessors are used when calling get_errno() or set_errno() - # on top of CPython - def __getitem__(self, index): - assert index == 0 - try: - return ll2ctypes.TLS.errno - except AttributeError: - raise ValueError("no C function call occurred so far, " - "errno is undefined") - def __setitem__(self, index, value): - assert index == 0 - ll2ctypes.TLS.errno = value - -get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', CConstantErrno) + from pypy.translator.c.primitive import PrimitiveType + # XXX we cannot really enumerate all C types here, do it on a case-by-case + # basis + if TYPE == CCHARPP: + c_type = 'char **' + elif TYPE == CCHARP: + c_type = 'char *' + else: + c_type = PrimitiveType[TYPE] + assert c_type.endswith(' @') + c_type = c_type[:-2] # cut the trailing ' @' + + getter_name = 'get_' + name + setter_name = 'set_' + name + c_getter = "%(c_type)s %(getter_name)s () { return %(name)s; }" % locals() + c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals() + + lines = ["#include <%s>" % i for i in includes] + lines.append(c_getter) + lines.append(c_setter) + sources = ('\n'.join(lines),) + + kwds = {'includes': includes, 'sources':sources, + 'include_dirs':include_dirs} + getter = llexternal(getter_name, [], TYPE, **kwds) + setter = llexternal(setter_name, [TYPE], lltype.Void, **kwds) + return getter, setter + +## # XXX THIS IS ONLY A QUICK HACK TO MAKE IT WORK +## # In general, we need to re-think a few things to be more consistent, +## # e.g. what if a CStruct, COpaque or CExternVariable requires +## # some #include... +## assert not isinstance(TYPE, lltype.ContainerType) +## CTYPE = lltype.FixedSizeArray(TYPE, 1) +## c_variable_ref = _CConstantClass('(&%s)' % (name,), lltype.Ptr(CTYPE)) +## def getter(): +## return c_variable_ref[0] +## def setter(newvalue): +## c_variable_ref[0] = newvalue +## return (func_with_new_name(getter, '%s_getter' % (name,)), +## func_with_new_name(setter, '%s_setter' % (name,))) # char, represented as a Python character # (use SIGNEDCHAR or UCHAR for the small integer types) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Wed Nov 21 15:40:43 2007 @@ -8,7 +8,7 @@ from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.annlowlevel import llhelper - +from pypy.rlib import rposix class TestLL2Ctypes(object): @@ -598,7 +598,7 @@ # the next line is a random external function call, # to check that it doesn't reset errno strlen("hi!") - err = rffi.get_errno() + err = rposix.get_errno() import errno assert err == errno.EBADF assert not ALLOCATED # detects memory leaks in the test @@ -676,3 +676,8 @@ fcntl_str(12345, 3, "xxx") fcntl_int(12345, 1, 0) + def test_llexternal_source(self): + fn = rffi.llexternal('fn', [], rffi.INT, sources = ["int fn() { return 42; }"]) + res = fn() + assert res == 42 + Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Wed Nov 21 15:40:43 2007 @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem.rffi import * +from pypy.rlib.rposix import get_errno, set_errno from pypy.translator.c.test.test_genc import compile from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc from pypy.rpython.lltypesystem import lltype @@ -287,8 +288,7 @@ assert offsetof(S, "c_b") == struct.calcsize("hi") - struct.calcsize("i") assert offsetof(S, "c_c") == struct.calcsize("hii") - struct.calcsize("i") -def test_prebuild_constant(): - py.test.skip("WIP") +def test_prebuilt_constant(): h_source = py.code.Source(""" #ifndef _CONSTANTS #define _CONSTANTS Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Nov 21 15:40:43 2007 @@ -16,6 +16,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype from pypy.rpython.tool import rffi_platform as platform +from pypy.rlib import rposix posix = __import__(os.name) if sys.platform.startswith('win'): @@ -72,7 +73,7 @@ def c_func_llimpl(): res = rffi.cast(rffi.LONG, c_func()) if res == -1: - raise OSError(rffi.get_errno(), "%s failed" % name) + raise OSError(rposix.get_errno(), "%s failed" % name) return res c_func_llimpl.func_name = name + '_llimpl' @@ -84,7 +85,7 @@ def c_func_llimpl(arg): res = rffi.cast(rffi.LONG, c_func(arg)) if res == -1: - raise OSError(rffi.get_errno(), "%s failed" % name) + raise OSError(rposix.get_errno(), "%s failed" % name) c_func_llimpl.func_name = name + '_llimpl' @@ -100,7 +101,7 @@ l_args = rffi.liststr2charpp(args) os_execv(path, l_args) rffi.free_charpp(l_args) - raise OSError(rffi.get_errno(), "execv failed") + raise OSError(rposix.get_errno(), "execv failed") return extdef([str, [str]], s_ImpossibleValue, llimpl=execv_llimpl, export_name="ll_os.ll_os_execv") @@ -117,7 +118,7 @@ childpid = os_spawnv(mode, path, l_args) rffi.free_charpp(l_args) if childpid == -1: - raise OSError(rffi.get_errno(), "os_spawnv failed") + raise OSError(rposix.get_errno(), "os_spawnv failed") return rffi.cast(lltype.Signed, childpid) return extdef([int, str, [str]], int, llimpl=spawnv_llimpl, @@ -130,7 +131,7 @@ def dup_llimpl(fd): newfd = rffi.cast(lltype.Signed, os_dup(rffi.cast(rffi.INT, fd))) if newfd == -1: - raise OSError(rffi.get_errno(), "dup failed") + raise OSError(rposix.get_errno(), "dup failed") return newfd return extdef([int], int, llimpl=dup_llimpl, @@ -145,7 +146,7 @@ error = rffi.cast(lltype.Signed, os_dup2(rffi.cast(rffi.INT, fd), rffi.cast(rffi.INT, newfd))) if error == -1: - raise OSError(rffi.get_errno(), "dup2 failed") + raise OSError(rposix.get_errno(), "dup2 failed") return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") @@ -204,7 +205,7 @@ actime, modtime = tp error = os_utime_platform(path, actime, modtime) if error == -1: - raise OSError(rffi.get_errno(), "os_utime failed") + raise OSError(rposix.get_errno(), "os_utime failed") os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' s_string = SomeString() @@ -287,7 +288,7 @@ try: result = os_times(l_tmsbuf) if result == rffi.cast(self.CLOCK_T, -1): - raise OSError(rffi.get_errno(), "times failed") + raise OSError(rposix.get_errno(), "times failed") return ( l_tmsbuf.c_tms_utime / CLOCK_TICKS_PER_SECOND, l_tmsbuf.c_tms_stime / CLOCK_TICKS_PER_SECOND, @@ -306,7 +307,7 @@ def setsid_llimpl(): result = rffi.cast(lltype.Signed, os_setsid()) if result == -1: - raise OSError(rffi.get_errno(), "os_setsid failed") + raise OSError(rposix.get_errno(), "os_setsid failed") return result return extdef([], int, export_name="ll_os.ll_os_setsid", @@ -333,7 +334,7 @@ l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw') result = os_uname(l_utsbuf) if result == -1: - raise OSError(rffi.get_errno(), "os_uname failed") + raise OSError(rposix.get_errno(), "os_uname failed") retval = ( rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_sysname)), rffi.charp2str(rffi.cast(rffi.CCHARP, l_utsbuf.c_nodename)), @@ -380,7 +381,7 @@ def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: - raise OSError(rffi.get_errno(), "os_open failed") + raise OSError(rposix.get_errno(), "os_open failed") return result def os_open_oofakeimpl(o_path, flags, mode): @@ -405,7 +406,7 @@ got = rffi.cast(lltype.Signed, os_read(rffi.cast(rffi.INT, fd), inbuf, rffi.cast(rffi.SIZE_T, count))) if got < 0: - raise OSError(rffi.get_errno(), "os_read failed") + raise OSError(rposix.get_errno(), "os_read failed") # XXX too many copies of the data! l = [inbuf[i] for i in range(got)] finally: @@ -434,7 +435,7 @@ rffi.cast(rffi.INT, fd), outbuf, rffi.cast(rffi.SIZE_T, count))) if written < 0: - raise OSError(rffi.get_errno(), "os_write failed") + raise OSError(rposix.get_errno(), "os_write failed") finally: lltype.free(outbuf, flavor='raw') return written @@ -453,7 +454,7 @@ def close_llimpl(fd): error = rffi.cast(lltype.Signed, os_close(rffi.cast(rffi.INT, fd))) if error == -1: - raise OSError(rffi.get_errno(), "close failed") + raise OSError(rposix.get_errno(), "close failed") return extdef([int], s_None, llimpl=close_llimpl, export_name="ll_os.ll_os_close", oofakeimpl=os.close) @@ -492,7 +493,7 @@ rffi.cast(rffi.INT, how)) res = rffi.cast(lltype.SignedLongLong, res) if res < 0: - raise OSError(rffi.get_errno(), "os_lseek failed") + raise OSError(rposix.get_errno(), "os_lseek failed") return res def os_lseek_oofakeimpl(fd, pos, how): @@ -515,7 +516,7 @@ os_ftruncate(rffi.cast(rffi.INT, fd), rffi.cast(rffi.LONGLONG, length))) if res < 0: - raise OSError(rffi.get_errno(), "os_lseek failed") + raise OSError(rposix.get_errno(), "os_lseek failed") return extdef([int, r_longlong], s_None, llimpl = ftruncate_llimpl, @@ -602,7 +603,7 @@ res = os_getcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) if res: break # ok - error = rffi.get_errno() + error = rposix.get_errno() lltype.free(buf, flavor='raw') if error != errno.ERANGE: raise OSError(error, "getcwd failed") @@ -710,13 +711,13 @@ def os_listdir_llimpl(path): dirp = os_opendir(path) if not dirp: - raise OSError(rffi.get_errno(), "os_opendir failed") - rffi.set_errno(0) + raise OSError(rposix.get_errno(), "os_opendir failed") + rposix.set_errno(0) result = [] while True: direntp = os_readdir(dirp) if not direntp: - error = rffi.get_errno() + error = rposix.get_errno() break namep = rffi.cast(rffi.CCHARP, direntp.c_d_name) name = rffi.charp2str(namep) @@ -773,7 +774,7 @@ write_fd = filedes[1] lltype.free(filedes, flavor='raw') if error != 0: - raise OSError(rffi.get_errno(), "os_pipe failed") + raise OSError(rposix.get_errno(), "os_pipe failed") return (read_fd, write_fd) return extdef([], (int, int), @@ -796,7 +797,7 @@ res = os_readlink(l_path, buf, bufsize) lltype.free(l_path, flavor='raw') if res < 0: - error = rffi.get_errno() # failed + error = rposix.get_errno() # failed lltype.free(buf, flavor='raw') raise OSError(error, "readlink failed") elif res < bufsize: @@ -843,7 +844,7 @@ status = status_p[0] lltype.free(status_p, flavor='raw') if result == -1: - raise OSError(rffi.get_errno(), "os_waitpid failed") + raise OSError(rposix.get_errno(), "os_waitpid failed") return (rffi.cast(lltype.Signed, result), rffi.cast(lltype.Signed, status)) @@ -893,7 +894,7 @@ def unlink_llimpl(pathname): res = os_unlink(pathname) if res < 0: - raise OSError(rffi.get_errno(), "os_unlink failed") + raise OSError(rposix.get_errno(), "os_unlink failed") return extdef([str], s_None, llimpl=unlink_llimpl, export_name="ll_os.ll_os_unlink") @@ -905,7 +906,7 @@ def chdir_llimpl(path): res = os_chdir(path) if res < 0: - raise OSError(rffi.get_errno(), "os_chdir failed") + raise OSError(rposix.get_errno(), "os_chdir failed") return extdef([str], s_None, llimpl=chdir_llimpl, export_name="ll_os.ll_os_chdir") @@ -926,7 +927,7 @@ else: res = os_mkdir(pathname, mode) if res < 0: - raise OSError(rffi.get_errno(), "os_mkdir failed") + raise OSError(rposix.get_errno(), "os_mkdir failed") return extdef([str, int], s_None, llimpl=mkdir_llimpl, export_name="ll_os.ll_os_mkdir") @@ -938,7 +939,7 @@ def rmdir_llimpl(pathname): res = os_rmdir(pathname) if res < 0: - raise OSError(rffi.get_errno(), "os_rmdir failed") + raise OSError(rposix.get_errno(), "os_rmdir failed") return extdef([str], s_None, llimpl=rmdir_llimpl, export_name="ll_os.ll_os_rmdir") @@ -951,7 +952,7 @@ def chmod_llimpl(path, mode): res = os_chmod(path, rffi.cast(rffi.MODE_T, mode)) if res < 0: - raise OSError(rffi.get_errno(), "os_chmod failed") + raise OSError(rposix.get_errno(), "os_chmod failed") return extdef([str, int], s_None, llimpl=chmod_llimpl, export_name="ll_os.ll_os_chmod") @@ -964,7 +965,7 @@ def rename_llimpl(oldpath, newpath): res = os_rename(oldpath, newpath) if res < 0: - raise OSError(rffi.get_errno(), "os_rename failed") + raise OSError(rposix.get_errno(), "os_rename failed") return extdef([str, str], s_None, llimpl=rename_llimpl, export_name="ll_os.ll_os_rename") @@ -989,7 +990,7 @@ res = os_kill(rffi.cast(rffi.PID_T, pid), rffi.cast(rffi.INT, sig)) if res < 0: - raise OSError(rffi.get_errno(), "os_kill failed") + raise OSError(rposix.get_errno(), "os_kill failed") return extdef([int, int], s_None, llimpl=kill_llimpl, export_name="ll_os.ll_os_kill") @@ -1002,7 +1003,7 @@ def link_llimpl(oldpath, newpath): res = os_link(oldpath, newpath) if res < 0: - raise OSError(rffi.get_errno(), "os_link failed") + raise OSError(rposix.get_errno(), "os_link failed") return extdef([str, str], s_None, llimpl=link_llimpl, export_name="ll_os.ll_os_link") @@ -1015,7 +1016,7 @@ def symlink_llimpl(oldpath, newpath): res = os_symlink(oldpath, newpath) if res < 0: - raise OSError(rffi.get_errno(), "os_symlink failed") + raise OSError(rposix.get_errno(), "os_symlink failed") return extdef([str, str], s_None, llimpl=symlink_llimpl, export_name="ll_os.ll_os_symlink") @@ -1027,7 +1028,7 @@ def fork_llimpl(): childpid = os_fork() if childpid == -1: - raise OSError(rffi.get_errno(), "os_fork failed") + raise OSError(rposix.get_errno(), "os_fork failed") return rffi.cast(lltype.Signed, childpid) return extdef([], int, llimpl=fork_llimpl, @@ -1102,7 +1103,7 @@ def ttyname_llimpl(fd): l_name = os_ttyname(fd) if not l_name: - raise OSError(rffi.get_errno(), "ttyname raised") + raise OSError(rposix.get_errno(), "ttyname raised") return rffi.charp2str(l_name) return extdef([int], str, "ll_os.ttyname", Modified: pypy/dist/pypy/rpython/module/ll_os_environ.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_environ.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_environ.py Wed Nov 21 15:40:43 2007 @@ -3,6 +3,7 @@ from pypy.rpython.controllerentry import Controller from pypy.rpython.extfunc import register_external from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rlib import rposix # ____________________________________________________________ # @@ -84,7 +85,7 @@ error = os_putenv(l_string) if error: rffi.free_charp(l_string) - raise OSError(rffi.get_errno(), "os_putenv failed") + raise OSError(rposix.get_errno(), "os_putenv failed") # keep 'l_string' alive - we know that the C library needs it # until the next call to putenv() with the same 'name'. l_oldstring = envkeepalive.byname.get(name, lltype.nullptr(rffi.CCHARP.TO)) @@ -115,7 +116,7 @@ error = os_unsetenv(l_name) # 'error' is None on OS/X rffi.free_charp(l_name) if error: - raise OSError(rffi.get_errno(), "os_unsetenv failed") + raise OSError(rposix.get_errno(), "os_unsetenv failed") try: l_oldstring = envkeepalive.byname[name] except KeyError: Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Nov 21 15:40:43 2007 @@ -10,6 +10,7 @@ from pypy.rpython.extfunc import register_external from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE +from pypy.rlib import rposix # XXX on Windows, stat() is flawed; see CPython's posixmodule.c for # an implementation based on the Win32 API @@ -208,7 +209,7 @@ if arg_is_path: rffi.free_charp(arg) if error != 0: - raise OSError(rffi.get_errno(), "os_?stat failed") + raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Wed Nov 21 15:40:43 2007 @@ -6,6 +6,7 @@ from pypy.translator.tool.cbuild import cache_c_module from pypy.tool.autopath import pypydir from pypy.rpython.ootypesystem import ootype +from pypy.rlib import rposix class CConfig: _includes_ = ['src/ll_strtod.h'] @@ -51,7 +52,7 @@ def llimpl(sign, beforept, afterpt, exponent): res = ll_parts_to_float(sign, beforept, afterpt, exponent) - if res == -1 and rffi.get_errno() == 42: + if res == -1 and rposix.get_errno() == 42: raise ValueError("Wrong literal for float") return res Modified: pypy/dist/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_time.py (original) +++ pypy/dist/pypy/rpython/module/ll_time.py Wed Nov 21 15:40:43 2007 @@ -8,6 +8,7 @@ from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem import lltype from pypy.rpython.extfunc import BaseLazyRegistering, registering, extdef +from pypy.rlib import rposix class CConfig: if sys.platform.startswith('win'): @@ -163,9 +164,9 @@ t.c_tv_sec = int(secs) t.c_tv_usec = int(frac*1000000.0) if rffi.cast(rffi.LONG, c_select(0, void, void, void, t)) != 0: - errno = rffi.get_errno() + errno = rposix.get_errno() if errno != EINTR: - raise OSError(rffi.get_errno(), "Select failed") + raise OSError(rposix.get_errno(), "Select failed") finally: lltype.free(t, flavor='raw') Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Wed Nov 21 15:40:43 2007 @@ -158,6 +158,7 @@ modname = str(cache_dir.join(modname)) compile_c_module(cfiles, modname, include_dirs=include_dirs, libraries=libraries) + return modname + '.so' def make_module_from_c(cfile, include_dirs=None, libraries=[]): cfile = py.path.local(cfile) From cfbolz at codespeak.net Wed Nov 21 15:56:56 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 15:56:56 +0100 (CET) Subject: [pypy-svn] r48905 - pypy/branch/new-genc-tests-wrapper/pypy/rlib Message-ID: <20071121145656.5C7DE814B@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 15:56:54 2007 New Revision: 48905 Modified: pypy/branch/new-genc-tests-wrapper/pypy/rlib/rmarshal.py Log: add another way to construct a marshaller directly from an annotation Modified: pypy/branch/new-genc-tests-wrapper/pypy/rlib/rmarshal.py ============================================================================== --- pypy/branch/new-genc-tests-wrapper/pypy/rlib/rmarshal.py (original) +++ pypy/branch/new-genc-tests-wrapper/pypy/rlib/rmarshal.py Wed Nov 21 15:56:54 2007 @@ -23,6 +23,11 @@ with new data when the marshaller is called. """ s_obj = annotation(type, None) + return _get_marshaller_s_obj(s_obj) +get_marshaller._annspecialcase_ = 'specialize:memo' + + +def _get_marshaller_s_obj(s_obj): try: # look for a marshaller in the 'dumpers' list return find_dumper(s_obj) @@ -30,7 +35,6 @@ # ask the annotation to produce an appropriate dumper pair(_tag, s_obj).install_marshaller() return find_dumper(s_obj) -get_marshaller._annspecialcase_ = 'specialize:memo' def get_loader(type): s_obj = annotation(type, None) From rxe at codespeak.net Wed Nov 21 16:04:27 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 21 Nov 2007 16:04:27 +0100 (CET) Subject: [pypy-svn] r48906 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071121150427.1A1F48160@code0.codespeak.net> Author: rxe Date: Wed Nov 21 16:04:26 2007 New Revision: 48906 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: (fijal, antocuni, rxe) remove monkey patching hack Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Nov 21 16:04:26 2007 @@ -221,8 +221,7 @@ for c_source in c_sources: ccode.append('\n') - for l in c_source: - ccode.append(l + '\n') + ccode.append(c_source + '\n') ccode.append('\n') # append our source file Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Nov 21 16:04:26 2007 @@ -39,33 +39,8 @@ del _ext_modules[:-leave] else: del _ext_modules[:] - -def setup_module(mod): - from pypy.rpython.lltypesystem import lltype - from pypy.rpython.lltypesystem.rffi import llexternal - - c_source = py.code.Source(""" - int get_errno() { - return errno; - } - """) - get_errno = llexternal('get_errno', [], lltype.Signed, sources=[c_source]) - - c_source = py.code.Source(""" - void set_errno(int _errno) { - errno = _errno; - } - """) - set_errno = llexternal('set_errno', [lltype.Signed], lltype.Void, sources=[c_source]) - global _get_errno, _set_errno - import pypy.rpython.lltypesystem.rffi - pypy.rpython.lltypesystem.rffi.get_errno, _get_errno = get_errno, pypy.rpython.lltypesystem.rffi.get_errno - pypy.rpython.lltypesystem.rffi.set_errno, _set_errno= set_errno, pypy.rpython.lltypesystem.rffi.set_errno def teardown_module(mod): - import pypy.rpython.lltypesystem.rffi - pypy.rpython.lltypesystem.rffi.get_errno = _get_errno - pypy.rpython.lltypesystem.rffi.set_errno = _set_errno _cleanup() def llvm_test(): From fijal at codespeak.net Wed Nov 21 16:32:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Nov 2007 16:32:21 +0100 (CET) Subject: [pypy-svn] r48907 - in pypy/dist/pypy/rpython: . module Message-ID: <20071121153221.1AC9B8197@code0.codespeak.net> Author: fijal Date: Wed Nov 21 16:32:20 2007 New Revision: 48907 Modified: pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/module/ll_strtod.py Log: Get rid of a hack, not needed any more Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Wed Nov 21 16:32:20 2007 @@ -92,7 +92,8 @@ class BaseLazyRegistering(object): __metaclass__ = LazyRegisteringMeta - __ATTRIBUTES = ['includes', 'include_dirs', 'libraries', 'library_dirs'] + __ATTRIBUTES = ['includes', 'include_dirs', 'libraries', 'library_dirs', + 'sources'] def configure(self, CConfig): classes_seen = self.__dict__.setdefault('__classes_seen', {}) Modified: pypy/dist/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/ll_strtod.py Wed Nov 21 16:32:20 2007 @@ -10,20 +10,10 @@ class CConfig: _includes_ = ['src/ll_strtod.h'] + _sources_ = ['#include '] class RegisterStrtod(BaseLazyRegistering): def __init__(self): - # HACK HACK HACK - # we need to have some sane way of doing stuff below - # problem: we don't have a way to call things in our header files - from pypy.tool.udir import udir - c_file = udir.join('test_strtod.c') - c_file.write(py.code.Source(""" - #include - """)) - cache_c_module([c_file], '_ll_strtod') - self._libraries_ = [str(py.path.local(pypydir).join('_cache', - '_ll_strtod.so'))] self.configure(CConfig) @registering(rarithmetic.formatd) From ac at codespeak.net Wed Nov 21 17:05:03 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Wed, 21 Nov 2007 17:05:03 +0100 (CET) Subject: [pypy-svn] r48908 - in pypy/branch/dist-future-fixing/pypy/interpreter: . pyparser pyparser/test Message-ID: <20071121160503.C6E1B81C2@code0.codespeak.net> Author: ac Date: Wed Nov 21 17:05:03 2007 New Revision: 48908 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py Log: (jacob, arre) __future__ imports and 'with'-statements work again. Yay! Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py Wed Nov 21 17:05:03 2007 @@ -2,6 +2,8 @@ General classes for bytecode compilers. Compiler instances are stored into 'space.getexecutioncontext().compiler'. """ + +import sys from codeop import PyCF_DONT_IMPLY_DEDENT from pypy.interpreter.error import OperationError @@ -111,7 +113,7 @@ else: return 0 -from pypy.interpreter.pyparser.future import futureFlags +from pypy.interpreter.pyparser import future class CPythonCompiler(PyCodeCompiler): """Faked implementation of a compiler, using the underlying compile().""" @@ -119,8 +121,10 @@ def __init__(self, space): self.space = space self.w_compile_hook = space.w_None - - self.compiler_flags = futureFlags.allowed_flags + if sys.version_info >= (2.5): + self.compiler_flags = future.futureFlags_2_5.allowed_flags + else: + self.compiler_flags = future.futureFlags_2_4.allowed_flags def compile(self, source, filename, mode, flags): from pypy.tool import stdlib___future__ @@ -218,8 +222,11 @@ self.grammar_version = override_version or space.config.objspace.pyversion self.parser = make_pyparser(self.grammar_version) self.additional_rules = {} - - self.compiler_flags = futureFlags.allowed_flags + if self.grammar_version >= '2.5': + self.futureFlags = future.futureFlags_2_5 + else: + self.futureFlags = future.futureFlags_2_4 + self.compiler_flags = self.futureFlags.allowed_flags def compile(self, source, filename, mode, flags): from pyparser.error import SyntaxError @@ -241,7 +248,7 @@ for rulename, buildfunc in self.additional_rules.iteritems(): assert isinstance(buildfunc, Function) builder.user_build_rules[rulename] = buildfunc - flags |= getFutures(source) + flags |= getFutures(self.futureFlags, source) self.parser.parse_source(source, mode, builder, flags) ast_tree = builder.rule_stack[-1] encoding = builder.source_encoding @@ -261,7 +268,7 @@ raise try: astcompiler.misc.set_filename(filename, ast_tree) - flag_names = futureFlags.get_flag_names(space, flags) + flag_names = self.futureFlags.get_flag_names(space, flags) if mode == 'exec': codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names) elif mode == 'single': Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py Wed Nov 21 17:05:03 2007 @@ -869,13 +869,6 @@ names.append((name, as_name)) if index < l: # case ',' index += 1 -## if from_name == '__future__': -## for name, asname in names: -## if name == 'with_statement': -## # found from __future__ import with_statement -## if not builder.with_enabled: -## builder.enable_with() -## #raise pythonparse.AlternateGrammarException() builder.push(ast.From(from_name, names, atoms[0].lineno)) @@ -1069,17 +1062,9 @@ self.rule_stack = [] self.space = space self.source_encoding = None -## self.with_enabled = False self.build_rules = ASTRULES_Template self.user_build_rules = {} -## def enable_with(self): -## if self.with_enabled: -## return -## self.with_enabled = True -## # XXX -## # self.keywords.update({'with':None, 'as': None}) - def context(self): return AstBuilderContext(self.rule_stack) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py Wed Nov 21 17:05:03 2007 @@ -40,9 +40,6 @@ class NameToken(Token): """A token that is not a keyword""" - def __init__(self, parser, keywords=None): - Token.__init__(self, parser.tokens['NAME']) - self.keywords = keywords def match(self, source, builder, level=0): """Matches a token. @@ -60,7 +57,7 @@ if tk.codename == self.codename: # XXX (adim): this is trunk's keyword management # if tk.value not in builder.keywords: - if tk.value not in self.keywords: + if not tk.isKeyword: ret = builder.token( tk.codename, tk.value, source ) return ret source.restore( ctx ) @@ -78,7 +75,7 @@ return False # XXX (adim): this is trunk's keyword management # if other.value in builder.keywords: - if other.value in self.keywords: + if other.isKeyword: return False return True @@ -107,7 +104,7 @@ self.keywords = [] NAME = dest_parser.add_token(Token('NAME')) # NAME = dest_parser.tokens['NAME'] - self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords) + self.tokens[NAME] = NameToken(NAME) # XXX Temporary. We should be able to get rid of it later self.parser = dest_parser Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py Wed Nov 21 17:05:03 2007 @@ -27,8 +27,8 @@ from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \ CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT -def getFutures(source): - futures = FutureAutomaton(source) +def getFutures(futureFlags, source): + futures = FutureAutomaton(futureFlags, source) try: futures.start() except (IndexError, DoneException), e: @@ -62,7 +62,8 @@ precede a future statement. """ - def __init__(self, string): + def __init__(self, futureFlags, string): + self.futureFlags = futureFlags self.s = string self.pos = 0 self.docstringConsumed = False @@ -237,7 +238,7 @@ def setFlag(self, feature): try: - self.flags |= futureFlags.compiler_features[feature] + self.flags |= self.futureFlags.compiler_features[feature] except IndexError: pass @@ -271,12 +272,5 @@ flag_names.append(name) return flag_names -# XXX This is a hack to deal with the fact that we currently are -# using the Python 2.4.1 libraries even when running Python 2.5 -# and that we have a hacked __future__ module. -from pypy.config.pypyoption import get_pypy_config -config = get_pypy_config(translating=False) -if config.objspace.pyversion == '2.4' and False: - futureFlags = FutureFlags((2, 4, 4, 'final', 0)) -else: - futureFlags = FutureFlags((2, 5, 0, 'final', 0)) +futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0)) +futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0)) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py Wed Nov 21 17:05:03 2007 @@ -655,6 +655,7 @@ class Token(GrammarElement): """Represents a Token in a grammar rule (a lexer token)""" + isKeyword = True def __init__(self, codename, value=None): GrammarElement.__init__(self, codename) self.value = value @@ -679,7 +680,7 @@ ctx = source.context() tk = source.next() - if tk.codename == self.codename: + if tk.codename == self.codename and tk.isKeyword: if self.value is None: ret = builder.token( tk.codename, tk.value, source ) return ret @@ -713,7 +714,7 @@ # if (self.value is not None and builder.keywords is not None # and self.value not in builder.keywords): # return False - res = other.codename == self.codename and self.value in [None, other.value] + res = other.isKeyword and other.codename == self.codename and self.value in [None, other.value] #print "matching", self, other, res return res Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py Wed Nov 21 17:05:03 2007 @@ -62,7 +62,7 @@ SyntaxError.__init__(self, msg, lineno, offset, line) self.token_stack = token_stack -def generate_tokens(parser, lines, flags): +def generate_tokens(parser, lines, flags, keywords): """ This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since the original function is not RPYTHON (uses yield) @@ -251,6 +251,8 @@ last_comment = '' elif initial in namechars: # ordinary name tok = Token(parser.tokens['NAME'], token) + if token not in keywords: + tok.isKeyword = False token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial == '\\': # continued stmt @@ -312,12 +314,9 @@ class PythonSource(TokenSource): """This source uses Jonathan's tokenizer""" - def __init__(self, parser, strings, flags=0): - # TokenSource.__init__(self) - #self.parser = parser - + def __init__(self, parser, strings, keywords, flags=0): self.input = strings - tokens = generate_tokens(parser, strings, flags) + tokens = generate_tokens(parser, strings, flags, keywords) self.token_stack = tokens self._current_line = '' # the current line (as a string) self._lineno = -1 @@ -393,14 +392,3 @@ Source = PythonSource -def tokenize_file(filename): - f = file(filename).read() - src = Source(f) - token = src.next() - while token != ("ENDMARKER", None) and token != (None, None): - print token - token = src.next() - -if __name__ == '__main__': - import sys - tokenize_file(sys.argv[1]) Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py Wed Nov 21 17:05:03 2007 @@ -10,7 +10,7 @@ from pypy.interpreter import gateway from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration -#from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT +from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT # XXX seems dead #import pypy.interpreter.pyparser.pysymbol as pysymbol import pypy.interpreter.pyparser.pytoken as pytoken @@ -124,12 +124,20 @@ def parse_lines(self, lines, goal, builder, flags=0): - # builder.keywords = self.keywords.copy() - # if flags & CO_FUTURE_WITH_STATEMENT: - # builder.enable_with() goalnumber = self.symbols[goal] target = self.root_rules[goalnumber] - src = Source(self, lines, flags) + keywords = {} # dict.fromkeys(self.keywords) + for keyword in self.keywords: + keywords[keyword] = None + + if flags & CO_FUTURE_WITH_STATEMENT: + keywords.update({'with': None, + 'as': None }) + else: + keywords.pop('with', None) + keywords.pop('as', None) + src = Source(self, lines, keywords, flags) + if not target.match(src, builder): line, lineno = src.debug() # XXX needs better error messages Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py Wed Nov 21 17:05:03 2007 @@ -3,7 +3,7 @@ from pypy.tool import stdlib___future__ as fut def run(s): - f = future.FutureAutomaton(s) + f = future.FutureAutomaton(future.futureFlags_2_5, s) try: f.start() except IndexError, future.DoneException: @@ -124,12 +124,12 @@ def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - flags = future.getFutures(s) + flags = future.getFutures(future.futureFlags_2_5, s) assert flags == (fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED | fut.CO_FUTURE_WITH_STATEMENT) def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' - flags = future.getFutures(s) + flags = future.getFutures(future.futureFlags_2_5, s) assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0 Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py ============================================================================== --- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py (original) +++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py Wed Nov 21 17:05:03 2007 @@ -18,7 +18,7 @@ def parse_source(source): """returns list of parsed tokens""" - lexer = Source( P, source.splitlines(True)) + lexer = Source( P, source.splitlines(True), {}) tokens = [] last_token = Token(NULLTOKEN, None) while last_token.codename != ENDMARKER: From cfbolz at codespeak.net Wed Nov 21 18:30:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 18:30:52 +0100 (CET) Subject: [pypy-svn] r48909 - pypy/branch/ropes-unicode/pypy/rlib Message-ID: <20071121173052.DD0F881CB@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 18:30:51 2007 New Revision: 48909 Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py Log: try to make the annotator happier Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rlib/runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rlib/runicode.py Wed Nov 21 18:30:51 2007 @@ -13,6 +13,7 @@ else: raise UnicodeEncodeError( encoding, s[startingpos], startingpos, endingpos, msg) +raise_unicode_exception._annspecialcase_ = "specialize:arg(6)" # ____________________________________________________________ # unicode decoding @@ -235,7 +236,7 @@ if not final: break r, pos = errorhandler(errors, 'utf-16', "truncated data", - s, pos, len(s), True) + s, pos, len(s)) result.append(r) if len(s) - pos < 2: break From cfbolz at codespeak.net Wed Nov 21 18:33:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 18:33:44 +0100 (CET) Subject: [pypy-svn] r48910 - in pypy/branch/ropes-unicode/pypy: module/_codecs rlib Message-ID: <20071121173344.EB9A481D3@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 18:33:44 2007 New Revision: 48910 Modified: pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py pypy/branch/ropes-unicode/pypy/rlib/runicode.py Log: samuele tells me that doing it this way is saner Modified: pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py Wed Nov 21 18:33:44 2007 @@ -51,7 +51,6 @@ else: replace = space.str_w(w_replace) return replace, newpos - unicode_call_errorhandler._annspecialcase_ = "specialize:arg(6)" return unicode_call_errorhandler Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rlib/runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rlib/runicode.py Wed Nov 21 18:33:44 2007 @@ -13,7 +13,6 @@ else: raise UnicodeEncodeError( encoding, s[startingpos], startingpos, endingpos, msg) -raise_unicode_exception._annspecialcase_ = "specialize:arg(6)" # ____________________________________________________________ # unicode decoding From rxe at codespeak.net Wed Nov 21 18:53:02 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 21 Nov 2007 18:53:02 +0100 (CET) Subject: [pypy-svn] r48911 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071121175302.CCA558156@code0.codespeak.net> Author: rxe Date: Wed Nov 21 18:53:02 2007 New Revision: 48911 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/genexterns.c Log: (fijal, rxe, lots of others) introducing hacks of entry point to add rffi handling to get argv Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 21 18:53:02 2007 @@ -5,7 +5,7 @@ class CodeWriter(object): tail = '' #/tail - cconv = 'fastcc' #ccc/fastcc + cconv = 'ccc' #ccc/fastcc linkage = 'internal ' #/internal (disabled for now because of the JIT) def __init__(self, file, db, tail=None, cconv=None, linkage=None): Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 21 18:53:02 2007 @@ -15,6 +15,36 @@ from pypy.translator.llvm.externs2ll import setup_externs, generate_llfile from pypy.translator.llvm.gc import GcPolicy from pypy.translator.llvm.log import log +from pypy.rlib.nonconst import NonConstant +from pypy.annotation.listdef import s_list_of_strings +from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.annotation import model as annmodel +from pypy.rpython.lltypesystem import rffi + +def augment_entrypoint(translator, entrypoint): + bk = translator.annotator.bookkeeper + graph_entrypoint = bk.getdesc(entrypoint).getuniquegraph() + s_result = translator.annotator.binding(graph_entrypoint.getreturnvar()) + get_argc = rffi.llexternal('_pypy_getargc', [], rffi.INT) + get_argv = rffi.llexternal('_pypy_getargv', [], rffi.CCHARPP) + + def return_list_of_strings(): + argc = get_argc() + argv = get_argv() + return [rffi.charp2str(argv[i]) for i in range(argc)] + + def new_entrypoint(): + return entrypoint(return_list_of_strings()) + + mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) + mixlevelannotator.getgraph(return_list_of_strings, [], s_list_of_strings) + graph = mixlevelannotator.getgraph(new_entrypoint, [], s_result) + mixlevelannotator.finish() + + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(translator) + + return new_entrypoint class GenLLVM(object): debug = False @@ -82,6 +112,9 @@ create ll file for c file create codewriter """ + if self.standalone: + func = augment_entrypoint(self.translator, func) + # XXX please dont ask! from pypy.translator.c.genc import CStandaloneBuilder cbuild = CStandaloneBuilder(self.translator, func, config=self.config) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Wed Nov 21 18:53:02 2007 @@ -30,7 +30,7 @@ #ifdef ENTRY_POINT_DEFINED int _argc; -char **argv; +char **_argv; int _pypy_getargc() { return _argc; @@ -40,14 +40,25 @@ return _argv; } +/* we still need to forward declare our entry point */ +int __ENTRY_POINT__(void); + +#include + int main(int argc, char *argv[]) { - char *errmsg = RPython_StartupCode(); + int res; + char *errmsg; + errmsg = RPython_StartupCode(); if (errmsg) { fprintf(stderr, "Fatal error during initialization: %s\n", errmsg); return 1; } - return __ENTRY_POINT__(); + _argc = argc; + _argv = argv; + + res = __ENTRY_POINT__(); + return res; } #else From arigo at codespeak.net Wed Nov 21 18:56:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Nov 2007 18:56:00 +0100 (CET) Subject: [pypy-svn] r48912 - pypy/dist/pypy/translator/llvm Message-ID: <20071121175600.C8BB18180@code0.codespeak.net> Author: arigo Date: Wed Nov 21 18:56:00 2007 New Revision: 48912 Modified: pypy/dist/pypy/translator/llvm/genllvm.py Log: untested clean-up. Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 21 18:56:00 2007 @@ -28,16 +28,14 @@ get_argc = rffi.llexternal('_pypy_getargc', [], rffi.INT) get_argv = rffi.llexternal('_pypy_getargv', [], rffi.CCHARPP) - def return_list_of_strings(): + def new_entrypoint(): argc = get_argc() argv = get_argv() - return [rffi.charp2str(argv[i]) for i in range(argc)] - - def new_entrypoint(): - return entrypoint(return_list_of_strings()) + args = [rffi.charp2str(argv[i]) for i in range(argc)] + return entrypoint(args) + entrypoint._annenforceargs_ = [s_list_of_strings] mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) - mixlevelannotator.getgraph(return_list_of_strings, [], s_list_of_strings) graph = mixlevelannotator.getgraph(new_entrypoint, [], s_result) mixlevelannotator.finish() From cfbolz at codespeak.net Wed Nov 21 19:46:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 21 Nov 2007 19:46:22 +0100 (CET) Subject: [pypy-svn] r48913 - in pypy/branch/ropes-unicode/pypy: module/_codecs rlib rlib/test Message-ID: <20071121184622.B8ED9812F@code0.codespeak.net> Author: cfbolz Date: Wed Nov 21 19:46:21 2007 New Revision: 48913 Modified: pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py pypy/branch/ropes-unicode/pypy/rlib/runicode.py pypy/branch/ropes-unicode/pypy/rlib/test/test_runicode.py Log: hack even differently, by separating the decode from the encode error handler Modified: pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py Wed Nov 21 19:46:21 2007 @@ -8,11 +8,12 @@ self.codec_search_cache = {} self.codec_error_registry = {} self.codec_need_encodings = True - self.error_handler = self.make_errorhandler(space) + self.decode_error_handler = self.make_errorhandler(space, True) + self.encode_error_handler = self.make_errorhandler(space, False) - def make_errorhandler(self, space): + def make_errorhandler(self, space, decode): def unicode_call_errorhandler(errors, encoding, reason, input, - startpos, endpos, decode=True): + startpos, endpos): w_errorhandler = lookup_error(space, errors) if decode: @@ -205,7 +206,7 @@ def wrap_encoder(space, uni, errors="strict"): state = space.fromcache(CodecState) func = getattr(runicode, rname) - result = func(uni, len(uni), errors, state.error_handler) + result = func(uni, len(uni), errors, state.encode_error_handler) return space.newtuple([space.wrap(result), space.wrap(len(result))]) wrap_encoder.unwrap_spec = [ObjSpace, unicode, str] globals()[name] = wrap_encoder @@ -218,7 +219,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result, consumed = func(string, len(string), errors, - final, state.error_handler) + final, state.decode_error_handler) return space.newtuple([space.wrap(result), space.wrap(consumed)]) wrap_decoder.unwrap_spec = [ObjSpace, str, str, W_Root] globals()[name] = wrap_decoder @@ -259,7 +260,7 @@ if final: consumed = 0 res, consumed, byteorder = runicode.str_decode_utf_16_helper( - data, len(data), errors, final, state.error_handler, byteorder) + data, len(data), errors, final, state.decode_error_handler, byteorder) return space.newtuple([space.wrap(res), space.wrap(consumed), space.wrap(byteorder)]) utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rlib/runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rlib/runicode.py Wed Nov 21 19:46:21 2007 @@ -5,14 +5,17 @@ BYTEORDER = sys.byteorder -def raise_unicode_exception(errors, encoding, msg, s, startingpos, endingpos, - decode=True): - if decode: - raise UnicodeDecodeError( - encoding, s[startingpos], startingpos, endingpos, msg) - else: - raise UnicodeEncodeError( - encoding, s[startingpos], startingpos, endingpos, msg) +def raise_unicode_exception_decode(errors, encoding, msg, s, + startingpos, endingpos): + assert isinstance(s, str) + raise UnicodeDecodeError( + encoding, s[startingpos], startingpos, endingpos, msg) + +def raise_unicode_exception_encode(errors, encoding, msg, u, + startingpos, endingpos): + assert isinstance(u, unicode) + raise UnicodeEncodeError( + encoding, u[startingpos], startingpos, endingpos, msg) # ____________________________________________________________ # unicode decoding @@ -37,7 +40,7 @@ ] def str_decode_utf_8(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): if (size == 0): return u'', 0 result = [] @@ -158,25 +161,25 @@ def str_decode_utf_16(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "native") return result, length def str_decode_utf_16_be(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "big") return result, length def str_decode_utf_16_le(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "little") return result, length def str_decode_utf_16_helper(s, size, errors, final=True, - errorhandler=raise_unicode_exception, + errorhandler=raise_unicode_exception_decode, byteorder="native"): bo = 0 @@ -277,7 +280,7 @@ return u"".join(result), pos, bo def str_decode_latin_1(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 result = [] @@ -288,7 +291,7 @@ def str_decode_ascii(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_decode): # ASCII is equivalent to the first 128 ordinals in Unicode. result = [] pos = 0 @@ -308,7 +311,7 @@ # unicode encoding -def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception_encode): assert(size >= 0) result = [] i = 0 @@ -353,7 +356,7 @@ def unicode_encode_ucs1_helper(p, size, errors, - errorhandler=raise_unicode_exception, limit=256): + errorhandler=raise_unicode_exception_encode, limit=256): if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" @@ -378,16 +381,16 @@ while collend < len(p) and ord(p[collend]) >= limit: collend += 1 r, pos = errorhandler(errors, encoding, reason, p, - collstart, collend, False) + collstart, collend) result.append(r) return "".join(result) -def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception_encode): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res -def unicode_encode_ascii(p, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_ascii(p, size, errors, errorhandler=raise_unicode_exception_encode): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res @@ -403,7 +406,7 @@ result.append(lo) def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=raise_unicode_exception, + errorhandler=raise_unicode_exception_encode, byteorder='little'): result = [] if (byteorder == 'native'): @@ -429,15 +432,15 @@ return "".join(result) def unicode_encode_utf_16(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_encode): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") def unicode_encode_utf_16_be(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_encode): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") def unicode_encode_utf_16_le(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=raise_unicode_exception_encode): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") Modified: pypy/branch/ropes-unicode/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rlib/test/test_runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rlib/test/test_runicode.py Wed Nov 21 19:46:21 2007 @@ -38,14 +38,13 @@ def checkencodeerror(self, s, encoding, start, stop): called = [False] def errorhandler(errors, enc, msg, t, startingpos, - endingpos, decode): + endingpos): called[0] = True assert errors == "foo!" assert enc == encoding assert t is s assert start == startingpos assert stop == endingpos - assert not decode return "42424242", stop encoder = self.getencoder(encoding) result = encoder(s, len(s), "foo!", errorhandler) @@ -55,7 +54,7 @@ def checkdecodeerror(self, s, encoding, start, stop, addstuff=True): called = [0] def errorhandler(errors, enc, msg, t, startingpos, - endingpos, decode=True): + endingpos): called[0] += 1 if called[0] == 1: assert errors == "foo!" @@ -63,7 +62,6 @@ assert t is s assert start == startingpos assert stop == endingpos - assert decode return u"42424242", stop return "", endingpos decoder = self.getdecoder(encoding) From arigo at codespeak.net Thu Nov 22 10:25:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Nov 2007 10:25:42 +0100 (CET) Subject: [pypy-svn] r48915 - in pypy/dist/pypy: rlib rpython/lltypesystem translator/sandbox/test Message-ID: <20071122092542.193D180B3@code0.codespeak.net> Author: arigo Date: Thu Nov 22 10:25:41 2007 New Revision: 48915 Modified: pypy/dist/pypy/rlib/rposix.py pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Log: Fix the sandboxing handling of errno. Add a test. Modified: pypy/dist/pypy/rlib/rposix.py ============================================================================== --- pypy/dist/pypy/rlib/rposix.py (original) +++ pypy/dist/pypy/rlib/rposix.py Thu Nov 22 10:25:41 2007 @@ -16,5 +16,5 @@ ll2ctypes.TLS.errno = value get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', CConstantErrno, - includes=['errno.h']) + includes=['errno.h'], sandboxsafe=True) Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Thu Nov 22 10:25:41 2007 @@ -242,7 +242,8 @@ def COpaquePtr(*args, **kwds): return lltype.Ptr(COpaque(*args, **kwds)) -def CExternVariable(TYPE, name, _CConstantClass=CConstant, includes=[], include_dirs=[]): +def CExternVariable(TYPE, name, _CConstantClass=CConstant, includes=[], + include_dirs=[], sandboxsafe=False): """Return a pair of functions - a getter and a setter - to access the given global C variable. """ @@ -269,7 +270,7 @@ sources = ('\n'.join(lines),) kwds = {'includes': includes, 'sources':sources, - 'include_dirs':include_dirs} + 'include_dirs':include_dirs, 'sandboxsafe': sandboxsafe} getter = llexternal(getter_name, [], TYPE, **kwds) setter = llexternal(setter_name, [TYPE], lltype.Void, **kwds) return getter, setter Modified: pypy/dist/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/dist/pypy/translator/sandbox/test/test_sandbox.py Thu Nov 22 10:25:41 2007 @@ -160,6 +160,21 @@ return 0 self.run(entry_point, ["3.011"], "2 4 13 75 2 1 3\n") + def test_safefuncs_exception(self): + import math + def entry_point(argv): + a = float(argv[1]) + x = math.log(a) + print int(x * 100.0) + try: + math.log(-a) + except ValueError: + print 'as expected, got a ValueError' + else: + print 'did not get a ValueError!' + return 0 + self.run(entry_point, ["3.011"], "110\nas expected, got a ValueError\n") + def test_os_path_safe(self): def entry_point(argv): print os.path.join('tmp', argv[1]) From arigo at codespeak.net Thu Nov 22 10:27:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Nov 2007 10:27:31 +0100 (CET) Subject: [pypy-svn] r48916 - pypy/dist/pypy/interpreter/test Message-ID: <20071122092731.1F6D78156@code0.codespeak.net> Author: arigo Date: Thu Nov 22 10:27:30 2007 New Revision: 48916 Modified: pypy/dist/pypy/interpreter/test/test_module.py Log: Fix test. Modified: pypy/dist/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_module.py (original) +++ pypy/dist/pypy/interpreter/test/test_module.py Thu Nov 22 10:27:30 2007 @@ -51,5 +51,4 @@ if not hasattr(sys, "pypy_objspaceclass"): skip("need PyPy for sys.__file__ checking") assert sys.__file__ - assert os.path.basename(sys.__file__).startswith('*.py') - + assert os.path.basename(sys.__file__) == 'sys' From cfbolz at codespeak.net Thu Nov 22 10:29:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 10:29:34 +0100 (CET) Subject: [pypy-svn] r48917 - pypy/branch/ropes-unicode/pypy/rlib Message-ID: <20071122092934.570D88156@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 10:29:32 2007 New Revision: 48917 Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py Log: Yet Another Attempt (TM) at making the rtyper happy with the unicode encoding/decoding. Thanks Samuele. Modified: pypy/branch/ropes-unicode/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/ropes-unicode/pypy/rlib/runicode.py (original) +++ pypy/branch/ropes-unicode/pypy/rlib/runicode.py Thu Nov 22 10:29:32 2007 @@ -40,7 +40,9 @@ ] def str_decode_utf_8(s, size, errors, final=False, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode if (size == 0): return u'', 0 result = [] @@ -161,27 +163,28 @@ def str_decode_utf_16(s, size, errors, final=True, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "native") return result, length def str_decode_utf_16_be(s, size, errors, final=True, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "big") return result, length def str_decode_utf_16_le(s, size, errors, final=True, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "little") return result, length def str_decode_utf_16_helper(s, size, errors, final=True, - errorhandler=raise_unicode_exception_decode, + errorhandler=None, byteorder="native"): - + if errorhandler is None: + errorhandler = raise_unicode_exception_decode bo = 0 consumed = 0 @@ -280,7 +283,7 @@ return u"".join(result), pos, bo def str_decode_latin_1(s, size, errors, final=False, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 result = [] @@ -291,7 +294,9 @@ def str_decode_ascii(s, size, errors, final=False, - errorhandler=raise_unicode_exception_decode): + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode # ASCII is equivalent to the first 128 ordinals in Unicode. result = [] pos = 0 @@ -311,7 +316,7 @@ # unicode encoding -def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception_encode): +def unicode_encode_utf_8(s, size, errors, errorhandler=None): assert(size >= 0) result = [] i = 0 @@ -356,7 +361,9 @@ def unicode_encode_ucs1_helper(p, size, errors, - errorhandler=raise_unicode_exception_encode, limit=256): + errorhandler=None, limit=256): + if errorhandler is None: + errorhandler = raise_unicode_exception_encode if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" @@ -386,11 +393,11 @@ return "".join(result) -def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception_encode): +def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res -def unicode_encode_ascii(p, size, errors, errorhandler=raise_unicode_exception_encode): +def unicode_encode_ascii(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res @@ -406,7 +413,7 @@ result.append(lo) def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=raise_unicode_exception_encode, + errorhandler=None, byteorder='little'): result = [] if (byteorder == 'native'): @@ -432,15 +439,15 @@ return "".join(result) def unicode_encode_utf_16(s, size, errors, - errorhandler=raise_unicode_exception_encode): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") def unicode_encode_utf_16_be(s, size, errors, - errorhandler=raise_unicode_exception_encode): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") def unicode_encode_utf_16_le(s, size, errors, - errorhandler=raise_unicode_exception_encode): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") From arigo at codespeak.net Thu Nov 22 10:35:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Nov 2007 10:35:34 +0100 (CET) Subject: [pypy-svn] r48918 - pypy/dist/pypy/interpreter Message-ID: <20071122093534.6A50B816E@code0.codespeak.net> Author: arigo Date: Thu Nov 22 10:35:33 2007 New Revision: 48918 Modified: pypy/dist/pypy/interpreter/gateway.py Log: Fix for: "rm -r _cache; py.py" Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Thu Nov 22 10:35:33 2007 @@ -924,9 +924,10 @@ cls.seed = md5.new(str(GI_VERSION)).digest() if GI_VERSION != GI_VERSION_RENDERED or GI_VERSION is None: for pth in p.listdir(): - try: - pth.remove() - except: pass + if pth.check(file=1): + try: + pth.remove() + except: pass f = file(get_tmp_file_name(str(ini)), "w") f.write("""\ # This folder acts as a cache for code snippets which have been From pedronis at codespeak.net Thu Nov 22 12:38:30 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 22 Nov 2007 12:38:30 +0100 (CET) Subject: [pypy-svn] r48919 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071122113830.3859081EF@code0.codespeak.net> Author: pedronis Date: Thu Nov 22 12:38:29 2007 New Revision: 48919 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: today planning Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Thu Nov 22 12:38:29 2007 @@ -14,20 +14,13 @@ Thursday afternoon: - discuss callback mechanism for rffi/lltype/ll2ctypes -Friday: - - breakday - Saturday: - discuss approaches for extension modules Translation toolchain +++++++++++++++++++++ - - fix the nightly CPython test runs DONE - - - kill remaining suggested_primitives DONE - - - remove the symbol table from the C backend DONE + - refactor rffi header, sources, includes management - fix windows compilation (christian) @@ -36,9 +29,6 @@ - finish rctypes removal - - remove support for suggested_primitives and the extfunctable (alexander, - carl friedrich) - - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) @@ -46,24 +36,22 @@ for rtti, simplify translator/c/gc.py - kill opaque hacks in the C backend - - clean up the tangle of including headers in the C backend + - cleanup pyobj.py and rclass stuff and remove the ability to make cpython extensions, just dll + update gencapicall impl (cfbolz, samuele) - - remove the use of pyobjs in the test wrapping functions IN-PROGRESS - (richard, samuele) + - clean up the tangle of including headers in the C backend - kill half concrete wrapper - - implement global variables in rffi (anto, maciek) + - implement global variables in rffi (anto, maciek) DONE - - - refactor OO external objects. discussion happened, the outcome was that it - doesn't make sense, because javascript and cli are too different - - review pdbplus, especially the graph commands, also in the light of https://codespeak.net/issue/pypy-dev/issue303 and the fact that we can have more than one translator/annotator around (with the timeshifter) + - fix test_thread etc + Interpreter +++++++++++ From tverwaes at codespeak.net Thu Nov 22 13:44:35 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 13:44:35 +0100 (CET) Subject: [pypy-svn] r48920 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071122124435.2002A8155@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 13:44:33 2007 New Revision: 48920 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: moving from varsize to primsize, since for compiledmethod != pointersobject Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Nov 22 13:44:33 2007 @@ -141,10 +141,10 @@ self._vars = [w_nil] * size def at0(self, index0): - return self.fetchvarpointer(index0) + return self.fetch(index0) def atput0(self, index0, w_value): - self.storevarpointer(index0, w_value) + self.store(index0, w_value) def fetch(self, n0): return self._vars[n0] @@ -166,6 +166,9 @@ def instsize(self): return self.getclass().as_class_get_shadow().instsize() + def primsize(self): + return self.varsize() + def size(self): return len(self._vars) @@ -217,6 +220,9 @@ def size(self): return len(self.bytes) + def primsize(self): + return self.size() + def __str__(self): return self.as_string() @@ -348,19 +354,21 @@ self.primitive is not None) def size(self): - return self.varsize() + return self.literalsize() + len(self.bytes) - def staticsize(self): + def literalsize(self): return len(self.literals) * constants.BYTES_PER_WORD - def varsize(self): - # XXX - return self.staticsize() + len(self.bytes) + def primsize(self): + return self.size() + self.headersize() + + def headersize(self): + return constants.BYTES_PER_WORD def at0(self, index0): # XXX from pypy.lang.smalltalk import utility - index0 = index0 - self.staticsize() + index0 = index0 - self.literalsize() if index0 < 0: # XXX Do something useful with this.... we are not a block # of memory as smalltalk expects but wrapped in py-os @@ -369,7 +377,7 @@ def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility - index0 = index0 - self.staticsize() + index0 = index0 - self.literalsize() if index0 < 0: # XXX Do something useful with this.... we are not a block # of memory as smalltalk expects but wrapped in py-os Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Nov 22 13:44:33 2007 @@ -301,7 +301,7 @@ def func(interp, w_obj): if not w_obj.shadow_of_my_class().isvariable(): raise PrimitiveFailedError() - return utility.wrap_int(w_obj.varsize()) + return utility.wrap_int(w_obj.primsize()) @expose_primitive(STRING_AT, unwrap_spec=[object, index1_0]) def func(interp, w_obj, n0): Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Nov 22 13:44:33 2007 @@ -227,14 +227,14 @@ assert w_false is objtable.w_false def test_compile_method(): - py.test.skip("not working") + #py.test.skip("not working") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] ifFalse: [ (self - 1) fib + (self - 2) fib ]""" perform(w(10).getclass(), "compile:classified:notifying:", w(sourcecode), w('pypy'), w(None)) assert perform(w(10), "fib") == w(89) - + def w(any): if any is None: return objtable.w_nil Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Nov 22 13:44:33 2007 @@ -206,6 +206,12 @@ w_obj = mockclass(3, varsized=True).as_class_get_shadow().new(5) assert prim(primitives.SIZE, [w_obj]).value == 5 +def test_size_of_compiled_method(): + varsize = 3 + text = "abc" + w_cm = model.W_CompiledMethod(varsize, text, 1, 1) + assert prim(primitives.SIZE, [w_cm]).value == (varsize+1)*constants.BYTES_PER_WORD + len(text) + def test_string_at(): assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") From akuhn at codespeak.net Thu Nov 22 14:01:01 2007 From: akuhn at codespeak.net (akuhn at codespeak.net) Date: Thu, 22 Nov 2007 14:01:01 +0100 (CET) Subject: [pypy-svn] r48921 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071122130101.CDFD281EF@code0.codespeak.net> Author: akuhn Date: Thu Nov 22 14:01:01 2007 New Revision: 48921 Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Log: marked broken test as skip Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Nov 22 14:01:01 2007 @@ -726,6 +726,7 @@ def test_bc_primBytecodeAt_with_instvars(): # ^ self at: 1 + py.test.skip("Broken, we are fixing it.") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -741,6 +742,7 @@ def test_bc_primBytecodeAtPut_with_instvars(): # ^ self at: 1 put: #b + py.test.skip("Broken, we are fixing it.") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable Modified: pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_miniimage.py Thu Nov 22 14:01:01 2007 @@ -227,7 +227,7 @@ assert w_false is objtable.w_false def test_compile_method(): - #py.test.skip("not working") + py.test.skip("Not working yet.") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] From tismer at codespeak.net Thu Nov 22 14:18:07 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 22 Nov 2007 14:18:07 +0100 (CET) Subject: [pypy-svn] r48922 - pypy/branch/windows48396 Message-ID: <20071122131807.5829A81F7@code0.codespeak.net> Author: tismer Date: Thu Nov 22 14:18:05 2007 New Revision: 48922 Added: pypy/branch/windows48396/ - copied from r48396, pypy/dist/ Log: last known working version for windows From cfbolz at codespeak.net Thu Nov 22 15:31:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 15:31:41 +0100 (CET) Subject: [pypy-svn] r48923 - pypy/branch/remove-extcompiler-rctypes Message-ID: <20071122143141.6450081B9@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 15:31:39 2007 New Revision: 48923 Added: pypy/branch/remove-extcompiler-rctypes/ - copied from r48922, pypy/dist/ Log: (pedronis, cfbolz): a branch for removing the extcompiler, rctypes, raymond, and the ability to have the C backend generate dlls that happen to be CPython extension modules too. From cfbolz at codespeak.net Thu Nov 22 15:50:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 15:50:13 +0100 (CET) Subject: [pypy-svn] r48924 - in pypy/dist/pypy: annotation config module/_codecs objspace/std objspace/std/test rlib rlib/test rpython rpython/lltypesystem rpython/ootypesystem rpython/test Message-ID: <20071122145013.E30E581F4@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 15:50:13 2007 New Revision: 48924 Added: pypy/dist/pypy/objspace/std/ropeunicodeobject.py - copied unchanged from r48923, pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py Modified: pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/module/_codecs/interp_codecs.py pypy/dist/pypy/objspace/std/floattype.py pypy/dist/pypy/objspace/std/inttype.py pypy/dist/pypy/objspace/std/longtype.py pypy/dist/pypy/objspace/std/model.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/rope.py pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/test/test_rope.py pypy/dist/pypy/objspace/std/test/test_unicodeobject.py pypy/dist/pypy/objspace/std/unicodetype.py pypy/dist/pypy/rlib/runicode.py pypy/dist/pypy/rlib/test/test_runicode.py pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/test/test_runicode.py Log: merge the ropes branch: svn merge -r 48691:HEAD svn+ssh://codespeak.net/svn/pypy/branch/ropes-unicode log: ------------------------------------------------------------------------ r48691 | cfbolz | 2007-11-14 20:39:36 +0100 (Wed, 14 Nov 2007) | 2 lines Changed paths: A /pypy/branch/ropes-unicode (from /pypy/dist:48690) a new branch to try some crazy ideas about ropes and unicode ------------------------------------------------------------------------ r48706 | cfbolz | 2007-11-15 13:22:15 +0100 (Thu, 15 Nov 2007) | 3 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py major refactoring of ropes to have a chance to work with unicode. one test failing. ------------------------------------------------------------------------ r48713 | cfbolz | 2007-11-15 18:05:33 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py off by one error ? all tests pass ------------------------------------------------------------------------ r48714 | cfbolz | 2007-11-15 18:22:25 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py make the ropes of the std obj space work again ------------------------------------------------------------------------ r48715 | cfbolz | 2007-11-15 18:39:02 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py move start/endswith helpers to rope.py ------------------------------------------------------------------------ r48716 | cfbolz | 2007-11-15 18:44:18 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py some tests for unicode support in ropes ------------------------------------------------------------------------ r48717 | cfbolz | 2007-11-15 19:01:12 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py tests and more fixes for unicode ops with ropes ------------------------------------------------------------------------ r48718 | cfbolz | 2007-11-15 19:10:53 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py more passing tests ------------------------------------------------------------------------ r48719 | cfbolz | 2007-11-15 22:19:08 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py some more functionality for ropes ------------------------------------------------------------------------ r48720 | cfbolz | 2007-11-15 23:55:32 +0100 (Thu, 15 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py add a split function to the rope module ------------------------------------------------------------------------ r48722 | cfbolz | 2007-11-16 00:13:20 +0100 (Fri, 16 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py use split for expandtabs too ------------------------------------------------------------------------ r48723 | cfbolz | 2007-11-16 00:28:38 +0100 (Fri, 16 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py a failing unicode test ------------------------------------------------------------------------ r48724 | cfbolz | 2007-11-16 00:38:15 +0100 (Fri, 16 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py hm, applevel skips are different ------------------------------------------------------------------------ r48725 | cfbolz | 2007-11-16 00:38:33 +0100 (Fri, 16 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py share some code with normal strings ------------------------------------------------------------------------ r48768 | cfbolz | 2007-11-18 00:04:31 +0100 (Sun, 18 Nov 2007) | 4 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/config/pypyoption.py M /pypy/branch/ropes-unicode/pypy/objspace/std/objspace.py M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py A /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py checking in some very much in-progress work before I leave for the sprint: the start of an implementation of unicode strings as ropes. Far from completion, but I don't want to risk my laptop being stolen. ------------------------------------------------------------------------ r48781 | cfbolz | 2007-11-19 12:16:46 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/floattype.py M /pypy/branch/ropes-unicode/pypy/objspace/std/inttype.py M /pypy/branch/ropes-unicode/pypy/objspace/std/longtype.py M /pypy/branch/ropes-unicode/pypy/objspace/std/model.py M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/stringobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/unicodetype.py try to fix ropes that represent unicode chars ------------------------------------------------------------------------ r48788 | cfbolz | 2007-11-19 14:51:53 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/annotation/unaryop.py M /pypy/branch/ropes-unicode/pypy/rpython/lltypesystem/rstr.py M /pypy/branch/ropes-unicode/pypy/rpython/ootypesystem/rstr.py M /pypy/branch/ropes-unicode/pypy/rpython/rstr.py M /pypy/branch/ropes-unicode/pypy/rpython/test/test_runicode.py add encoding and decoding with latin-1 ------------------------------------------------------------------------ r48789 | cfbolz | 2007-11-19 15:05:44 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py kill some dead code and some useless XXXs ------------------------------------------------------------------------ r48790 | cfbolz | 2007-11-19 15:28:31 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py bug in partition ------------------------------------------------------------------------ r48791 | cfbolz | 2007-11-19 15:30:05 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py fix test and another bug ------------------------------------------------------------------------ r48792 | cfbolz | 2007-11-19 15:38:06 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py fix a bug in rindex ------------------------------------------------------------------------ r48793 | cfbolz | 2007-11-19 15:55:16 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py more bugs ------------------------------------------------------------------------ r48796 | cfbolz | 2007-11-19 16:01:00 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py bugs in count ------------------------------------------------------------------------ r48801 | cfbolz | 2007-11-19 17:09:13 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py rpython fix: don't mix string and unicode ------------------------------------------------------------------------ r48802 | cfbolz | 2007-11-19 17:09:23 +0100 (Mon, 19 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_unicodeobject.py test and fix in swapcase ------------------------------------------------------------------------ r48884 | cfbolz | 2007-11-21 10:20:34 +0100 (Wed, 21 Nov 2007) | 3 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/objspace/std/rope.py M /pypy/branch/ropes-unicode/pypy/objspace/std/ropeunicodeobject.py M /pypy/branch/ropes-unicode/pypy/objspace/std/test/test_rope.py fix rope translation failures. add tests for the code that tries to quickly encode/decode ropes. ------------------------------------------------------------------------ r48886 | cfbolz | 2007-11-21 10:33:04 +0100 (Wed, 21 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/config/pypyoption.py fix typo ------------------------------------------------------------------------ r48909 | cfbolz | 2007-11-21 18:30:51 +0100 (Wed, 21 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/rlib/runicode.py try to make the annotator happier ------------------------------------------------------------------------ r48910 | cfbolz | 2007-11-21 18:33:44 +0100 (Wed, 21 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py M /pypy/branch/ropes-unicode/pypy/rlib/runicode.py samuele tells me that doing it this way is saner ------------------------------------------------------------------------ r48913 | cfbolz | 2007-11-21 19:46:21 +0100 (Wed, 21 Nov 2007) | 2 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/module/_codecs/interp_codecs.py M /pypy/branch/ropes-unicode/pypy/rlib/runicode.py M /pypy/branch/ropes-unicode/pypy/rlib/test/test_runicode.py hack even differently, by separating the decode from the encode error handler ------------------------------------------------------------------------ r48917 | cfbolz | 2007-11-22 10:29:32 +0100 (Thu, 22 Nov 2007) | 3 lines Changed paths: M /pypy/branch/ropes-unicode/pypy/rlib/runicode.py Yet Another Attempt (TM) at making the rtyper happy with the unicode encoding/decoding. Thanks Samuele. ------------------------------------------------------------------------ Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Thu Nov 22 15:50:13 2007 @@ -466,7 +466,7 @@ if not s_enc.is_constant(): raise TypeError("Non-constant encoding not supported") enc = s_enc.const - if enc != 'ascii': + if enc not in ('ascii', 'latin-1'): raise TypeError("Encoding %s not supported for unicode" % (enc,)) return SomeString() method_encode.can_only_throw = [UnicodeEncodeError] @@ -482,7 +482,7 @@ if not s_enc.is_constant(): raise TypeError("Non-constant encoding not supported") enc = s_enc.const - if enc != 'ascii': + if enc not in ('ascii', 'latin-1'): raise TypeError("Encoding %s not supported for strings" % (enc,)) return SomeUnicodeString() method_decode.can_only_throw = [UnicodeDecodeError] Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Nov 22 15:50:13 2007 @@ -145,6 +145,10 @@ suggests=[("objspace.std.withprebuiltchar", True), ("objspace.std.sharesmallstr", True)]), + BoolOption("withropeunicode", "use ropes for the unicode implementation", + default=False, + requires=[("objspace.std.withrope", True)]), + BoolOption("withmultidict", "use dictionaries optimized for flexibility", default=False), Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/interp_codecs.py Thu Nov 22 15:50:13 2007 @@ -8,11 +8,12 @@ self.codec_search_cache = {} self.codec_error_registry = {} self.codec_need_encodings = True - self.error_handler = self.make_errorhandler(space) + self.decode_error_handler = self.make_errorhandler(space, True) + self.encode_error_handler = self.make_errorhandler(space, False) - def make_errorhandler(self, space): + def make_errorhandler(self, space, decode): def unicode_call_errorhandler(errors, encoding, reason, input, - startpos, endpos, decode=True): + startpos, endpos): w_errorhandler = lookup_error(space, errors) if decode: @@ -51,7 +52,6 @@ else: replace = space.str_w(w_replace) return replace, newpos - unicode_call_errorhandler._annspecialcase_ = "specialize:arg(6)" return unicode_call_errorhandler @@ -210,7 +210,7 @@ def wrap_encoder(space, uni, errors="strict"): state = space.fromcache(CodecState) func = getattr(runicode, rname) - result = func(uni, len(uni), errors, state.error_handler) + result = func(uni, len(uni), errors, state.encode_error_handler) return space.newtuple([space.wrap(result), space.wrap(len(result))]) wrap_encoder.unwrap_spec = [ObjSpace, unicode, str] globals()[name] = wrap_encoder @@ -223,7 +223,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result, consumed = func(string, len(string), errors, - final, state.error_handler) + final, state.decode_error_handler) return space.newtuple([space.wrap(result), space.wrap(consumed)]) wrap_decoder.unwrap_spec = [ObjSpace, str, str, W_Root] globals()[name] = wrap_decoder @@ -264,7 +264,7 @@ if final: consumed = 0 res, consumed, byteorder = runicode.str_decode_utf_16_helper( - data, len(data), errors, final, state.error_handler, byteorder) + data, len(data), errors, final, state.decode_error_handler, byteorder) return space.newtuple([space.wrap(res), space.wrap(consumed), space.wrap(byteorder)]) utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root] Modified: pypy/dist/pypy/objspace/std/floattype.py ============================================================================== --- pypy/dist/pypy/objspace/std/floattype.py (original) +++ pypy/dist/pypy/objspace/std/floattype.py Thu Nov 22 15:50:13 2007 @@ -19,7 +19,10 @@ raise OperationError(space.w_ValueError, space.wrap(e.msg)) elif space.is_true(space.isinstance(w_value, space.w_unicode)): - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: if USE_NEW_S2F: Modified: pypy/dist/pypy/objspace/std/inttype.py ============================================================================== --- pypy/dist/pypy/objspace/std/inttype.py (original) +++ pypy/dist/pypy/objspace/std/inttype.py Thu Nov 22 15:50:13 2007 @@ -66,7 +66,10 @@ except ParseStringOverflowError, e: w_longval = retry_to_w_long(space, e.parser) elif space.is_true(space.isinstance(w_value, space.w_unicode)): - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w string = unicode_to_decimal_w(space, w_value) try: value = string_to_int(string) @@ -95,7 +98,10 @@ base = space.int_w(w_base) if space.is_true(space.isinstance(w_value, space.w_unicode)): - from pypy.objspace.std.unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w s = unicode_to_decimal_w(space, w_value) else: try: Modified: pypy/dist/pypy/objspace/std/longtype.py ============================================================================== --- pypy/dist/pypy/objspace/std/longtype.py (original) +++ pypy/dist/pypy/objspace/std/longtype.py Thu Nov 22 15:50:13 2007 @@ -18,7 +18,10 @@ space.wrap(e.msg)) elif space.is_true(space.isinstance(w_value, space.w_unicode)): try: - from unicodeobject import unicode_to_decimal_w + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w + else: + from pypy.objspace.std.unicodeobject import unicode_to_decimal_w w_value = string_to_w_long(space, unicode_to_decimal_w(space, w_value)) except ParseStringError, e: raise OperationError(space.w_ValueError, Modified: pypy/dist/pypy/objspace/std/model.py ============================================================================== --- pypy/dist/pypy/objspace/std/model.py (original) +++ pypy/dist/pypy/objspace/std/model.py Thu Nov 22 15:50:13 2007 @@ -17,6 +17,8 @@ "withmultilist" : ["listmultiobject.W_ListMultiObject"], "withrope" : ["ropeobject.W_RopeObject", "ropeobject.W_RopeIterObject"], + "withropeunicode": ["ropeunicodeobject.W_RopeUnicodeObject", + "ropeunicodeobject.W_RopeUnicodeIterObject"], "withrangelist" : ["rangeobject.W_RangeListObject", "rangeobject.W_RangeIterObject"], "withtproxy" : ["proxyobject.W_TransparentList", @@ -68,6 +70,7 @@ from pypy.objspace.std import listmultiobject from pypy.objspace.std import stringobject from pypy.objspace.std import ropeobject + from pypy.objspace.std import ropeunicodeobject from pypy.objspace.std import strsliceobject from pypy.objspace.std import strjoinobject from pypy.objspace.std import typeobject @@ -186,9 +189,15 @@ (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), ] else: - self.typeorder[ropeobject.W_RopeObject] += [ - (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), - ] + if config.objspace.std.withropeunicode: + self.typeorder[ropeobject.W_RopeObject] += [ + (ropeunicodeobject.W_RopeUnicodeObject, + ropeunicodeobject.delegate_Rope2RopeUnicode), + ] + else: + self.typeorder[ropeobject.W_RopeObject] += [ + (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), + ] if config.objspace.std.withstrslice: self.typeorder[strsliceobject.W_StringSliceObject] += [ Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Thu Nov 22 15:50:13 2007 @@ -400,7 +400,8 @@ from pypy.objspace.std.stringtype import wrapstr return wrapstr(self, x) if isinstance(x, unicode): - return W_UnicodeObject(x) + from pypy.objspace.std.unicodetype import wrapunicode + return wrapunicode(self, x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, Wrappable): Modified: pypy/dist/pypy/objspace/std/rope.py ============================================================================== --- pypy/dist/pypy/objspace/std/rope.py (original) +++ pypy/dist/pypy/objspace/std/rope.py Thu Nov 22 15:50:13 2007 @@ -9,8 +9,8 @@ # XXX should optimize the numbers NEW_NODE_WHEN_LENGTH = 16 +CONVERT_WHEN_SMALLER = 8 MAX_DEPTH = 32 # maybe should be smaller -MIN_SLICE_LENGTH = 64 CONCATENATE_WHEN_MULTIPLYING = 128 HIGHEST_BIT_SET = intmask(1L << (NBITS - 1)) @@ -54,58 +54,87 @@ class StringNode(object): hash_cache = 0 def length(self): - return 0 + raise NotImplementedError("base class") + + def is_ascii(self): + raise NotImplementedError("base class") + + def is_bytestring(self): + raise NotImplementedError("base class") def depth(self): return 0 - def rebalance(self): - return self - def hash_part(self): raise NotImplementedError("base class") - def flatten(self): - return '' + def check_balanced(self): + return True + + def getchar(self, index): + raise NotImplementedError("abstract base class") - def __add__(self, other): - return concatenate(self, other) - - def __getitem__(self, index): - if isinstance(index, slice): - start, stop, step = index.indices(self.length()) - # XXX sucks - slicelength = len(xrange(start, stop, step)) - return getslice(self, start, stop, step, slicelength) - return self.getitem(index) + def getunichar(self, index): + raise NotImplementedError("abstract base class") - def getitem(self, index): + def getint(self, index): raise NotImplementedError("abstract base class") - def getitem_slice(self, start, stop): - # XXX really horrible, in most cases - result = [] - for i in range(start, stop): - result.append(self.getitem(i)) - return rope_from_charlist(result) + def getrope(self, index): + raise NotImplementedError("abstract base class") + + def getslice(self, start, stop): + raise NotImplementedError("abstract base class") def view(self): view([self]) - def check_balanced(self): - return True + def rebalance(self): + return self + + def flatten_string(self): + raise NotImplementedError("abstract base class") + + def flatten_unicode(self): + raise NotImplementedError("abstract base class") + + def __add__(self, other): + return concatenate(self, other) + + +class LiteralNode(StringNode): + def find_int(self, what, start, stop): + raise NotImplementedError("abstract base class") + + def literal_concat(self, other): + raise NotImplementedError("abstract base class") -class LiteralStringNode(StringNode): +class LiteralStringNode(LiteralNode): def __init__(self, s): + assert isinstance(s, str) self.s = s + is_ascii = True + for c in s: + if ord(c) >= 128: + is_ascii = False + self._is_ascii = is_ascii def length(self): return len(self.s) - def flatten(self): + def is_ascii(self): + return self._is_ascii + + def is_bytestring(self): + return True + + def flatten_string(self): return self.s + def flatten_unicode(self): + return self.s.decode('latin-1') + def hash_part(self): h = self.hash_cache if not h: @@ -117,13 +146,41 @@ h = self.hash_cache = x return h - def getitem(self, index): + def getchar(self, index): return self.s[index] - def getitem_slice(self, start, stop): + def getunichar(self, index): + return unichr(ord(self.s[index])) + + def getint(self, index): + return ord(self.s[index]) + + def getrope(self, index): + return LiteralStringNode.PREBUILT[ord(self.s[index])] + + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralStringNode(self.s[start:stop]) + + def find_int(self, what, start, stop): + if what >= 256: + return -1 + result = self.s.find(chr(what), start, stop) + if result == -1: + return -1 + return result + + def literal_concat(self, other): + if (isinstance(other, LiteralStringNode) and + len(other.s) + len(self.s) < NEW_NODE_WHEN_LENGTH): + return LiteralStringNode(self.s + other.s) + elif (isinstance(other, LiteralUnicodeNode) and + len(other.u) + len(self.s) < NEW_NODE_WHEN_LENGTH and + len(self.s) < CONVERT_WHEN_SMALLER): + return LiteralUnicodeNode(self.s.decode("latin-1") + other.u) + return BinaryConcatNode(self, other) + def dot(self, seen, toplevel=False): if self in seen: return @@ -139,6 +196,80 @@ del i +class LiteralUnicodeNode(LiteralNode): + def __init__(self, u): + assert isinstance(u, unicode) + self.u = u + + def length(self): + return len(self.u) + + def flatten_unicode(self): + return self.u + + def is_ascii(self): + return False # usually not + + def is_bytestring(self): + return False + + def hash_part(self): + h = self.hash_cache + if not h: + x = 0 + for c in self.u: + x = (1000003*x) + ord(c) + x = intmask(x) + x |= HIGHEST_BIT_SET + h = self.hash_cache = x + return h + + def getunichar(self, index): + return self.u[index] + + def getint(self, index): + return ord(self.u[index]) + + def getrope(self, index): + ch = ord(self.u[index]) + if ch < 256: + return LiteralStringNode.PREBUILT[ch] + if len(self.u) == 1: + return self + return LiteralUnicodeNode(unichr(ch)) + + def getslice(self, start, stop): + assert 0 <= start <= stop + return LiteralUnicodeNode(self.u[start:stop]) + + def find_int(self, what, start, stop): + result = self.u.find(unichr(what), start, stop) + if result == -1: + return -1 + return result + + def literal_concat(self, other): + if (isinstance(other, LiteralUnicodeNode) and + len(other.u) + len(self.u) < NEW_NODE_WHEN_LENGTH): + return LiteralUnicodeNode(self.u + other.u) + elif (isinstance(other, LiteralStringNode) and + len(other.s) + len(self.u) < NEW_NODE_WHEN_LENGTH and + len(other.s) < CONVERT_WHEN_SMALLER): + return LiteralUnicodeNode(self.u + other.s.decode("latin-1")) + return BinaryConcatNode(self, other) + + def dot(self, seen, toplevel=False): + if self in seen: + return + seen[self] = True + addinfo = repr(self.u).replace('"', "'") or "_" + if len(addinfo) > 10: + addinfo = addinfo[:3] + "..." + addinfo[-3:] + yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( + id(self), len(self.u), + repr(addinfo).replace('"', '').replace("\\", "\\\\"))) + + class BinaryConcatNode(StringNode): def __init__(self, left, right): self.left = left @@ -149,6 +280,14 @@ raise self._depth = max(left.depth(), right.depth()) + 1 self.balanced = False + self._is_ascii = left.is_ascii() and right.is_ascii() + self._is_bytestring = left.is_bytestring() and right.is_bytestring() + + def is_ascii(self): + return self._is_ascii + + def is_bytestring(self): + return self._is_bytestring def check_balanced(self): if self.balanced: @@ -172,16 +311,41 @@ def depth(self): return self._depth - def getitem(self, index): + def getchar(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getchar(index - llen) + else: + return self.left.getchar(index) + + def getunichar(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getunichar(index - llen) + else: + return self.left.getunichar(index) + + def getint(self, index): + llen = self.left.length() + if index >= llen: + return self.right.getint(index - llen) + else: + return self.left.getint(index) + + def getrope(self, index): llen = self.left.length() if index >= llen: - return self.right.getitem(index - llen) + return self.right.getrope(index - llen) else: - return self.left.getitem(index) + return self.left.getrope(index) - def flatten(self): + def flatten_string(self): f = fringe(self) - return "".join([node.flatten() for node in f]) + return "".join([node.flatten_string() for node in f]) + + def flatten_unicode(self): + f = fringe(self) + return u"".join([node.flatten_unicode() for node in f]) def hash_part(self): h = self.hash_cache @@ -213,120 +377,44 @@ for line in child.dot(seen): yield line -class SliceNode(StringNode): - def __init__(self, start, stop, node): - assert 0 <= start <= stop - self.start = start - self.stop = stop - self.node = node - - def length(self): - return self.stop - self.start - - def getitem_slice(self, start, stop): - return self.node.getitem_slice(self.start + start, self.start + stop) - - def getitem(self, index): - return self.node.getitem(self.start + index) - - def flatten(self): - return self.node.flatten()[self.start: self.stop] - - def hash_part(self): - h = self.hash_cache - if not h: - x = 0 - for i in range(self.start, self.stop): - x = (1000003*x) + ord(self.node.getitem(i)) - x = intmask(x) - x |= HIGHEST_BIT_SET - h = self.hash_cache = x - return h - - def dot(self, seen, toplevel=False): - if self in seen: - return - seen[self] = True - yield '"%s" [shape=octagon,label="slice\\nstart=%s, stop=%s"];' % ( - id(self), self.start, self.stop) - yield '"%s" -> "%s";' % (id(self), id(self.node)) - for line in self.node.dot(seen): - yield line - -class EfficientGetitemWraper(StringNode): - def __init__(self, node): - assert isinstance(node, BinaryConcatNode) - self.node = node - self.iter = SeekableCharIterator(node) - self.nextpos = 0 - self.accesses = 0 - self.seeks = 0 - - def length(self): - return self.node.length() - - def depth(self): - return self.node.depth() - - def rebalance(self): - return EfficientGetitemWraper(self.node.rebalance()) - - def hash_part(self): - return self.node.hash_part() - - def flatten(self): - return self.node.flatten() - - def getitem(self, index): - self.accesses += 1 - nextpos = self.nextpos - self.nextpos = index + 1 - if index < nextpos: - self.iter.seekback(nextpos - index) - self.seeks += nextpos - index - elif index > nextpos: - self.iter.seekforward(index - nextpos) - self.seeks += index - nextpos - return self.iter.next() - - def view(self): - return self.node.view() - - def check_balanced(self): - return self.node.check_balanced() - - def concatenate(node1, node2): if node1.length() == 0: return node2 if node2.length() == 0: return node1 - if (isinstance(node2, LiteralStringNode) and - len(node2.s) <= NEW_NODE_WHEN_LENGTH): - if isinstance(node1, LiteralStringNode): - if len(node1.s) + len(node2.s) <= NEW_NODE_WHEN_LENGTH: - return LiteralStringNode(node1.s + node2.s) + if isinstance(node2, LiteralNode): + if isinstance(node1, LiteralNode): + return node1.literal_concat(node2) elif isinstance(node1, BinaryConcatNode): r = node1.right - if isinstance(r, LiteralStringNode): - if len(r.s) + len(node2.s) <= NEW_NODE_WHEN_LENGTH: - return BinaryConcatNode(node1.left, - LiteralStringNode(r.s + node2.s)) + if isinstance(r, LiteralNode): + return BinaryConcatNode(node1.left, + r.literal_concat(node2)) result = BinaryConcatNode(node1, node2) if result.depth() > MAX_DEPTH: #XXX better check return result.rebalance() return result -def getslice(node, start, stop, step, slicelength): +def getslice(node, start, stop, step, slicelength=-1): + if slicelength == -1: + # XXX for testing only + slicelength = len(xrange(start, stop, step)) if step != 1: start, stop, node = find_straddling(node, start, stop) - iter = SeekableCharIterator(node) + iter = SeekableItemIterator(node) iter.seekforward(start) - result = [iter.next()] - for i in range(slicelength - 1): - iter.seekforward(step - 1) - result.append(iter.next()) - return rope_from_charlist(result) + if node.is_bytestring(): + result = [iter.nextchar()] + for i in range(slicelength - 1): + iter.seekforward(step - 1) + result.append(iter.nextchar()) + return rope_from_charlist(result) + else: + result = [iter.nextunichar()] + for i in range(slicelength - 1): + iter.seekforward(step - 1) + result.append(iter.nextunichar()) + return rope_from_unicharlist(result) return getslice_one(node, start, stop) def getslice_one(node, start, stop): @@ -342,7 +430,7 @@ getslice_right(node.left, start), getslice_left(node.right, stop - node.left.length())) else: - return getslice_primitive(node, start, stop) + return node.getslice(start, stop) def find_straddling(node, start, stop): while 1: @@ -371,7 +459,7 @@ else: return concatenate(getslice_right(node.left, start), node.right) - return getslice_primitive(node, start, node.length()) + return node.getslice(start, node.length()) def getslice_left(node, stop): while 1: @@ -385,16 +473,9 @@ else: return concatenate(node.left, getslice_left(node.right, stop - llen)) - return getslice_primitive(node, 0, stop) + return node.getslice(0, stop) -def getslice_primitive(node, start, stop): - if stop - start >= MIN_SLICE_LENGTH: - if isinstance(node, SliceNode): - return SliceNode(start + node.start, stop + node.start, - node.node) - return SliceNode(start, stop, node) - return node.getitem_slice(start, stop) def multiply(node, times): if times <= 0: @@ -510,10 +591,69 @@ size += len(chars) return rebalance(nodelist, size) +def rope_from_unicharlist(charlist): + nodelist = [] + length = len(charlist) + if not length: + return LiteralStringNode.EMPTY + i = 0 + while i < length: + unichunk = [] + while i < length: + c = ord(charlist[i]) + if c < 256: + break + unichunk.append(unichr(c)) + i += 1 + if unichunk: + nodelist.append(LiteralUnicodeNode(u"".join(unichunk))) + strchunk = [] + while i < length: + c = ord(charlist[i]) + if c >= 256: + break + strchunk.append(chr(c)) + i += 1 + if strchunk: + nodelist.append(LiteralStringNode("".join(strchunk))) + return rebalance(nodelist, length) + +def rope_from_unicode(uni): + nodelist = [] + length = len(uni) + if not length: + return LiteralStringNode.EMPTY + i = 0 + while i < length: + start = i + while i < length: + c = ord(uni[i]) + if c < 256: + break + i += 1 + if i != start: + nodelist.append(LiteralUnicodeNode(uni[start:i])) + start = i + strchunk = [] + while i < length: + c = ord(uni[i]) + if c >= 256: + break + i += 1 + if i != start: + nodelist.append(LiteralStringNode(uni[start:i].encode("latin-1"))) + return rebalance(nodelist, length) + +def rope_from_unichar(unichar): + intval = ord(unichar) + if intval > 256: + return LiteralUnicodeNode(unichar) + return LiteralStringNode.PREBUILT[intval] + # __________________________________________________________________________ # searching -def find_char(node, c, start=0, stop=-1): +def find_int(node, what, start=0, stop=-1): offset = 0 length = node.length() if stop == -1: @@ -524,14 +664,11 @@ start = newstart stop = newstop assert 0 <= start <= stop - if isinstance(node, LiteralStringNode): - result = node.s.find(c, start, stop) - if result == -1: - return -1 - return result + offset - elif isinstance(node, SliceNode): - return find_char(node.node, c, node.start + start, - node.start + stop) - node.start + offset + if isinstance(node, LiteralNode): + pos = node.find_int(what, start, stop) + if pos == -1: + return pos + return pos + offset iter = FringeIterator(node) #import pdb; pdb.set_trace() i = 0 @@ -546,20 +683,10 @@ continue searchstart = max(0, start - i) searchstop = min(stop - i, nodelength) - if isinstance(fringenode, LiteralStringNode): - st = fringenode.s - localoffset = 0 - else: - assert isinstance(fringenode, SliceNode) - n = fringenode.node - assert isinstance(n, LiteralStringNode) - st = n.s - localoffset = -fringenode.start - searchstart += fringenode.start - searchstop += fringenode.stop - pos = st.find(c, searchstart, searchstop) + assert isinstance(fringenode, LiteralNode) + pos = fringenode.find_int(what, searchstart, searchstop) if pos != -1: - return pos + i + offset + localoffset + return pos + i + offset i += nodelength return -1 @@ -570,7 +697,7 @@ if stop > len1 or stop == -1: stop = len1 if len2 == 1: - return find_char(node, subnode.getitem(0), start, stop) + return find_int(node, subnode.getint(0), start, stop) if len2 == 0: if (stop - start) < 0: return -1 @@ -578,61 +705,29 @@ restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) -def _find(node, substring, start, stop, restart): - len2 = len(substring) - i = 0 - m = start - iter = SeekableCharIterator(node) - iter.seekforward(start) - c = iter.next() - while m + i < stop: - if c == substring[i]: - i += 1 - if i == len2: - return m - if m + i < stop: - c = iter.next() - else: - # mismatch, go back to the last possible starting pos - if i==0: - m += 1 - if m + i < stop: - c = iter.next() - else: - e = restart[i-1] - new_m = m + i - e - assert new_m <= m + i - seek = m + i - new_m - if seek: - iter.seekback(m + i - new_m) - c = iter.next() - m = new_m - i = e - return -1 - def _find_node(node, subnode, start, stop, restart): len2 = subnode.length() m = start - iter = SeekableCharIterator(node) + iter = SeekableItemIterator(node) iter.seekforward(start) - c = iter.next() + c = iter.nextint() i = 0 - subiter = SeekableCharIterator(subnode) - d = subiter.next() + subiter = SeekableItemIterator(subnode) + d = subiter.nextint() while m + i < stop: if c == d: i += 1 if i == len2: return m - d = subiter.next() + d = subiter.nextint() if m + i < stop: - c = iter.next() + c = iter.nextint() else: # mismatch, go back to the last possible starting pos if i == 0: m += 1 if m + i < stop: - c = iter.next() + c = iter.nextint() else: e = restart[i - 1] new_m = m + i - e @@ -640,71 +735,51 @@ seek = m + i - new_m if seek: iter.seekback(m + i - new_m) - c = iter.next() + c = iter.nextint() m = new_m subiter.seekback(i - e + 1) - d = subiter.next() + d = subiter.nextint() i = e return -1 -def construct_restart_positions(s): - l = len(s) - restart = [0] * l - restart[0] = 0 - i = 1 - j = 0 - while i < l: - if s[i] == s[j]: - j += 1 - restart[i] = j - i += 1 - elif j>0: - j = restart[j-1] - else: - restart[i] = 0 - i += 1 - j = 0 - return restart - def construct_restart_positions_node(node): - # really a bit overkill - l = node.length() - restart = [0] * l + length = node.length() + restart = [0] * length restart[0] = 0 i = 1 j = 0 - iter1 = CharIterator(node) - iter1.next() - c1 = iter1.next() - iter2 = SeekableCharIterator(node) - c2 = iter2.next() - while i < l: + iter1 = ItemIterator(node) + iter1.nextint() + c1 = iter1.nextint() + iter2 = SeekableItemIterator(node) + c2 = iter2.nextint() + while 1: if c1 == c2: j += 1 - if j != l: - c2 = iter2.next() + if j < length: + c2 = iter2.nextint() restart[i] = j i += 1 - if i != l: - c1 = iter1.next() + if i < length: + c1 = iter1.nextint() else: break elif j>0: new_j = restart[j-1] assert new_j < j - iter2.seekback(j - new_j) - c2 = iter2.next() + iter2.seekback(j - new_j + 1) + c2 = iter2.nextint() j = new_j else: restart[i] = 0 i += 1 - if i != l: - c1 = iter1.next() + if i < length: + c1 = iter1.nextint() else: break j = 0 - iter2 = SeekableCharIterator(node) - c2 = iter2.next() + iter2 = SeekableItemIterator(node) + c2 = iter2.nextint() return restart def view(objs): @@ -785,37 +860,82 @@ return result -class CharIterator(object): - def __init__(self, node): +class ItemIterator(object): + def __init__(self, node, start=0): self.iter = FringeIterator(node) self.node = None self.nodelength = 0 self.index = 0 + if start: + self._advance_to(start) + + def _advance_to(self, index): + assert index > 0 + assert self.index == 0 + while 1: + node = self.iter.next() + length = node.length() + if index < length: + self.index = index + self.node = node + self.nodelength = length + break + index -= length + assert index >= 0 - def next(self): + def getnode(self): node = self.node if node is None: while 1: node = self.node = self.iter.next() nodelength = self.nodelength = node.length() if nodelength != 0: - break - self.index = 0 + self.index = 0 + return node + return node + + def advance_index(self): index = self.index - result = node.getitem(index) if index == self.nodelength - 1: self.node = None else: self.index = index + 1 + + def nextchar(self): + node = self.getnode() + index = self.index + result = node.getchar(self.index) + self.advance_index() return result -class ReverseCharIterator(object): + def nextunichar(self): + node = self.getnode() + index = self.index + result = node.getunichar(self.index) + self.advance_index() + return result + + def nextrope(self): + node = self.getnode() + index = self.index + result = node.getrope(self.index) + self.advance_index() + return result + + def nextint(self): + node = self.getnode() + index = self.index + result = node.getint(self.index) + self.advance_index() + return result + +class ReverseItemIterator(object): def __init__(self, node): self.iter = ReverseFringeIterator(node) self.node = None self.index = 0 - def next(self): + def getnode(self): node = self.node index = self.index if node is None: @@ -823,16 +943,36 @@ node = self.node = self.iter.next() index = self.index = node.length() - 1 if index != -1: - break - result = node.getitem(index) - if index == 0: + return node + return node + + + def advance_index(self): + if self.index == 0: self.node = None else: - self.index = index - 1 + self.index -= 1 + + def nextchar(self): + node = self.getnode() + result = node.getchar(self.index) + self.advance_index() return result + def nextint(self): + node = self.getnode() + result = node.getint(self.index) + self.advance_index() + return result -class SeekableCharIterator(object): + def nextunichar(self): + node = self.getnode() + result = node.getunichar(self.index) + self.advance_index() + return result + + +class SeekableItemIterator(object): def __init__(self, node): self.iter = SeekableFringeIterator(node) self.node = self.nextnode() @@ -848,15 +988,34 @@ self.index = 0 return node - def next(self): + + def advance_index(self): + if self.index == self.nodelength - 1: + self.node = None + self.index += 1 + + def nextchar(self): node = self.node if node is None: node = self.nextnode() - index = self.index - result = self.node.getitem(index) - if self.index == self.nodelength - 1: - self.node = None - self.index = index + 1 + result = self.node.getchar(self.index) + self.advance_index() + return result + + def nextunichar(self): + node = self.node + if node is None: + node = self.nextnode() + result = self.node.getunichar(self.index) + self.advance_index() + return result + + def nextint(self): + node = self.node + if node is None: + node = self.nextnode() + result = self.node.getint(self.index) + self.advance_index() return result def seekforward(self, numchars): @@ -898,16 +1057,16 @@ class FindIterator(object): def __init__(self, node, sub, start=0, stop=-1): self.node = node + self.sub = sub len1 = self.length = node.length() - substring = self.substring = sub.flatten() # XXX for now - len2 = len(substring) + len2 = sub.length() self.search_length = len2 if len2 == 0: self.restart_positions = None elif len2 == 1: self.restart_positions = None else: - self.restart_positions = construct_restart_positions(substring) + self.restart_positions = construct_restart_positions_node(sub) self.start = start if stop == -1 or stop > len1: stop = len1 @@ -921,8 +1080,8 @@ self.start += 1 return start elif self.search_length == 1: - result = find_char(self.node, self.substring[0], - self.start, self.stop) + result = find_int(self.node, self.sub.getint(0), + self.start, self.stop) if result == -1: self.start = self.length raise StopIteration @@ -930,8 +1089,8 @@ return result if self.start >= self.stop: raise StopIteration - result = _find(self.node, self.substring, self.start, - self.stop, self.restart_positions) + result = _find_node(self.node, self.sub, self.start, + self.stop, self.restart_positions) if result == -1: self.start = self.length raise StopIteration @@ -952,15 +1111,18 @@ if (isinstance(node1, LiteralStringNode) and isinstance(node2, LiteralStringNode)): return node1.s == node2.s - iter1 = CharIterator(node1) - iter2 = CharIterator(node2) + if (isinstance(node1, LiteralUnicodeNode) and + isinstance(node2, LiteralUnicodeNode)): + return node1.u == node2.u + iter1 = ItemIterator(node1) + iter2 = ItemIterator(node2) # XXX could be cleverer and detect partial equalities while 1: try: - c = iter1.next() + c = iter1.nextint() except StopIteration: return True - if c != iter2.next(): + if c != iter2.nextint(): return False def compare(node1, node2): @@ -975,16 +1137,211 @@ cmplen = min(len1, len2) i = 0 - iter1 = CharIterator(node1) - iter2 = CharIterator(node2) + iter1 = ItemIterator(node1) + iter2 = ItemIterator(node2) while i < cmplen: - diff = ord(iter1.next()) - ord(iter2.next()) + diff = iter1.nextint() - iter2.nextint() if diff != 0: return diff i += 1 return len1 - len2 +def startswith(self, prefix, start, end): + if prefix.length() == 0: + return True + if self.length() == 0: + return False + stop = start + prefix.length() + if stop > end: + return False + iter1 = ItemIterator(self, start) + iter2 = ItemIterator(prefix) + for i in range(prefix.length()): + if iter1.nextint() != iter2.nextint(): + return False + return True + +def endswith(self, suffix, start, end): + if suffix.length() == 0: + return True + if self.length() == 0: + return False + begin = end - suffix.length() + if begin < start: + return False + iter1 = ItemIterator(self, begin) + iter2 = ItemIterator(suffix) + for i in range(suffix.length()): + if iter1.nextint() != iter2.nextint(): + return False + return True + +def strip(node, left=True, right=True, predicate=lambda i: chr(i).isspace(), + *extraargs): + length = node.length() + + lpos = 0 + rpos = length + + if left: + iter = ItemIterator(node) + while lpos < rpos and predicate(iter.nextint(), *extraargs): + lpos += 1 + + if right: + iter = ReverseItemIterator(node) + while rpos > lpos and predicate(iter.nextint(), *extraargs): + rpos -= 1 + + assert rpos >= lpos + return getslice_one(node, lpos, rpos) +strip._annspecialcase_ = "specialize:arg(3)" + +def split(node, sub, maxsplit=-1): + startidx = 0 + substrings = [] + iter = FindIterator(node, sub) + while maxsplit != 0: + try: + foundidx = iter.next() + except StopIteration: + break + substrings.append(getslice_one(node, startidx, foundidx)) + startidx = foundidx + sub.length() + maxsplit = maxsplit - 1 + substrings.append(getslice_one(node, startidx, node.length())) + return substrings + + +def split_chars(node, maxsplit=-1, predicate=lambda x: chr(x).isspace()): + result = [] + length = node.length() + if not length: + return result + i = 0 + iter = ItemIterator(node) + while True: + # find the beginning of the next word + while i < length: + if not predicate(iter.nextint()): + break # found + i += 1 + else: + break # end of string, finished + + # find the end of the word + if maxsplit == 0: + j = length # take all the rest of the string + else: + j = i + 1 + while j < length and not predicate(iter.nextint()): + j += 1 + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + # the word is value[i:j] + result.append(getslice_one(node, i, j)) + + # continue to look from the character following the space after the word + i = j + 1 + return result + + +def rsplit_chars(node, maxsplit=-1, predicate=lambda x: chr(x).isspace()): + result = [] + length = node.length() + i = length - 1 + iter = ReverseItemIterator(node) + while True: + # starting from the end, find the end of the next word + while i >= 0: + if not predicate(iter.nextint()): + break # found + i -= 1 + else: + break # end of string, finished + + # find the start of the word + # (more precisely, 'j' will be the space character before the word) + if maxsplit == 0: + j = -1 # take all the rest of the string + else: + j = i - 1 + while j >= 0 and not predicate(iter.nextint()): + j -= 1 + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + # the word is value[j+1:i+1] + j1 = j + 1 + assert j1 >= 0 + result.append(getslice_one(node, j1, i + 1)) + + # continue to look from the character before the space before the word + i = j - 1 + + result.reverse() + return result + + +def split_completely(node, maxsplit=-1): + upper = node.length() + if maxsplit > 0 and maxsplit < upper + 2: + upper = maxsplit - 1 + assert upper >= 0 + substrings = [by] + iter = ItemIterator(node) + for i in range(upper): + substrings.append(iter.nextrope()) + substrings.append(rope.getslice_one(node, upper, length)) + + +def splitlines(node, keepends=False): + length = node.length() + if length == 0: + return [] + + result = [] + iter = ItemIterator(node) + i = j = 0 + last = ord(" ") + char = iter.nextint() + while i < length: + # Find a line and append it + while char != ord('\n') and char != ord('\r'): + try: + i += 1 + last = char + char = iter.nextint() + except StopIteration: + break + # Skip the line break reading CRLF as one line break + eol = i + i += 1 + last = char + try: + char = iter.nextint() + except StopIteration: + pass + else: + if last == ord('\r') and char == ord('\n'): + i += 1 + try: + last = char + char = iter.nextint() + except StopIteration: + pass + if keepends: + eol = i + result.append(getslice_one(node, j, eol)) + j = i + + if j == 0: + result.append(node) + elif j < length: + result.append(getslice_one(node, j, length)) + + return result + # __________________________________________________________________________ # misc @@ -994,6 +1351,82 @@ return -1 x = rope.hash_part() x <<= 1 # get rid of the bit that is always set - x ^= ord(rope.getitem(0)) + x ^= rope.getint(0) x ^= rope.length() return intmask(x) + +# ____________________________________________________________ +# to and from unicode conversion + +def str_decode_ascii(rope): + assert rope.is_bytestring() + if rope.is_ascii(): + return rope + return None + +def str_decode_latin1(rope): + assert rope.is_bytestring() + return rope + +def str_decode_utf8(rope): + from pypy.rlib.runicode import str_decode_utf_8 + if rope.is_ascii(): + return rope + elif isinstance(rope, BinaryConcatNode): + lresult = str_decode_utf8(rope.left) + if lresult is not None: + return BinaryConcatNode(lresult, + str_decode_utf8(rope.right)) + elif isinstance(rope, LiteralStringNode): + try: + result, consumed = str_decode_utf_8(rope.s, len(rope.s), "strict", + False) + except UnicodeDecodeError: + return None + if consumed < len(rope.s): + return None + return rope_from_unicode(result) + s = rope.flatten_string() + try: + result, consumed = str_decode_utf_8(s, len(s), "strict", True) + return rope_from_unicode(result) + except UnicodeDecodeError: + pass + + +def unicode_encode_ascii(rope): + if rope.is_ascii(): + return rope + +def unicode_encode_latin1(rope): + if rope.is_bytestring(): + return rope + +def unicode_encode_utf8(rope): + from pypy.rlib.runicode import unicode_encode_utf_8 + if rope.is_ascii(): + return rope + elif isinstance(rope, BinaryConcatNode): + return BinaryConcatNode(unicode_encode_utf8(rope.left), + unicode_encode_utf8(rope.right)) + elif isinstance(rope, LiteralUnicodeNode): + return LiteralStringNode( + unicode_encode_utf_8(rope.u, len(rope.u), "strict")) + elif isinstance(rope, LiteralStringNode): + return LiteralStringNode(_str_encode_utf_8(rope.s)) + +def _str_encode_utf_8(s): + size = len(s) + result = [] + i = 0 + while i < size: + ch = ord(s[i]) + i += 1 + if (ch < 0x80): + # Encode ASCII + result.append(chr(ch)) + continue + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + return "".join(result) Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Thu Nov 22 15:50:13 2007 @@ -11,12 +11,14 @@ from pypy.objspace.std.stringtype import wrapchar from pypy.objspace.std import rope -from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY +from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY,\ + _upper, _lower class W_RopeObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef def __init__(w_self, node): + assert node.is_bytestring() w_self._node = node def __repr__(w_self): @@ -24,7 +26,7 @@ return "%s(%r)" % (w_self.__class__.__name__, w_self._node) def unwrap(w_self, space): - return w_self._node.flatten() + return w_self._node.flatten_string() def create_if_subclassed(w_self): if type(w_self) is W_RopeObject: @@ -49,7 +51,7 @@ def __init__(w_self, w_rope, index=0): w_self.node = node = w_rope._node - w_self.char_iter = rope.CharIterator(node) + w_self.item_iter = rope.ItemIterator(node) w_self.index = index registerimplementation(W_RopeIterObject) @@ -58,27 +60,13 @@ l = w_self._node.length() if l == 0: return space.w_False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for i in range(l): - if not fun(iter.next()): + if not fun(iter.nextchar()): return space.w_False return space.w_True _is_generic._annspecialcase_ = "specialize:arg(2)" -def _upper(ch): - if ch.islower(): - o = ord(ch) - 32 - return chr(o) - else: - return ch - -def _lower(ch): - if ch.isupper(): - o = ord(ch) + 32 - return chr(o) - else: - return ch - _isspace = lambda c: c.isspace() _isdigit = lambda c: c.isdigit() _isalpha = lambda c: c.isalpha() @@ -104,9 +92,9 @@ if l == 0: return space.w_False cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for idx in range(l): - c = iter.next() + c = iter.nextchar() if c.islower(): return space.w_False elif not cased and c.isupper(): @@ -121,9 +109,9 @@ if l == 0: return space.w_False cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for idx in range(l): - c = iter.next() + c = iter.nextchar() if c.isupper(): return space.w_False elif not cased and c.islower(): @@ -138,9 +126,9 @@ cased = False previous_is_cased = False - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(w_self._node) for pos in range(0, w_self._node.length()): - ch = iter.next() + ch = iter.nextchar() if ch.isupper(): if previous_is_cased: return space.w_False @@ -155,42 +143,35 @@ return space.newbool(cased) -def str_upper__Rope(space, w_self): - l = w_self._node.length() +def _local_transform(node, transform): + l = node.length() res = [' '] * l - iter = rope.CharIterator(w_self._node) + iter = rope.ItemIterator(node) for i in range(l): - ch = iter.next() - res[i] = _upper(ch) + ch = iter.nextchar() + res[i] = transform(ch) return W_RopeObject(rope.rope_from_charlist(res)) +_local_transform._annspecialcase_ = "specialize:arg(1)" + +def str_upper__Rope(space, w_self): + return _local_transform(w_self._node, _upper) def str_lower__Rope(space, w_self): - l = w_self._node.length() - res = [' '] * l - iter = rope.CharIterator(w_self._node) - for i in range(l): - ch = iter.next() - res[i] = _lower(ch) + return _local_transform(w_self._node, _lower) - return W_RopeObject(rope.rope_from_charlist(res)) +def _swapcase(ch): + if ch.isupper(): + o = ord(ch) + 32 + return chr(o) + elif ch.islower(): + o = ord(ch) - 32 + return chr(o) + else: + return ch def str_swapcase__Rope(space, w_self): - l = w_self._node.length() - res = [' '] * l - iter = rope.CharIterator(w_self._node) - for i in range(l): - ch = iter.next() - if ch.isupper(): - o = ord(ch) + 32 - res[i] = chr(o) - elif ch.islower(): - o = ord(ch) - 32 - res[i] = chr(o) - else: - res[i] = ch - - return W_RopeObject(rope.rope_from_charlist(res)) + return _local_transform(w_self._node, _swapcase) def str_capitalize__Rope(space, w_self): @@ -198,8 +179,8 @@ length = node.length() buffer = [' '] * length if length > 0: - iter = rope.CharIterator(node) - ch = iter.next() + iter = rope.ItemIterator(node) + ch = iter.nextchar() if ch.islower(): o = ord(ch) - 32 buffer[0] = chr(o) @@ -207,7 +188,7 @@ buffer[0] = ch for i in range(1, length): - ch = iter.next() + ch = iter.nextchar() if ch.isupper(): o = ord(ch) + 32 buffer[i] = chr(o) @@ -218,15 +199,16 @@ return W_RopeObject(rope.rope_from_charlist(buffer)) + def str_title__Rope(space, w_self): node = w_self._node length = node.length() buffer = [' '] * length prev_letter = ' ' - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) for pos in range(0, length): - ch = iter.next() + ch = iter.nextchar() if not prev_letter.isalpha(): buffer[pos] = _upper(ch) else: @@ -237,106 +219,39 @@ return W_RopeObject(rope.rope_from_charlist(buffer)) def str_split__Rope_None_ANY(space, w_self, w_none, w_maxsplit=-1): + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - res_w = [] - node = w_self._node - length = node.length() - i = 0 - iter = rope.CharIterator(node) - while True: - # find the beginning of the next word - while i < length: - if not iter.next().isspace(): - break # found - i += 1 - else: - break # end of string, finished - - # find the end of the word - if maxsplit == 0: - j = length # take all the rest of the string - else: - j = i + 1 - while j < length and not iter.next().isspace(): - j += 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[i:j] - res_w.append(W_RopeObject(rope.getslice_one(node, i, j))) - - # continue to look from the character following the space after the word - i = j + 1 - + res_w = [W_RopeObject(node) + for node in rope.split_chars(selfnode, maxsplit)] return space.newlist(res_w) - def str_split__Rope_Rope_ANY(space, w_self, w_by, w_maxsplit=-1): maxsplit = space.int_w(w_maxsplit) - res_w = [] - start = 0 selfnode = w_self._node bynode = w_by._node bylen = bynode.length() if bylen == 0: raise OperationError(space.w_ValueError, space.wrap("empty separator")) - - iter = rope.FindIterator(selfnode, bynode) - while maxsplit != 0: - try: - next = iter.next() - except StopIteration: - break - res_w.append(W_RopeObject(rope.getslice_one(selfnode, start, next))) - start = next + bylen - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - res_w.append(W_RopeObject(rope.getslice_one( - selfnode, start, selfnode.length()))) + res_w = [W_RopeObject(node) + for node in rope.split(selfnode, bynode, maxsplit)] return space.newlist(res_w) + def str_rsplit__Rope_None_ANY(space, w_self, w_none, w_maxsplit=-1): - # XXX works but flattens + selfnode = w_self._node maxsplit = space.int_w(w_maxsplit) - res_w = [] - value = w_self._node.flatten() - i = len(value)-1 - while True: - # starting from the end, find the end of the next word - while i >= 0: - if not value[i].isspace(): - break # found - i -= 1 - else: - break # end of string, finished - - # find the start of the word - # (more precisely, 'j' will be the space character before the word) - if maxsplit == 0: - j = -1 # take all the rest of the string - else: - j = i - 1 - while j >= 0 and not value[j].isspace(): - j -= 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[j+1:i+1] - j1 = j + 1 - assert j1 >= 0 - res_w.append(space.wrap(value[j1:i+1])) - - # continue to look from the character before the space before the word - i = j - 1 - - res_w.reverse() + res_w = [W_RopeObject(node) + for node in rope.rsplit_chars(selfnode, maxsplit)] return space.newlist(res_w) + def str_rsplit__Rope_Rope_ANY(space, w_self, w_by, w_maxsplit=-1): # XXX works but flattens maxsplit = space.int_w(w_maxsplit) res_w = [] - value = w_self._node.flatten() + value = w_self._node.flatten_string() end = len(value) - by = w_by._node.flatten() + by = w_by._node.flatten_string() bylen = len(by) if bylen == 0: raise OperationError(space.w_ValueError, space.wrap("empty separator")) @@ -446,8 +361,8 @@ def str_rfind__Rope_Rope_ANY_ANY(space, w_self, w_sub, w_start, w_end): # XXX works but flattens (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - self = self.flatten() - sub = sub.flatten() + self = self.flatten_string() + sub = sub.flatten_string() res = self.rfind(sub, start, end) return wrapint(space, res) @@ -475,8 +390,8 @@ if not sub.length(): raise OperationError(space.w_ValueError, space.wrap("empty separator")) - flattened_self = self.flatten() - flattened_sub = sub.flatten() + flattened_self = self.flatten_string() + flattened_sub = sub.flatten_string() pos = flattened_self.rfind(flattened_sub) if pos == -1: return space.newtuple([W_RopeObject.EMPTY, W_RopeObject.EMPTY, w_self]) @@ -501,8 +416,8 @@ def str_rindex__Rope_Rope_ANY_ANY(space, w_self, w_sub, w_start, w_end): (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) # XXX works but flattens - self = self.flatten() - sub = sub.flatten() + self = self.flatten_string() + sub = sub.flatten_string() res = self.rfind(sub, start, end) if res < 0: raise OperationError(space.w_ValueError, @@ -527,9 +442,9 @@ upper = maxsplit - 1 assert upper >= 0 substrings = [by] - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) for i in range(upper): - substrings.append(rope.LiteralStringNode.PREBUILT[ord(iter.next())]) + substrings.append(iter.nextrope()) substrings.append(by) substrings.append(rope.getslice_one(node, upper, length)) try: @@ -537,91 +452,39 @@ except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) - startidx = 0 - substrings = [] - iter = rope.FindIterator(node, sub) - try: - foundidx = iter.next() - except StopIteration: + substrings = rope.split(node, sub, maxsplit) + if not substrings: return w_self.create_if_subclassed() - while maxsplit != 0: - substrings.append(rope.getslice_one(node, startidx, foundidx)) - startidx = foundidx + sub.length() - try: - foundidx = iter.next() - except StopIteration: - break - maxsplit = maxsplit - 1 - substrings.append(rope.getslice_one(node, startidx, length)) try: return W_RopeObject(rope.join(by, substrings)) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) -def _strip(space, w_self, w_chars, left, right): - "internal function called by str_xstrip methods" - node = w_self._node - length = node.length() - u_chars = space.str_w(w_chars) - - lpos = 0 - rpos = length - - if left: - #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.CharIterator(node) - while lpos < rpos and iter.next() in u_chars: - lpos += 1 - - if right: - iter = rope.ReverseCharIterator(node) - while rpos > lpos and iter.next() in u_chars: - rpos -= 1 - - return W_RopeObject(rope.getslice_one(node, lpos, rpos)) -def _strip_none(space, w_self, left, right): - "internal function called by str_xstrip methods" - node = w_self._node - length = node.length() - - lpos = 0 - rpos = length - - if left: - #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - iter = rope.CharIterator(node) - while lpos < rpos and iter.next().isspace(): - lpos += 1 - - if right: - iter = rope.ReverseCharIterator(node) - while rpos > lpos and iter.next().isspace(): - rpos -= 1 - - assert rpos >= lpos # annotator hint, don't remove - return W_RopeObject(rope.getslice_one(node, lpos, rpos)) +def _contains(i, string): + return chr(i) in string def str_strip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=1, right=1) + return W_RopeObject(rope.strip(w_self._node, True, True, + _contains, w_chars._node.flatten_string())) def str_strip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=1) + return W_RopeObject(rope.strip(w_self._node, left=True, right=True)) def str_rstrip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=0, right=1) + return W_RopeObject(rope.strip(w_self._node, False, True, + _contains, w_chars._node.flatten_string())) def str_rstrip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=0, right=1) + return W_RopeObject(rope.strip(w_self._node, False, True)) - def str_lstrip__Rope_Rope(space, w_self, w_chars): - return _strip(space, w_self, w_chars, left=1, right=0) + return W_RopeObject(rope.strip(w_self._node, True, False, + _contains, w_chars._node.flatten_string())) def str_lstrip__Rope_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=0) - + return W_RopeObject(rope.strip(w_self._node, left=True, right=False)) def str_center__Rope_ANY_ANY(space, w_self, w_arg, w_fillchar): @@ -663,65 +526,41 @@ i += 1 return wrapint(space, i) -def ropeendswith(self, suffix, start, end): - if suffix.length() == 0: - return True - if self.length() == 0: - return False - begin = end - suffix.length() - if begin < start: - return False - iter1 = rope.SeekableCharIterator(self) - iter1.seekforward(begin) - iter2 = rope.CharIterator(suffix) - for i in range(suffix.length()): - if iter1.next() != iter2.next(): - return False - return True - def str_endswith__Rope_Rope_ANY_ANY(space, w_self, w_suffix, w_start, w_end): (self, suffix, start, end) = _convert_idx_params(space, w_self, w_suffix, w_start, w_end) - return space.newbool(ropeendswith(self, suffix, start, end)) + return space.newbool(rope.endswith(self, suffix, start, end)) def str_endswith__Rope_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): + if space.is_true(space.isinstance(w_suffix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "endswith", w_suffixes, w_start, + w_end) suffix = rope_w(space, w_suffix) - if ropeendswith(self, suffix, start, end): + if rope.endswith(self, suffix, start, end): return space.w_True return space.w_False -def ropestartswith(self, prefix, start, end): - if prefix.length() == 0: - return True - if self.length() == 0: - return False - stop = start + prefix.length() - if stop > end: - return False - iter1 = rope.SeekableCharIterator(self) - iter1.seekforward(start) - iter2 = rope.CharIterator(prefix) - for i in range(prefix.length()): - if iter1.next() != iter2.next(): - return False - return True - def str_startswith__Rope_Rope_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (self, prefix, start, end) = _convert_idx_params(space, w_self, w_prefix, w_start, w_end) - return space.newbool(ropestartswith(self, prefix, start, end)) + return space.newbool(rope.startswith(self, prefix, start, end)) def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): + if space.is_true(space.isinstance(w_prefix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "startswith", w_prefixes, w_start, + w_end) prefix = rope_w(space, w_prefix) - if ropestartswith(self, prefix, start, end): + if rope.startswith(self, prefix, start, end): return space.w_True return space.w_False @@ -732,16 +571,16 @@ distance = tabsize if length: distance = 0 - iter = rope.ReverseCharIterator(node) + iter = rope.ReverseItemIterator(node) while 1: # no sophisticated linebreak support now # '\r' just for passing adapted CPython test try: - char = iter.next() + char = iter.nextint() except StopIteration: break - if char == "\n" or char == "\r": + if char == ord("\n") or char == ord("\r"): break distance += 1 @@ -760,86 +599,22 @@ return W_RopeObject.EMPTY tabsize = space.int_w(w_tabsize) - expanded = [] - iter = rope.FindIterator(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) - #split = u_self.split("\t") - #u_expanded = oldtoken = split.pop(0) - - #for token in split: - # u_expanded += " " * _tabindent(oldtoken,u_tabsize) + token - # oldtoken = token - start = 0 - try: - start = iter.next() - last = rope.getslice_one(node, 0, start) - start += 1 - except StopIteration: - return w_self.create_if_subclassed() - expanded.append(last) - while 1: + splitted = rope.split(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) + last = splitted[0] + expanded = [last] + for i in range(1, len(splitted)): expanded.append(rope.multiply(rope.LiteralStringNode.PREBUILT[ord(" ")], _tabindent(last, tabsize))) - try: - next = iter.next() - except StopIteration: - break - last = rope.getslice_one(node, start, next) + last = splitted[i] expanded.append(last) - start = next + 1 - - expanded.append(rope.getslice_one(node, start, length)) return W_RopeObject(rope.rebalance(expanded)) def str_splitlines__Rope_ANY(space, w_self, w_keepends): - #import pdb; pdb.set_trace() - keepends = bool(space.int_w(w_keepends)) # truth value, but type checked + keepends = bool(space.int_w(w_keepends)) # truth value, but type checked node = w_self._node - length = node.length() - if length == 0: - return space.newlist([]) - - strs_w = [] - iter = rope.CharIterator(node) - i = j = 0 - last = " " - char = iter.next() - while i < length: - # Find a line and append it - while char != '\n' and char != '\r': - try: - i += 1 - last = char - char = iter.next() - except StopIteration: - break - # Skip the line break reading CRLF as one line break - eol = i - i += 1 - last = char - try: - char = iter.next() - except StopIteration: - pass - else: - if last == '\r' and char == '\n': - i += 1 - try: - last = char - char = iter.next() - except StopIteration: - pass - if keepends: - eol = i - strs_w.append(W_RopeObject(rope.getslice_one(node, j, eol))) - j = i - - if j == 0: - strs_w.append(w_self.create_if_subclassed()) - elif j < length: - strs_w.append(W_RopeObject(rope.getslice_one(node, j, length))) - - return space.newlist(strs_w) + return space.newlist( + [W_RopeObject(n) for n in rope.splitlines(node, keepends)]) def str_zfill__Rope_ANY(space, w_self, w_width): node = w_self._node @@ -853,7 +628,7 @@ return W_RopeObject(rope.multiply(zero, width)) middle = width - length - firstchar = node.getitem(0) + firstchar = node.getchar(0) if length > 0 and (firstchar == '+' or firstchar == '-'): return W_RopeObject(rope.rebalance( [rope.LiteralStringNode.PREBUILT[ord(firstchar)], @@ -865,7 +640,7 @@ rope.multiply(zero, middle), node)) def str_w__Rope(space, w_str): - return w_str._node.flatten() + return w_str._node.flatten_string() def hash__Rope(space, w_str): return wrapint(space, rope.hash_rope(w_str._node)) @@ -910,7 +685,7 @@ exc = space.call_function(space.w_IndexError, space.wrap("string index out of range")) raise OperationError(space.w_IndexError, exc) - return wrapchar(space, node.getitem(ival)) + return wrapchar(space, node.getchar(ival)) def getitem__Rope_Slice(space, w_str, w_slice): node = w_str._node @@ -930,13 +705,6 @@ if mul <= 0: return W_RopeObject.EMPTY node = w_str._node - length = node.length() -# try: -# buflen = ovfcheck(mul * length) -# except OverflowError: -# raise OperationError( -# space.w_OverflowError, -# space.wrap("repeated string is too long: %d %d" % (length, mul))) try: return W_RopeObject(rope.multiply(node, mul)) except OverflowError: @@ -966,8 +734,8 @@ return w_str return W_RopeObject(w_str._node) -def iter__Rope(space, w_list): - return W_RopeIterObject(w_list) +def iter__Rope(space, w_str): + return W_RopeIterObject(w_str) def ord__Rope(space, w_str): node = w_str._node @@ -976,7 +744,7 @@ space.w_TypeError, space.wrap("ord() expected a character, but string " "of length %d found"% (w_str._node.length(),))) - return space.wrap(ord(node.flatten()[0])) + return space.wrap(ord(node.flatten_string()[0])) def getnewargs__Rope(space, w_str): return space.newtuple([W_RopeObject(w_str._node)]) @@ -989,15 +757,16 @@ buf = [' '] * (length * 4 + 2) # safely overallocate quote = "'" - if rope.find_char(node, quote) != -1 and rope.find_char(node, '"') == -1: + if (rope.find_int(node, ord(quote)) != -1 and + rope.find_int(node, ord('"')) == -1): quote = '"' buf[0] = quote - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) while 1: try: - c = iter.next() + c = iter.nextchar() i += 1 except StopIteration: break @@ -1047,10 +816,10 @@ node = w_string._node chars = [] - iter = rope.CharIterator(node) + iter = rope.ItemIterator(node) while 1: try: - c = iter.next() + c = iter.nextchar() w_char = W_RopeObject.PREBUILT[ord(c)] if not space.is_true(space.contains(w_deletechars, w_char)): chars.append(table[ord(c)]) @@ -1060,11 +829,12 @@ def str_decode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ - unicode_from_string, decode_object + getdefaultencoding + from pypy.objspace.std.ropeunicodeobject import decode_string encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - if encoding is None and errors is None: - return unicode_from_string(space, w_string) - return decode_object(space, w_string, encoding, errors) + if encoding is None: + encoding = getdefaultencoding(space) + return decode_string(space, w_string, encoding, errors) def str_encode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ @@ -1082,7 +852,7 @@ if w_ropeiter.node is None: raise OperationError(space.w_StopIteration, space.w_None) try: - char = w_ropeiter.char_iter.next() + char = w_ropeiter.item_iter.nextchar() w_item = space.wrap(char) except StopIteration: w_ropeiter.node = None Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Thu Nov 22 15:50:13 2007 @@ -589,6 +589,10 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): + if space.is_true(space.isinstance(w_suffix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "endswith", w_suffixes, w_start, + w_end) suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -603,6 +607,10 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): + if space.is_true(space.isinstance(w_prefix, space.w_unicode)): + w_u = space.call_function(space.w_unicode, w_self) + return space.call_method(w_u, "startswith", w_prefixes, w_start, + w_end) prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/dist/pypy/objspace/std/test/test_rope.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_rope.py (original) +++ pypy/dist/pypy/objspace/std/test/test_rope.py Thu Nov 22 15:50:13 2007 @@ -1,8 +1,9 @@ import py -import random +import random, sys from pypy.objspace.std.rope import * -def make_random_string(operations=10, slicing=True, print_seed=True): +def make_random_string(operations=10, slicing=True, print_seed=True, + unicode=False): seed = random.randrange(10000) if print_seed: print seed @@ -14,21 +15,27 @@ else: choice = [0, 1] for i in range(operations): - a = (chr(random.randrange(ord('a'), ord('z') + 1)) * - random.randrange(500)) + if not unicode: + a = (chr(random.randrange(ord('a'), ord('z') + 1)) * + random.randrange(500)) + node = LiteralStringNode(a) + else: + a = (unichr(random.randrange(sys.maxunicode)) * + random.randrange(500)) + node = LiteralUnicodeNode(a) c = random.choice(choice) if c == 0: - curr = curr + LiteralStringNode(a) + curr = curr + node st = st + a elif c == 1: - curr = LiteralStringNode(a) + curr + curr = node + curr st = a + st else: if len(st) < 10: continue start = random.randrange(len(st) // 3) stop = random.randrange(len(st) // 3 * 2, len(st)) - curr = curr[start: stop] + curr = getslice_one(curr, start, stop) st = st[start: stop] return curr, st @@ -38,20 +45,20 @@ LiteralStringNode("d" * 32) + LiteralStringNode("ef" * 32) + LiteralStringNode("")) assert s.depth() == 3 - assert s.flatten() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) + assert s.flatten_string() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) s = s.rebalance() - assert s.flatten() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) + assert s.flatten_string() == "".join([c * 32 for c in "a", "bc", "d", "ef"]) def test_dont_rebalance_again(): s = (LiteralStringNode("a" * 32) + LiteralStringNode("b" * 32) + LiteralStringNode("d" * 32) + LiteralStringNode("e" * 32) + LiteralStringNode("")) assert s.depth() == 3 - assert s.flatten() == "".join([c * 32 for c in "abde"]) + assert s.flatten_string() == "".join([c * 32 for c in "abde"]) s = s.rebalance() assert s.check_balanced() assert s.balanced - assert s.flatten() == "".join([c * 32 for c in "abde"]) + assert s.flatten_string() == "".join([c * 32 for c in "abde"]) def test_random_addition_test(): seed = random.randrange(10000) @@ -67,9 +74,9 @@ else: curr = LiteralStringNode(a) + curr st = a + st - assert curr.flatten() == st + assert curr.flatten_string() == st curr = curr.rebalance() - assert curr.flatten() == st + assert curr.flatten_string() == st def test_getitem(): result = "".join([c * 32 for c in "a", "bc", "d", "ef"]) @@ -79,7 +86,16 @@ s2 = s1.rebalance() for i in range(len(result)): for s in [s1, s2]: - assert s[i] == result[i] + assert s.getchar(i) == result[i] + assert s.getint(i) == ord(result[i]) + +def test_getitem_unicode(): + s1, result = make_random_string(200, unicode=True) + s2 = s1.rebalance() + for i in range(len(result)): + for s in [s1, s2]: + assert s.getunichar(i) == result[i] + assert s.getint(i) == ord(result[i]) def test_getslice(): result = "".join([c * 32 for c in "a", "bc", "d", "ef"]) @@ -90,14 +106,14 @@ for s in [s1, s2]: for start in range(0, len(result)): for stop in range(start, len(result)): - assert s[start:stop].flatten() == result[start:stop] + assert getslice_one(s, start, stop).flatten_string() == result[start:stop] def test_getslice_bug(): s1 = LiteralStringNode("/home/arigo/svn/pypy/branch/rope-branch/pypy/bin") s2 = LiteralStringNode("/pypy") s = s1 + s2 r = getslice_one(s, 1, 5) - assert r.flatten() == "home" + assert r.flatten_string() == "home" def test_getslice_step(): @@ -105,14 +121,29 @@ LiteralStringNode("nopqrstu") + LiteralStringNode("vwxyz") + LiteralStringNode("zyxwvut") + LiteralStringNode("srqpomnlk")) s2 = s1.rebalance() - result = s1.flatten() - assert s2.flatten() == result + result = s1.flatten_string() + assert s2.flatten_string() == result for s in [s1, s2]: for start in range(0, len(result)): for stop in range(start, len(result)): for step in range(1, stop - start): - assert s[start:stop:step].flatten() == result[start:stop:step] + assert getslice(s, start, stop, step).flatten_string() == result[start:stop:step] +def test_getslice_step_unicode(): + s1 = (LiteralUnicodeNode(u"\uaaaa") + + LiteralUnicodeNode(u"\ubbbb" * 5) + + LiteralUnicodeNode(u"\uaaaa\ubbbb\u1000\u2000") + + LiteralUnicodeNode(u"vwxyz") + + LiteralUnicodeNode(u"zyxwvu\u1234" * 2) + + LiteralUnicodeNode(u"12355")) + s2 = s1.rebalance() + result = s1.flatten_unicode() + assert s2.flatten_unicode() == result + for s in [s1, s2]: + for start in range(0, len(result)): + for stop in range(start, len(result)): + for step in range(1, stop - start): + assert getslice(s, start, stop, step).flatten_unicode() == result[start:stop:step] def test_random_addition_and_slicing(): seed = random.randrange(10000) @@ -141,27 +172,79 @@ #import pdb; pdb.set_trace() start = random.randrange(len(st) // 3) stop = random.randrange(len(st) // 3 * 2, len(st)) - curr = curr[start: stop] + curr = getslice_one(curr, start, stop) st = st[start: stop] - assert curr.flatten() == st + assert curr.flatten_string() == st curr = curr.rebalance() - assert curr.flatten() == st + assert curr.flatten_string() == st def test_iteration(): rope, real_st = make_random_string(200) - iter = CharIterator(rope) + iter = ItemIterator(rope) for c in real_st: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) + +def test_iteration_startpos(): + rope, real_st = make_random_string(200) + for i in range(0, len(real_st), len(real_st) // 20): + iter = ItemIterator(rope, i) + x = i + for c in real_st[i:]: + x += 1 + c2 = iter.nextchar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + +def test_iteration_unicode(): + rope, real_st = make_random_string(200, unicode=True) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ItemIterator(rope) + for c in real_st: + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) def test_reverse_iteration(): rope, real_st = make_random_string(200) - iter = ReverseCharIterator(rope) + iter = ReverseItemIterator(rope) for c in py.builtin.reversed(real_st): - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) + +def test_reverse_iteration_unicode(): + rope, real_st = make_random_string(200, unicode=True) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextunichar() + assert c2 == c + py.test.raises(StopIteration, iter.nextchar) + iter = ReverseItemIterator(rope) + for c in py.builtin.reversed(real_st): + c2 = iter.nextint() + assert c2 == ord(c) + py.test.raises(StopIteration, iter.nextchar) def test_multiply(): strs = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), @@ -175,7 +258,22 @@ for r, st in strs: for i in times: r2 = multiply(r, i) - assert r2.flatten() == st * i + assert r2.flatten_string() == st * i + +def test_multiply_unicode(): + strs = [(LiteralUnicodeNode(u"\uaaaa"), u"\uaaaa"), + (LiteralUnicodeNode(u"\uaaaa\ubbbb\ucccc"), u"\uaaaa\ubbbb\ucccc"), + make_random_string(500)] + times = range(100) + for i in range(9, 30): + times.append(i ** 2 - 1) + times.append(i ** 2) + times.append(i ** 2 + 1) + times.append(i ** 2 + 2) + for r, st in strs: + for i in times: + r2 = multiply(r, i) + assert r2.flatten_unicode() == st * i def test_join(): seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), @@ -186,7 +284,7 @@ l = list(l) for s, st in seps: node = join(s, l) - result1 = node.flatten() + result1 = node.flatten_string() result2 = st.join(strs) for i in range(node.length()): assert result1[i] == result2[i] @@ -196,7 +294,7 @@ ':', '213', '>'] l = [LiteralStringNode(s) for s in strings] node = join(LiteralStringNode(""), l) - assert node.flatten() == ''.join(strings) + assert node.flatten_string() == ''.join(strings) def test_join_random(): l, strs = zip(*[make_random_string(10 * i) for i in range(1, 5)]) @@ -205,7 +303,19 @@ make_random_string(500)] for s, st in seps: node = join(s, l) - result1 = node.flatten() + result1 = node.flatten_string() + result2 = st.join(strs) + for i in range(node.length()): + assert result1[i] == result2[i] + +def test_join_random_unicode(): + l, strs = zip(*[make_random_string(10 * i, unicode=True) for i in range(1, 5)]) + l = list(l) + seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), + make_random_string(500)] + for s, st in seps: + node = join(s, l) + result1 = node.flatten_unicode() result2 = st.join(strs) for i in range(node.length()): assert result1[i] == result2[i] @@ -214,18 +324,18 @@ rope = BinaryConcatNode(BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")), LiteralStringNode("ghi")) - iter = SeekableCharIterator(rope) + iter = SeekableItemIterator(rope) for c in "abcdefgh": - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c for i in range(7): iter.seekback(i) for c in "abcdefghi"[-1-i:-1]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - c2 = iter.next() + c2 = iter.nextchar() assert c2 == "i" - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) def test_fringe_iterator(): ABC = LiteralStringNode("abc") @@ -277,49 +387,48 @@ LiteralStringNode("def")), LiteralStringNode("ghi")) rope = rope + rope - result = rope.flatten() + result = rope.flatten_string() for j in range(len(result) - 1): for i in range(len(result) - 1 - j): - iter = SeekableCharIterator(rope) + iter = SeekableItemIterator(rope) # if (j, i) == (3, 1): # import pdb; pdb.set_trace() for c in result[:j]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c iter.seekforward(i) for c in result[i + j:]: - c2 = iter.next() + c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.nextchar) -def test_find_char(): +def test_find_int(): rope, st = make_random_string() - rope = rope[10:100] + rope = getslice_one(rope, 10, 100) st = st[10:100] for i in range(len(st)): - print i for j in range(i + 1, len(st)): c = st[i:j][(j - i) // 2] - pos = find_char(rope, c, i, j) + pos = find_int(rope, ord(c), i, j) assert pos == st.find(c, i, j) -def test_find_char_bugs(): - r = find_char(LiteralStringNode("ascii"), " ", 0, 5) +def test_find_int_bugs(): + r = find_int(LiteralStringNode("ascii"), ord(" "), 0, 5) assert r == -1 - r = find_char(LiteralStringNode("a"), "a") + r = find_int(LiteralStringNode("a"), ord("a")) assert r == 0 - r = find_char(BinaryConcatNode( - LiteralStringNode("x"), - SliceNode(1, 9, LiteralStringNode("a" * 10))), "a") - assert r == 1 def test_restart_positions(): restart = construct_restart_positions_node( BinaryConcatNode(LiteralStringNode("aba"), LiteralStringNode("bcabab"))) assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] - restart = construct_restart_positions("ababcabab") - assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4] + restart = construct_restart_positions_node( + BinaryConcatNode(LiteralStringNode("aba"), LiteralStringNode("bcababb"))) + assert restart == [0, 0, 1, 2, 0, 1, 2, 3, 4, 0] + restart = construct_restart_positions_node(LiteralStringNode("ababb")) + assert restart == [0, 0, 1, 2, 0] + def test_find(): node = BinaryConcatNode(LiteralStringNode("aba"), @@ -331,20 +440,31 @@ pos = find(node, LiteralStringNode("a"), 0, node.length()) assert pos == 6 + +def test_find_unicode(): + node = BinaryConcatNode(LiteralUnicodeNode(u"\uaaaa\ubbbb\uaaaa"), + LiteralUnicodeNode(u"\ubbbb\ucccc\uaaaa\ubbbb\uaaaa\ubbbb")) + pos = find(node, LiteralUnicodeNode(u"\uaaaa\ubbbb\ucccc"), 0, node.length()) + assert pos == 2 + node = BinaryConcatNode(LiteralUnicodeNode(u"btffp"), + LiteralUnicodeNode(u"b\uaaaacbb")) + pos = find(node, LiteralUnicodeNode(u"\uaaaa"), 0, node.length()) + assert pos == 6 + def test_fromcharlist(): for i in range(0, 100, 10): chars = ["a"] * 50 + ["b"] * i node = rope_from_charlist(chars) - assert node.flatten() == "a" * 50 + "b" * i - assert rope_from_charlist([]).flatten() == "" + assert node.flatten_string() == "a" * 50 + "b" * i + assert rope_from_charlist([]).flatten_string() == "" def test_find_iterator(): - for searchstring in ["abc", "a", "", "x", "xyz"]: + for searchstring in ["abc", "a", "", "x", "xyz", "abababcabcabb"]: node = join(LiteralStringNode(searchstring), [LiteralStringNode("cde" * i) for i in range(1, 10)]) - # node.view() + #node.view() iter = FindIterator(node, LiteralStringNode(searchstring)) - s = node.flatten() + s = node.flatten_string() assert s == searchstring.join(["cde" * i for i in range(1, 10)]) start = 0 while 1: @@ -357,18 +477,41 @@ assert r1 == r2 start = r2 + max(len(searchstring), 1) + +def test_find_iterator_unicode(): + for searchstring in [ + u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", + u"\u6666\u7777\u8888", + u"\uAAAA\uBBBB\uAAAA\uBBBB\uAAAA\uBBBB\uCCCC\uAAAA\uBBBB\uCCCC\uAAAA\uBBBB\uBBBB"]: + node = join(LiteralUnicodeNode(searchstring), + [LiteralUnicodeNode(u"\ucccc\udddd" * i) + for i in range(1, 10)]) + iter = FindIterator(node, LiteralUnicodeNode(searchstring)) + s = node.flatten_unicode() + assert s == searchstring.join([u"\ucccc\udddd" * i for i in range(1, 10)]) + start = 0 + while 1: + r2 = s.find(searchstring, start) + try: + r1 = iter.next() + except StopIteration: + assert r2 == -1 + break + assert r1 == r2 + start = r2 + max(len(searchstring), 1) + def test_hash(): - from pypy.rlib.rarithmetic import _hash_string, intmask + from pypy.rlib.rarithmetic import intmask for i in range(10): rope, _ = make_random_string() if rope.length() == 0: assert hash_rope(rope) == -1 continue h = hash_rope(rope) - x = LiteralStringNode(rope.flatten()).hash_part() + x = LiteralStringNode(rope.flatten_string()).hash_part() assert x == rope.hash_part() x <<= 1 - x ^= ord(rope.getitem(0)) + x ^= rope.getint(0) x ^= rope.length() assert intmask(x) == h # hash again to check for cache effects @@ -404,7 +547,6 @@ hashes[(h & 0xff0000) >> 16] += 1 for h in hashes: assert h > 300 - print hashes def test_hash_distribution_small_strings(): random.seed(42) # prevent randomly failing test @@ -466,13 +608,26 @@ assert s.hash_part() == h assert s.hash_part() == h +def test_hash_part_unicode(): + a, st = make_random_string(5, unicode=True) + h = a.hash_part() + for split in range(1, len(st) - 1): + s1 = LiteralUnicodeNode(st[:split]) + s2 = LiteralUnicodeNode(st[split:]) + s = BinaryConcatNode(s1, s2) + if h is None: + h = s.hash_part() + else: + # try twice due to caching reasons + assert s.hash_part() == h + assert s.hash_part() == h + def test_hash_part_more(): for i in range(100): rope, st = make_random_string() h = rope.hash_part() assert LiteralStringNode(st).hash_part() == h - def test_equality(): l = [make_random_string() for i in range(3)] l.append((LiteralStringNode(""), "")) @@ -480,6 +635,13 @@ for rope2, st2 in l: assert eq(rope1, rope2) == (st1 == st2) +def test_equality_unicode(): + l = [make_random_string() for i in range(3)] + l.append((LiteralStringNode(""), "")) + for rope1, st1 in l: + for rope2, st2 in l: + assert eq(rope1, rope2) == (st1 == st2) + def test_compare_random(): l = [make_random_string() for i in range(3)] l.append((LiteralStringNode(""), "")) @@ -490,30 +652,187 @@ c = c // abs(c) assert c == cmp(st1, st2) +def test_compare_random_unicode(): + l = [make_random_string() for i in range(3)] + l.append((LiteralStringNode(""), "")) + for rope1, st1 in l: + for rope2, st2 in l: + c = compare(rope1, rope2) + if c: + c = c // abs(c) + assert c == cmp(st1, st2) + + def test_power(): for i in range(0, 60, 13): print i for j in range(1, 10000, 7): assert intmask(i ** j) == masked_power(i, j) -def test_EfficientGetitemWraper(): - node1, _ = make_random_string(slicing=False) - node2 = EfficientGetitemWraper(node1) - for i in range(node2.length()): - assert node1.getitem(i) == node2.getitem(i) - for j in range(1000): - i = random.randrange(node1.length()) - assert node1.getitem(i) == node2.getitem(i) def test_seekable_bug(): node = BinaryConcatNode(LiteralStringNode("abc"), LiteralStringNode("def")) - iter = SeekableCharIterator(node) - c = iter.next(); assert c == "a" - c = iter.next(); assert c == "b" - c = iter.next(); assert c == "c" + iter = SeekableItemIterator(node) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" iter.seekback(1) - c = iter.next(); assert c == "c" - c = iter.next(); assert c == "d" - c = iter.next(); assert c == "e" - c = iter.next(); assert c == "f" - py.test.raises(StopIteration, iter.next) + c = iter.nextchar(); assert c == "c" + c = iter.nextchar(); assert c == "d" + c = iter.nextchar(); assert c == "e" + c = iter.nextchar(); assert c == "f" + py.test.raises(StopIteration, iter.nextchar) + node = LiteralStringNode("abcdef") + iter = SeekableItemIterator(node) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" + iter.seekback(3) + c = iter.nextchar(); assert c == "a" + c = iter.nextchar(); assert c == "b" + c = iter.nextchar(); assert c == "c" + +def test_strip(): + node = BinaryConcatNode(LiteralStringNode(" \t\n abc "), + LiteralStringNode("def ")) + r = strip(node) + assert r.flatten_string() == "abc def" + r = strip(node, left=False) + assert r.flatten_string() == " \t\n abc def" + r = strip(node, right=False) + assert r.flatten_string() == "abc def " + + node = BinaryConcatNode(LiteralStringNode("aaaYYYYa"), + LiteralStringNode("abab")) + predicate = lambda i: chr(i) in "ab" + r = strip(node, predicate=predicate) + assert r.flatten_string() == "YYYY" + r = strip(node, left=False, predicate=predicate) + assert r.flatten_string() == "aaaYYYY" + r = strip(node, right=False, predicate=predicate) + assert r.flatten_string() == "YYYYaabab" + +def test_getrope(): + s1, result = make_random_string(200, unicode=True) + s2 = s1.rebalance() + for i in range(len(result)): + for s in [s1, s2]: + r = s.getrope(i) + assert r.length() == 1 + assert r.getint(0) == s.getint(i) + assert isinstance(r, LiteralNode) + assert r.getint(0) >= 128 or isinstance(r, LiteralStringNode) + assert r.getrope(0) is r + +def test_split(): + seps = [(LiteralStringNode("a"), "a"), (LiteralStringNode("abc"), "abc"), + (LiteralStringNode("d"), "d"), (LiteralStringNode(""), "")] + l, strs = zip(*[(LiteralStringNode("x"), "x"), + (LiteralStringNode("xyz"), "xyz"), + (LiteralStringNode("w"), "w")]) + l = list(l) + for s, st in seps: + node = join(s, l) + l2 = split(node, s) + for n1, n2 in zip(l, l2): + assert n1.flatten_string() == n2.flatten_string() + for i in range(4): + l1 = split(LiteralStringNode("ababababa"), LiteralStringNode("b"), i) + l2 = "ababababa".split("b", i) + assert len(l1) == len(l2) + for n, s in zip(l1, l2): + assert n.flatten_string() == s + +def test_splitlines(): + seps = [(LiteralStringNode("\n"), "\n"), (LiteralStringNode("\r"), "\r"), + (LiteralStringNode("\r\n"), "\r\n")] + l, strs = zip(*[(LiteralStringNode("xafnarsp"), "xafnarsp"), + (LiteralStringNode("xyzaaaa"), "xyzaaaa"), + (LiteralStringNode("wxxxx"), "wxxxx")]) + l = list(l) + for s, st in seps: + node = join(s, l) + l2 = splitlines(node) + for n1, n2 in zip(l, l2): + assert n1.flatten_string() == n2.flatten_string() + for keepends in [True, False]: + l1 = splitlines(LiteralStringNode("ab\nab\n\raba\rba"), keepends) + l2 = "ab\nab\n\raba\rba".splitlines(keepends) + assert len(l1) == len(l2) + for n, s in zip(l1, l2): + assert n.flatten_string() == s + +def test_rope_from_unicode(): + node = rope_from_unicode(u"aaabbbbbcccdddeeefffggggnnn") + assert node.is_bytestring() + assert node.is_ascii() + node = rope_from_unicode(u"a" * 30 + u"\ufffd" * 30 + "x" * 30) + assert node.length() == 90 + assert not node.is_ascii() + assert not node.is_bytestring() + +def test_encode(): + node = LiteralStringNode("abc") + assert unicode_encode_ascii(node) is node + assert unicode_encode_latin1(node) is node + assert unicode_encode_utf8(node) is node + node = LiteralStringNode("abc\xff") + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is node + assert unicode_encode_utf8(node).s == 'abc\xc3\xbf' + node = LiteralUnicodeNode(u"\uffffab") + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is None + assert unicode_encode_utf8(node).s == '\xef\xbf\xbfab' + node = BinaryConcatNode(LiteralStringNode("abc"), + LiteralUnicodeNode(u"\uffffab")) + assert unicode_encode_ascii(node) is None + assert unicode_encode_latin1(node) is None + res = unicode_encode_utf8(node) + assert res.left is node.left + assert res.right.s == '\xef\xbf\xbfab' + +def test_decode(): + node = LiteralStringNode("abc") + assert str_decode_ascii(node) is node + assert str_decode_latin1(node) is node + assert str_decode_utf8(node) is node + node = LiteralStringNode("abc\xff") + assert str_decode_ascii(node) is None + assert str_decode_latin1(node) is node + +def test_decode_utf8(): + # bad data + node = LiteralStringNode("\xd7\x50") + assert str_decode_utf8(node) is None + node = LiteralStringNode("\xf0\x90\x91") + assert str_decode_utf8(node) is None + + # correct data in one node + node = LiteralStringNode('\xef\xbf\xbfab') + assert str_decode_utf8(node).u == u"\uffffab" + + # binary node, left node can be decoded + node = BinaryConcatNode(LiteralStringNode('\xef\xbf\xbfab'), + LiteralStringNode('\xef\xbf\xbfab')) + res = str_decode_utf8(node) + assert res.left.u == u"\uffffab" + assert res.right.u == u"\uffffab" + + # binary node, left node alone cannot be decoded + node = BinaryConcatNode(LiteralStringNode('\xef'), + LiteralStringNode('\xbf\xbfab')) + res = str_decode_utf8(node) + assert res.u == u"\uffffab" + + # binary node, left node cannot be decoded, bad data + node = BinaryConcatNode(LiteralStringNode("\xf0\x90"), + LiteralStringNode("\x91")) + assert str_decode_utf8(node) is None + + # binary node, incomplete data + node = BinaryConcatNode(LiteralStringNode('ab\xef'), + LiteralStringNode('\xbf')) + + res = str_decode_utf8(node) + assert res is None Modified: pypy/dist/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_unicodeobject.py Thu Nov 22 15:50:13 2007 @@ -305,6 +305,15 @@ assert not u'hello'.startswith((u'he', u'hel'), 0, 1) assert u'hello'.startswith((u'he', u'hel'), 0, 2) raises(TypeError, u'hello'.startswith, (42,)) + + def test_startswith_endswith_convert(self): + assert 'hello'.startswith((u'he\u1111', u'he')) + assert not 'hello'.startswith((u'lo\u1111', u'llo')) + assert 'hello'.startswith((u'hellox\u1111', u'hello')) + assert not 'hello'.startswith((u'lo', u'he\u1111'), 0, -1) + assert not 'hello'.endswith((u'he\u1111', u'he')) + assert 'hello'.endswith((u'\u1111lo', u'llo')) + assert 'hello'.endswith((u'\u1111hellox', u'hello')) def test_endswith(self): assert u'ab'.endswith(u'ab') is True @@ -463,3 +472,78 @@ "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'") assert (repr(u'\n') == "u'\\n'") + + + def test_partition(self): + + assert (u'this is the par', u'ti', u'tion method') == \ + u'this is the partition method'.partition(u'ti') + + # from raymond's original specification + S = u'http://www.python.org' + assert (u'http', u'://', u'www.python.org') == S.partition(u'://') + assert (u'http://www.python.org', u'', u'') == S.partition(u'?') + assert (u'', u'http://', u'www.python.org') == S.partition(u'http://') + assert (u'http://www.python.', u'org', u'') == S.partition(u'org') + + raises(ValueError, S.partition, u'') + raises(TypeError, S.partition, None) + + def test_rpartition(self): + + assert (u'this is the rparti', u'ti', u'on method') == \ + u'this is the rpartition method'.rpartition(u'ti') + + # from raymond's original specification + S = u'http://www.python.org' + assert (u'http', u'://', u'www.python.org') == S.rpartition(u'://') + assert (u'', u'', u'http://www.python.org') == S.rpartition(u'?') + assert (u'', u'http://', u'www.python.org') == S.rpartition(u'http://') + assert (u'http://www.python.', u'org', u'') == S.rpartition(u'org') + + raises(ValueError, S.rpartition, u'') + raises(TypeError, S.rpartition, None) + + + def test_rindex(self): + from sys import maxint + assert u'abcdefghiabc'.rindex(u'') == 12 + assert u'abcdefghiabc'.rindex(u'def') == 3 + assert u'abcdefghiabc'.rindex(u'abc') == 9 + assert u'abcdefghiabc'.rindex(u'abc', 0, -1) == 0 + assert u'abcdefghiabc'.rindex(u'abc', -4*maxint, 4*maxint) == 9 + raises(ValueError, u'abcdefghiabc'.rindex, u'hib') + raises(ValueError, u'defghiabc'.rindex, u'def', 1) + raises(ValueError, u'defghiabc'.rindex, u'abc', 0, -1) + raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, 8) + raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, -1) + raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', 0, 0.0) + raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', -10.0, 30) + + def test_len_iter(self): + assert len(iter(u"abcdef\uffff")) == 7 + for i in range(10): + assert len(iter(unicode(i))) == 1 + + def test_rfind(self): + assert u'abcdefghiabc'.rfind(u'abc') == 9 + assert u'abcdefghiabc'.rfind(u'') == 12 + assert u'abcdefghiabc'.rfind(u'abcd') == 0 + assert u'abcdefghiabc'.rfind(u'abcz') == -1 + + + def test_count(self): + assert u"".count(u"x") ==0 + assert u"".count(u"") ==1 + assert u"Python".count(u"") ==7 + assert u"ab aaba".count(u"ab") ==2 + assert 'aaa'.count('a') == 3 + assert 'aaa'.count('b') == 0 + assert 'aaa'.count('a', -1) == 1 + assert 'aaa'.count('a', -10) == 3 + assert 'aaa'.count('a', 0, -1) == 2 + assert 'aaa'.count('a', 0, -10) == 0 + assert 'ababa'.count('aba') == 1 + + def test_swapcase(self): + assert u'\xe4\xc4\xdf'.swapcase() == u'\xc4\xe4\xdf' Modified: pypy/dist/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodetype.py (original) +++ pypy/dist/pypy/objspace/std/unicodetype.py Thu Nov 22 15:50:13 2007 @@ -5,6 +5,14 @@ from sys import maxint +def wrapunicode(space, uni): + from pypy.objspace.std.unicodeobject import W_UnicodeObject + from pypy.objspace.std.ropeunicodeobject import wrapunicode + if space.config.objspace.std.withropeunicode: + return wrapunicode(space, uni) + return W_UnicodeObject(uni) + + unicode_capitalize = SMM('capitalize', 1, doc='S.capitalize() -> unicode\n\nReturn a' ' capitalized version of S, i.e. make the first' @@ -218,8 +226,11 @@ def unicode_from_string(space, w_str): # this is a performance and bootstrapping hack - from pypy.objspace.std.unicodeobject import W_UnicodeObject + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import unicode_from_string + return unicode_from_string(space, w_str) encoding = getdefaultencoding(space) + from pypy.objspace.std.unicodeobject import W_UnicodeObject if encoding != 'ascii': return unicode_from_encoded_object(space, w_str, encoding, "strict") s = space.str_w(w_str) @@ -227,13 +238,14 @@ return W_UnicodeObject(s.decode("ascii")) except UnicodeDecodeError: # raising UnicodeDecodeError is messy, "please crash for me" - return unicode_from_object(space, w_str) + return unicode_from_encoded_object(space, w_str, "ascii", "strict") def descr__new__(space, w_unicodetype, w_obj='', w_encoding=None, w_errors=None): # NB. the default value of w_obj is really a *wrapped* empty string: # there is gateway magic at work from pypy.objspace.std.unicodeobject import W_UnicodeObject + from pypy.objspace.std.ropeunicodeobject import W_RopeUnicodeObject w_obj_type = space.type(w_obj) encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) @@ -253,7 +265,12 @@ w_value = unicode_from_object(space, w_obj) else: w_value = unicode_from_encoded_object(space, w_obj, encoding, errors) - # XXX this is not true when there are different unicode implementations + if space.config.objspace.std.withropeunicode: + assert isinstance(w_value, W_RopeUnicodeObject) + w_newobj = space.allocate_instance(W_RopeUnicodeObject, w_unicodetype) + W_RopeUnicodeObject.__init__(w_newobj, w_value._node) + return w_newobj + assert isinstance(w_value, W_UnicodeObject) w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype) W_UnicodeObject.__init__(w_newobj, w_value._value) Modified: pypy/dist/pypy/rlib/runicode.py ============================================================================== --- pypy/dist/pypy/rlib/runicode.py (original) +++ pypy/dist/pypy/rlib/runicode.py Thu Nov 22 15:50:13 2007 @@ -5,14 +5,17 @@ BYTEORDER = sys.byteorder -def raise_unicode_exception(errors, encoding, msg, s, startingpos, endingpos, - decode=True): - if decode: - raise UnicodeDecodeError( - encoding, s[startingpos], startingpos, endingpos, msg) - else: - raise UnicodeEncodeError( - encoding, s[startingpos], startingpos, endingpos, msg) +def raise_unicode_exception_decode(errors, encoding, msg, s, + startingpos, endingpos): + assert isinstance(s, str) + raise UnicodeDecodeError( + encoding, s[startingpos], startingpos, endingpos, msg) + +def raise_unicode_exception_encode(errors, encoding, msg, u, + startingpos, endingpos): + assert isinstance(u, unicode) + raise UnicodeEncodeError( + encoding, u[startingpos], startingpos, endingpos, msg) # ____________________________________________________________ # unicode decoding @@ -37,7 +40,9 @@ ] def str_decode_utf_8(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode if (size == 0): return u'', 0 result = [] @@ -158,27 +163,28 @@ def str_decode_utf_16(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "native") return result, length def str_decode_utf_16_be(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "big") return result, length def str_decode_utf_16_le(s, size, errors, final=True, - errorhandler=raise_unicode_exception): + errorhandler=None): result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final, errorhandler, "little") return result, length def str_decode_utf_16_helper(s, size, errors, final=True, - errorhandler=raise_unicode_exception, + errorhandler=None, byteorder="native"): - + if errorhandler is None: + errorhandler = raise_unicode_exception_decode bo = 0 consumed = 0 @@ -235,7 +241,7 @@ if not final: break r, pos = errorhandler(errors, 'utf-16', "truncated data", - s, pos, len(s), True) + s, pos, len(s)) result.append(r) if len(s) - pos < 2: break @@ -277,7 +283,7 @@ return u"".join(result), pos, bo def str_decode_latin_1(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=None): # latin1 is equivalent to the first 256 ordinals in Unicode. pos = 0 result = [] @@ -288,7 +294,9 @@ def str_decode_ascii(s, size, errors, final=False, - errorhandler=raise_unicode_exception): + errorhandler=None): + if errorhandler is None: + errorhandler = raise_unicode_exception_decode # ASCII is equivalent to the first 128 ordinals in Unicode. result = [] pos = 0 @@ -308,7 +316,7 @@ # unicode encoding -def unicode_encode_utf_8(s, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_utf_8(s, size, errors, errorhandler=None): assert(size >= 0) result = [] i = 0 @@ -353,7 +361,9 @@ def unicode_encode_ucs1_helper(p, size, errors, - errorhandler=raise_unicode_exception, limit=256): + errorhandler=None, limit=256): + if errorhandler is None: + errorhandler = raise_unicode_exception_encode if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" @@ -378,16 +388,16 @@ while collend < len(p) and ord(p[collend]) >= limit: collend += 1 r, pos = errorhandler(errors, encoding, reason, p, - collstart, collend, False) + collstart, collend) result.append(r) return "".join(result) -def unicode_encode_latin_1(p, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res -def unicode_encode_ascii(p, size, errors, errorhandler=raise_unicode_exception): +def unicode_encode_ascii(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res @@ -403,7 +413,7 @@ result.append(lo) def unicode_encode_utf_16_helper(s, size, errors, - errorhandler=raise_unicode_exception, + errorhandler=None, byteorder='little'): result = [] if (byteorder == 'native'): @@ -429,15 +439,15 @@ return "".join(result) def unicode_encode_utf_16(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native") def unicode_encode_utf_16_be(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big") def unicode_encode_utf_16_le(s, size, errors, - errorhandler=raise_unicode_exception): + errorhandler=None): return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little") Modified: pypy/dist/pypy/rlib/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_runicode.py (original) +++ pypy/dist/pypy/rlib/test/test_runicode.py Thu Nov 22 15:50:13 2007 @@ -38,14 +38,13 @@ def checkencodeerror(self, s, encoding, start, stop): called = [False] def errorhandler(errors, enc, msg, t, startingpos, - endingpos, decode): + endingpos): called[0] = True assert errors == "foo!" assert enc == encoding assert t is s assert start == startingpos assert stop == endingpos - assert not decode return "42424242", stop encoder = self.getencoder(encoding) result = encoder(s, len(s), "foo!", errorhandler) @@ -55,7 +54,7 @@ def checkdecodeerror(self, s, encoding, start, stop, addstuff=True): called = [0] def errorhandler(errors, enc, msg, t, startingpos, - endingpos, decode=True): + endingpos): called[0] += 1 if called[0] == 1: assert errors == "foo!" @@ -63,7 +62,6 @@ assert t is s assert start == startingpos assert stop == endingpos - assert decode return u"42424242", stop return "", endingpos decoder = self.getdecoder(encoding) Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Thu Nov 22 15:50:13 2007 @@ -108,6 +108,13 @@ self.ll = LLHelpers self.malloc = mallocstr + def ll_decode_latin1(self, value): + lgt = len(value.chars) + s = mallocunicode(lgt) + for i in range(lgt): + s.chars[i] = cast_primitive(UniChar, value.chars[i]) + return s + class UnicodeRepr(BaseLLStringRepr, AbstractUnicodeRepr): lowleveltype = Ptr(UNICODE) basetype = basestring @@ -131,6 +138,17 @@ result.chars[i] = cast_primitive(Char, c) return result + def ll_encode_latin1(self, s): + length = len(s.chars) + result = mallocstr(length) + for i in range(length): + c = s.chars[i] + if ord(c) > 255: + raise UnicodeEncodeError("character not in latin1 range") + result.chars[i] = cast_primitive(Char, c) + return result + + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Thu Nov 22 15:50:13 2007 @@ -48,6 +48,16 @@ def make_string(self, value): return ootype.make_string(value) + def ll_decode_latin1(self, value): + sb = ootype.new(ootype.UnicodeBuilder) + length = value.ll_strlen() + sb.ll_allocate(length) + for i in range(length): + c = value.ll_stritem_nonneg(i) + sb.ll_append_char(cast_primitive(UniChar, c)) + return sb.ll_build() + + class UnicodeRepr(BaseOOStringRepr, AbstractUnicodeRepr): lowleveltype = ootype.Unicode basetype = basestring @@ -66,6 +76,17 @@ sb.ll_append_char(cast_primitive(Char, c)) return sb.ll_build() + def ll_encode_latin1(self, value): + sb = ootype.new(ootype.StringBuilder) + length = value.ll_strlen() + sb.ll_allocate(length) + for i in range(length): + c = value.ll_stritem_nonneg(i) + if ord(c) > 255: + raise UnicodeEncodeError("%d > 255, not latin-1" % ord(c)) + sb.ll_append_char(cast_primitive(Char, c)) + return sb.ll_build() + class CharRepr(AbstractCharRepr, StringRepr): lowleveltype = Char Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Thu Nov 22 15:50:13 2007 @@ -256,9 +256,17 @@ return hop.gendirectcall(self.ll.ll_str2unicode, v_str) def rtype_method_decode(self, hop): - v_self = hop.inputarg(self, 0) + if not hop.args_s[1].is_constant(): + raise TyperError("encoding must be a constant") + encoding = hop.args_s[1].const + v_self = hop.inputarg(self.repr, 0) hop.exception_is_here() - return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + if encoding == 'ascii': + return hop.gendirectcall(self.ll.ll_str2unicode, v_self) + elif encoding == 'latin-1': + return hop.gendirectcall(self.ll_decode_latin1, v_self) + else: + raise TyperError("encoding %s not implemented" % (encoding, )) def rtype_float(self, hop): hop.has_implicit_exception(ValueError) # record that we know about it @@ -272,9 +280,18 @@ class __extend__(AbstractUnicodeRepr): def rtype_method_encode(self, hop): - v_self = hop.inputarg(self, 0) + if not hop.args_s[1].is_constant(): + raise TyperError("encoding must be constant") + encoding = hop.args_s[1].const + v_self = hop.inputarg(self.repr, 0) hop.exception_is_here() - return hop.gendirectcall(self.ll_str, v_self) + if encoding == "ascii": + return hop.gendirectcall(self.ll_str, v_self) + elif encoding == "latin-1": + return hop.gendirectcall(self.ll_encode_latin1, v_self) + else: + raise TyperError("encoding %s not implemented" % (encoding, )) + class __extend__(pairtype(AbstractStringRepr, Repr)): def rtype_mod((r_str, _), hop): Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Thu Nov 22 15:50:13 2007 @@ -84,26 +84,37 @@ def test_unicode_encode(self): def f(x): y = u'xxx' - return (y + unichr(x)).encode('ascii') + return (y + unichr(x)).encode('ascii') + y.encode('latin-1') assert self.ll_to_string(self.interpret(f, [38])) == f(38) def test_unicode_encode_error(self): - def f(x): - y = u'xxx' - try: - x = (y + unichr(x)).encode('ascii') - return len(x) - except UnicodeEncodeError: - return -1 - - assert self.interpret(f, [38]) == f(38) - assert self.interpret(f, [138]) == f(138) + def f(x, which): + if which: + y = u'xxx' + try: + x = (y + unichr(x)).encode('ascii') + return len(x) + except UnicodeEncodeError: + return -1 + else: + y = u'xxx' + try: + x = (y + unichr(x)).encode('latin-1') + return len(x) + except UnicodeEncodeError: + return -1 + + assert self.interpret(f, [38, True]) == f(38, True) + assert self.interpret(f, [138, True]) == f(138, True) + assert self.interpret(f, [38, False]) == f(38, False) + assert self.interpret(f, [138, False]) == f(138, False) + assert self.interpret(f, [300, False]) == f(300, False) def test_unicode_decode(self): def f(x): y = 'xxx' - return (y + chr(x)).decode('ascii') + return (y + chr(x)).decode('ascii') + chr(x).decode("latin-1") assert self.ll_to_string(self.interpret(f, [38])) == f(38) From cfbolz at codespeak.net Thu Nov 22 15:50:54 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 15:50:54 +0100 (CET) Subject: [pypy-svn] r48925 - pypy/branch/ropes-unicode Message-ID: <20071122145054.4333A81F5@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 15:50:53 2007 New Revision: 48925 Removed: pypy/branch/ropes-unicode/ Log: remove merged branch From cfbolz at codespeak.net Thu Nov 22 15:56:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 15:56:04 +0100 (CET) Subject: [pypy-svn] r48926 - in pypy/branch/remove-extcompiler-rctypes/pypy: annotation bin config doc interpreter module/_file/test module/mmap/test module/readline/test objspace/cpy rlib rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory/gctransform rpython/memory/gctransform/test rpython/rctypes/tool rpython/rctypes/tool/test translator/c translator/c/src translator/c/test translator/goal translator/tool Message-ID: <20071122145604.4DC0E81F7@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 15:56:01 2007 New Revision: 48926 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/bin/compilemodule.py pypy/branch/remove-extcompiler-rctypes/pypy/doc/extcompiler.txt pypy/branch/remove-extcompiler-rctypes/pypy/module/mmap/test/test_mixedmodule.py pypy/branch/remove-extcompiler-rctypes/pypy/module/readline/test/test_compiled.py pypy/branch/remove-extcompiler-rctypes/pypy/module/readline/test/test_mixedmodule.py pypy/branch/remove-extcompiler-rctypes/pypy/objspace/cpy/ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/test_rcpyclass.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/test/test_protect.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rcpy.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rctypes/tool/compilemodule.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rctypes/tool/test/test_compilemodule.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapping.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/goal/targetdemomodule.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/raymond.py Modified: pypy/branch/remove-extcompiler-rctypes/pypy/annotation/builtin.py pypy/branch/remove-extcompiler-rctypes/pypy/bin/checkmodule.py pypy/branch/remove-extcompiler-rctypes/pypy/config/pypyoption.py pypy/branch/remove-extcompiler-rctypes/pypy/doc/cleanup-todo.txt pypy/branch/remove-extcompiler-rctypes/pypy/doc/faq.txt pypy/branch/remove-extcompiler-rctypes/pypy/doc/index.txt pypy/branch/remove-extcompiler-rctypes/pypy/doc/project-ideas.txt pypy/branch/remove-extcompiler-rctypes/pypy/interpreter/mixedmodule.py pypy/branch/remove-extcompiler-rctypes/pypy/module/_file/test/test_file_extra.py pypy/branch/remove-extcompiler-rctypes/pypy/rlib/streamio.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/llinterp.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lloperation.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rclass.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rvirtualizable.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/boehm.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/framework.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/stacklessframework.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/transform.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rmodel.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/mem.h pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/module.h pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py Log: (pedronis, cfbolz): kill: the cpy object space, raymond (again), the extcompiler, support everywhere for the extcompiler Modified: pypy/branch/remove-extcompiler-rctypes/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/annotation/builtin.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/annotation/builtin.py Thu Nov 22 15:56:01 2007 @@ -401,7 +401,7 @@ from pypy.annotation.model import SomePtr from pypy.rpython.lltypesystem import lltype -def malloc(s_T, s_n=None, s_flavor=None, s_extra_args=None, s_zero=None): +def malloc(s_T, s_n=None, s_flavor=None, s_zero=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, pypy.rlib.rarithmetic.base_int)) assert s_T.is_constant() Modified: pypy/branch/remove-extcompiler-rctypes/pypy/bin/checkmodule.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/bin/checkmodule.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/bin/checkmodule.py Thu Nov 22 15:56:01 2007 @@ -4,9 +4,7 @@ Compiles the PyPy extension module from pypy/module// into a fake program which does nothing. Useful for testing whether a -modules compiles without doing a full translation, especially the ones -that cannot be compiled with compilemodule.py because rely on -ootypesystem features (such as _dotnet). Default backend is cli. +modules compiles without doing a full translation. Default backend is cli. WARNING: this is still incomplete: there are chances that the compilation fails with strange errors not due to the module. If a Modified: pypy/branch/remove-extcompiler-rctypes/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/config/pypyoption.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/config/pypyoption.py Thu Nov 22 15:56:01 2007 @@ -41,7 +41,7 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ ChoiceOption("name", "Object Space name", - ["std", "flow", "thunk", "cpy", "dump", "taint"], + ["std", "flow", "thunk", "dump", "taint"], "std", cmdline='--objspace -o'), Modified: pypy/branch/remove-extcompiler-rctypes/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/doc/cleanup-todo.txt (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/doc/cleanup-todo.txt Thu Nov 22 15:56:01 2007 @@ -40,6 +40,4 @@ - signal module - strtod (RPyton level) - - rewrite or kill the extcompiler - - review CPython regression test suite, enable running tests, fix bugs Modified: pypy/branch/remove-extcompiler-rctypes/pypy/doc/faq.txt ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/doc/faq.txt (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/doc/faq.txt Thu Nov 22 15:56:01 2007 @@ -126,16 +126,7 @@ called because they can contain a mixture of compiled and interpreted Python code. At the moment they all need to be translated together with the rest of PyPy. -We have a proof concept of what we call the `extension compiler`_ and -our support for a static variant of the ctypes interface (`rctypes`_) to -help with their development. At the moment both have quite some rough -edges. This kind of module can even be cross-compiled to regular CPython -extension modules, although this doesn't deliver completely satisfying -results so far. This area is going to improve over time. - .. _`mixed modules`: coding-guide.html#mixed-module-mechanism -.. _`extension compiler`: extcompiler.html -.. _`rctypes`: rctypes.html .. _`slower than CPython`: .. _`how fast is pypy`: Modified: pypy/branch/remove-extcompiler-rctypes/pypy/doc/index.txt ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/doc/index.txt (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/doc/index.txt Thu Nov 22 15:56:01 2007 @@ -26,10 +26,6 @@ * `JIT Generation in PyPy`_ * `Sandboxing Python code`_ -`extension compiler`_ describes the (in-progress) tool that can be used -to write modules in PyPy's style and compile them into regular CPython -extension modules. - `JavaScript backend`_ describes how to use the JavaScript backend to create AJAX-based web pages. @@ -263,8 +259,6 @@ `objspace/thunk.py`_ the `thunk object space`_, providing unique object features -`objspace/cpy/`_ an object space supporting the `extension compiler`_ - `objspace/flow/`_ the FlowObjSpace_ implementing `abstract interpretation` `objspace/std/`_ the StdObjSpace_ implementing CPython's objects and types @@ -341,7 +335,6 @@ .. _`GenC backend`: translation.html#genc .. _`LLVM backend`: translation.html#llvm .. _`CLI backend`: cli-backend.html -.. _`extension compiler`: extcompiler.html .. _`py.py`: getting-started.html#main-entry-point .. _`translatorshell.py`: getting-started.html#try-out-the-translator .. _JIT: jit.html Modified: pypy/branch/remove-extcompiler-rctypes/pypy/doc/project-ideas.txt ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/doc/project-ideas.txt (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/doc/project-ideas.txt Thu Nov 22 15:56:01 2007 @@ -215,7 +215,6 @@ .. _`py.execnet`: http://codespeak.net/py/current/doc/execnet.html .. _`Prolog interpreter`: http://codespeak.net/svn/user/cfbolz/hack/prolog/interpreter .. _`JavaScript interpreter`: ../../pypy/lang/js -.. _`extension compiler`: extcompiler.html .. _`object spaces`: objspace.html .. _`code templating solution`: http://codespeak.net/svn/pypy/extradoc/soc-2006/code-templating.txt Modified: pypy/branch/remove-extcompiler-rctypes/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/interpreter/mixedmodule.py Thu Nov 22 15:56:01 2007 @@ -171,8 +171,7 @@ if is_type: return space.gettypefor(value) - W_Object = getattr(space, 'W_Object', ()) # for cpyobjspace - assert isinstance(value, (W_Root, W_Object)), ( + assert isinstance(value, W_Root), ( "interpleveldef %s.%s must return a wrapped object " "(got %r instead)" % (pkgroot, spec, value)) return value @@ -203,33 +202,3 @@ return app.wget(space, attrname) return afileloader -# ____________________________________________________________ -# Helper to test mixed modules on top of CPython - -def testmodule(name, basepath='pypy.module'): - """Helper to test mixed modules on top of CPython, - running with the CPy Object Space. The module should behave - more or less as if it had been compiled, either with the - pypy/bin/compilemodule.py tool, or within pypy-c. - - Try: testmodule('_demo') - """ - import sys, new - from pypy.objspace.cpy.objspace import CPyObjSpace - space = CPyObjSpace() - - fullname = "%s.%s" % (basepath, name) - Module = __import__(fullname, - None, None, ["Module"]).Module - appname = Module.get_applevel_name() - mod = Module(space, space.wrap(appname)) - res = new.module(appname) - sys.modules[appname] = res - moddict = space.unwrap(mod.getdict()) - res.__dict__.update(moddict) - return res - -def compilemodule(name, interactive=False): - "Compile a PyPy module for CPython." - from pypy.rpython.rctypes.tool.compilemodule import compilemodule - return compilemodule(name, interactive=interactive) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/module/_file/test/test_file_extra.py Thu Nov 22 15:56:01 2007 @@ -1,6 +1,7 @@ import os, random, sys import pypy.tool.udir import py +py.test.skip("XXX convert this to a normal applevel test!") from pypy.interpreter.mixedmodule import testmodule udir = pypy.tool.udir.udir.ensure('test_file_extra', dir=1) @@ -206,7 +207,7 @@ # # Basic 'rb' mode -class TestFile(BaseROTests): +class AppTestFile(BaseROTests): expected_filename = str(udir.join('sample')) expected_mode = 'rb' extra_args = () Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rlib/streamio.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rlib/streamio.py Thu Nov 22 15:56:01 2007 @@ -226,7 +226,7 @@ os.lseek(self.fd, offset, whence) def tell(self): - #XXX for running on top of the cpy objspace. later we want r_longlong + #XXX we really want r_longlong return int(os.lseek(self.fd, 0, 1)) def read(self, n): Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/llinterp.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/llinterp.py Thu Nov 22 15:56:01 2007 @@ -680,13 +680,6 @@ ptr = self.heap.coalloc(obj, coallocator, zero=zero) return ptr - # only after gc transform - def op_cpy_malloc(self, obj, cpytype): # xxx - return self.heap.malloc(obj, flavor='cpy', extra_args=(cpytype,)) - - def op_cpy_free(self, obj): - return self.heap.free(obj, flavor='cpy') # xxx ? - def op_malloc_varsize(self, obj, flags, size): flavor = flags['flavor'] zero = flags.get('zero', False) @@ -786,12 +779,6 @@ def op_gc_pop_alive_pyobj(self, pyobj): raise NotImplementedError("gc_pop_alive_pyobj") - def op_gc_protect(self, obj): - raise NotImplementedError("gc_protect") - - def op_gc_unprotect(self, obj): - raise NotImplementedError("gc_unprotect") - def op_gc_reload_possibly_moved(self, v_newaddr, v_ptr): assert v_newaddr.concretetype is llmemory.Address assert isinstance(v_ptr.concretetype, lltype.Ptr) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lloperation.py Thu Nov 22 15:56:01 2007 @@ -365,8 +365,6 @@ 'raw_load': LLOp(sideeffects=False), 'raw_store': LLOp(), 'stack_malloc': LLOp(), # mmh - 'cpy_malloc': LLOp(), # xxx - 'cpy_free': LLOp(), # xxx 'adr_add': LLOp(canfold=True), 'adr_sub': LLOp(canfold=True), 'adr_delta': LLOp(canfold=True), @@ -396,8 +394,6 @@ 'gc_deallocate': LLOp(), 'gc_push_alive_pyobj': LLOp(), 'gc_pop_alive_pyobj': LLOp(), - 'gc_protect': LLOp(), - 'gc_unprotect': LLOp(), 'gc_reload_possibly_moved': LLOp(), 'gc_id': LLOp(canraise=(MemoryError,), sideeffects=False), 'gc_set_max_heap_size': LLOp(), Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py Thu Nov 22 15:56:01 2007 @@ -310,18 +310,8 @@ class GcStruct(RttiStruct): _gckind = 'gc' -class PyStruct(RttiStruct): - _gckind = 'cpy' - - def __init__(self, name, *fields, **kwds): - RttiStruct.__init__(self, name, *fields, **kwds) - if self._first_struct() == (None, None): - raise TypeError("a PyStruct must have another PyStruct or " - "PyObject as first field") - STRUCT_BY_FLAVOR = {'raw': Struct, - 'gc': GcStruct, - 'cpy': PyStruct} + 'gc': GcStruct} class Array(ContainerType): _gckind = 'raw' @@ -495,7 +485,7 @@ def _inline_is_varsize(self, last): return False def _defl(self, parent=None, parentindex=None): - return _pyobjheader(parent, parentindex) + return _pyobject(None) def _allocate(self, initialization, parent=None, parentindex=None): return self._defl(parent=parent, parentindex=parentindex) @@ -519,15 +509,12 @@ class GcForwardReference(ForwardReference): _gckind = 'gc' -class PyForwardReference(ForwardReference): - _gckind = 'cpy' class FuncForwardReference(ForwardReference): _gckind = 'prebuilt' FORWARDREF_BY_FLAVOR = {'raw': ForwardReference, 'gc': GcForwardReference, - 'cpy': PyForwardReference, 'prebuilt': FuncForwardReference} @@ -1351,9 +1338,6 @@ container = parent return container - def _setup_extra_args(self): - pass - def _struct_variety(flds, cache={}): flds = list(flds) flds.sort() @@ -1444,12 +1428,6 @@ assert isinstance(self._TYPE, FixedSizeArray) setattr(self, 'item%d' % index, value) - def _setup_extra_args(self, *args): - fieldname, FIELDTYPE = self._TYPE._first_struct() - if fieldname is not None: - getattr(self, fieldname)._setup_extra_args(*args) - else: - assert not args class _array(_parentable): _kind = "array" @@ -1732,28 +1710,10 @@ def _getid(self): return id(self.value) -class _pyobjheader(_parentable): - def __init__(self, parent=None, parentindex=None): - _parentable.__init__(self, PyObject) - if parent is not None: - self._setparentstructure(parent, parentindex) - # the extra attributes 'ob_type' and 'setup_fnptr' are - # not set by __init__(), but by malloc(extra_args=(...)) - - def _setup_extra_args(self, ob_type, setup_fnptr=None): - assert typeOf(ob_type) == Ptr(PyObject) - self.ob_type = ob_type - self.setup_fnptr = setup_fnptr - - def __repr__(self): - return '<%s>' % (self,) - - def __str__(self): - return "pyobjheader of type %r" % (getattr(self, 'ob_type', '???'),) - -def malloc(T, n=None, flavor='gc', immortal=False, extra_args=(), zero=False): - if zero or immortal or flavor == 'cpy': +def malloc(T, n=None, flavor='gc', immortal=False, zero=False): + assert flavor != 'cpy' + if zero or immortal: initialization = 'example' elif flavor == 'raw': initialization = 'raw' @@ -1770,7 +1730,6 @@ raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): raise TypeError, "gc flavor malloc of a non-GC non-immortal structure" - o._setup_extra_args(*extra_args) solid = immortal or not flavor.startswith('gc') # immortal or non-gc case return _ptr(Ptr(T), o, solid) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rclass.py Thu Nov 22 15:56:01 2007 @@ -72,21 +72,13 @@ NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE)) NONGCOBJECTPTR = Ptr(OBJECT) -# cpy case (XXX try to merge the typeptr with the ob_type) -CPYOBJECT = lltype.PyStruct('cpyobject', ('head', PyObject), - ('typeptr', CLASSTYPE)) -CPYOBJECTPTR = Ptr(CPYOBJECT) - OBJECT_BY_FLAVOR = {'gc': OBJECT, - 'raw': NONGCOBJECT, - 'cpy': CPYOBJECT} + 'raw': NONGCOBJECT} LLFLAVOR = {'gc' : 'gc', 'raw' : 'raw', - 'cpy' : 'cpy', 'stack': 'raw', } -RTTIFLAVORS = ('gc', 'cpy') def cast_vtable_to_typeptr(vtable): while typeOf(vtable).TO != OBJECT_VTABLE: @@ -190,7 +182,7 @@ vtable.subclassrange_max = sys.maxint rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() - if rinstance.gcflavor in RTTIFLAVORS: + if rinstance.gcflavor == 'gc': vtable.rtti = getRuntimeTypeInfo(rinstance.object_type) if rsubcls.classdef is None: name = 'object' @@ -346,11 +338,6 @@ self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) self.rbase.setup() - # - # PyObject wrapper support - if self.has_wrapper and '_wrapper_' not in self.rbase.allinstancefields: - fields['_wrapper_'] = 'wrapper', pyobj_repr - llfields.append(('wrapper', Ptr(PyObject))) MkStruct = lltype.STRUCT_BY_FLAVOR[LLFLAVOR[self.gcflavor]] if adtmeths is None: @@ -370,11 +357,11 @@ allinstancefields.update(fields) self.fields = fields self.allinstancefields = allinstancefields - if self.gcflavor in RTTIFLAVORS: + if self.gcflavor == 'gc': attachRuntimeTypeInfo(self.object_type) def _setup_repr_final(self): - if self.gcflavor in RTTIFLAVORS: + if self.gcflavor == 'gc': if (self.classdef is not None and self.classdef.classdesc.lookup('__del__') is not None): s_func = self.classdef.classdesc.s_read_attribute('__del__') @@ -404,18 +391,7 @@ return cast_pointer(self.lowleveltype, result) def create_instance(self): - if self.gcflavor == 'cpy': - from pypy.rpython import rcpy - extra_args = (rcpy.build_pytypeobject(self),) - else: - extra_args = () - return malloc(self.object_type, flavor=self.gcflavor, - extra_args=extra_args) - - def has_wrapper(self): - return self.classdef is not None and ( - self.rtyper.needs_wrapper(self.classdef.classdesc.pyobj)) - has_wrapper = property(has_wrapper) + return malloc(self.object_type, flavor=self.gcflavor) def get_ll_hash_function(self): if self.classdef is None: @@ -487,35 +463,27 @@ flags=flags) def setfield(self, vinst, attr, vvalue, llops, force_cast=False, - opname='setfield', flags={}): + flags={}): """Write the given attribute (or __class__ for the type) of 'vinst'.""" if attr in self.fields: mangled_name, r = self.fields[attr] cname = inputconst(Void, mangled_name) if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - llops.genop(opname, [vinst, cname, vvalue]) - # XXX this is a temporary hack to clear a dead PyObject + llops.genop('setfield', [vinst, cname, vvalue]) else: if self.classdef is None: raise MissingRTypeAttribute(attr) self.rbase.setfield(vinst, attr, vvalue, llops, force_cast=True, - opname=opname, flags=flags) + flags=flags) - def new_instance(self, llops, classcallhop=None, v_cpytype=None): + def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" flavor = self.gcflavor flags = {'flavor': flavor } ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - if self.classdef is not None: - if flavor == 'cpy': - if v_cpytype is None: - from pypy.rpython import rcpy - cpytype = rcpy.build_pytypeobject(self) - v_cpytype = inputconst(Ptr(PyObject), cpytype) - vlist.append(v_cpytype) vptr = llops.genop('malloc', vlist, resulttype = Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) @@ -689,129 +657,6 @@ v = rpair.rtype_eq(hop) return hop.genop("bool_not", [v], resulttype=Bool) -# -# _________________________ Conversions for CPython _________________________ - -# part I: wrapping, destructor, preserving object identity - -def call_destructor(thing, repr): - ll_call_destructor(thing, repr) - -def ll_call_destructor(thang, repr): - return 42 # will be mapped - -class Entry(ExtRegistryEntry): - _about_ = ll_call_destructor - s_result_annotation = None - - def specialize_call(self, hop): - v_inst, c_spec = hop.inputargs(*hop.args_r) - repr = c_spec.value - if repr.has_wrapper: - null = hop.inputconst(Ptr(PyObject), nullptr(PyObject)) - # XXX this bare_setfield is needed because we cannot do refcount operations - # XXX on a dead object. Actually this is an abuse. Instead, - # XXX we should consider a different operation for 'uninitialized fields' - repr.setfield(v_inst, '_wrapper_', null, hop.llops, - opname='bare_setfield') - hop.genop('gc_unprotect', [v_inst]) - - -def create_pywrapper(thing, repr): - return ll_create_pywrapper(thing, repr) - -def ll_create_pywrapper(thing, repr): - return 42 - -def into_cobject(v_inst, repr, llops): - llops.genop('gc_protect', [v_inst]) - ARG = repr.lowleveltype - reprPBC = llops.rtyper.annotator.bookkeeper.immutablevalue(repr) - fp_dtor = llops.rtyper.annotate_helper_fn(call_destructor, [ARG, reprPBC]) - FUNC = FuncType([ARG, Void], Void) - c_dtor = inputconst(Ptr(FUNC), fp_dtor) - return llops.gencapicall('PyCObject_FromVoidPtr', [v_inst, c_dtor], resulttype=pyobj_repr) - -def outof_cobject(v_obj, repr, llops): - v_inst = llops.gencapicall('PyCObject_AsVoidPtr', [v_obj], resulttype=repr) - llops.genop('gc_protect', [v_inst]) - return v_inst - -class Entry(ExtRegistryEntry): - _about_ = ll_create_pywrapper - s_result_annotation = annmodel.SomePtr(Ptr(PyObject)) - - def specialize_call(self, hop): - v_inst, c_spec = hop.inputargs(*hop.args_r) - repr = c_spec.value - v_res = into_cobject(v_inst, repr, hop.llops) - v_cobj = v_res - c_cls = hop.inputconst(pyobj_repr, repr.classdef.classdesc.pyobj) - c_0 = hop.inputconst(Signed, 0) - v_res = hop.llops.gencapicall('PyType_GenericAlloc', [c_cls, c_0], - resulttype=pyobj_repr) - c_self = hop.inputconst(pyobj_repr, '__self__') - hop.genop('setattr', [v_res, c_self, v_cobj], resulttype=pyobj_repr) - if repr.has_wrapper: - repr.setfield(v_inst, '_wrapper_', v_res, hop.llops) - hop.genop('gc_unprotect', [v_res]) # yes a weak ref - return v_res - - -def fetch_pywrapper(thing, repr): - return ll_fetch_pywrapper(thing, repr) - -def ll_fetch_pywrapper(thing, repr): - return 42 - -class Entry(ExtRegistryEntry): - _about_ = ll_fetch_pywrapper - s_result_annotation = annmodel.SomePtr(Ptr(PyObject)) - - def specialize_call(self, hop): - v_inst, c_spec = hop.inputargs(*hop.args_r) - repr = c_spec.value - if repr.has_wrapper: - return repr.getfield(v_inst, '_wrapper_', hop.llops) - else: - null = hop.inputconst(Ptr(PyObject), nullptr(PyObject)) - return null - - -def ll_wrap_object(obj, repr): - ret = fetch_pywrapper(obj, repr) - if not ret: - ret = create_pywrapper(obj, repr) - return ret - -class __extend__(pairtype(InstanceRepr, PyObjRepr)): - def convert_from_to((r_from, r_to), v, llops): - c_repr = inputconst(Void, r_from) - if r_from.has_wrapper: - return llops.gendirectcall(ll_wrap_object, v, c_repr) - else: - return llops.gendirectcall(create_pywrapper, v, c_repr) - -# part II: unwrapping, creating the instance - -class __extend__(pairtype(PyObjRepr, InstanceRepr)): - def convert_from_to((r_from, r_to), v, llops): - if r_to.has_wrapper: - init, context = llops.rtyper.get_wrapping_hint(r_to.classdef) - if context is init: - # saving an extra __new__ method, we create the instance on __init__ - v_inst = r_to.new_instance(llops) - v_cobj = into_cobject(v_inst, r_to, llops) - c_self = inputconst(pyobj_repr, '__self__') - llops.genop('setattr', [v, c_self, v_cobj], resulttype=pyobj_repr) - r_to.setfield(v_inst, '_wrapper_', v, llops) - llops.genop('gc_unprotect', [v]) - return v_inst - # if we don't have a wrapper field, we just don't support __init__ - c_self = inputconst(pyobj_repr, '__self__') - v = llops.genop('getattr', [v, c_self], resulttype=r_from) - return outof_cobject(v, r_to, llops) - # ____________________________________________________________ # # Low-level implementation of operations on classes and instances Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rvirtualizable.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rvirtualizable.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rvirtualizable.py Thu Nov 22 15:56:01 2007 @@ -82,8 +82,8 @@ else: self.rbase.set_vable(llops, vinst, force_cast=True) - def new_instance(self, llops, classcallhop=None, v_cpytype=None): - vptr = InstanceRepr.new_instance(self, llops, classcallhop, v_cpytype) + def new_instance(self, llops, classcallhop=None): + vptr = InstanceRepr.new_instance(self, llops, classcallhop) self.set_vable(llops, vptr) return vptr @@ -139,7 +139,7 @@ return InstanceRepr.getfield(self, vinst, attr, llops, force_cast) def setfield(self, vinst, attr, vvalue, llops, force_cast=False, - opname='setfield', flags={}): + flags={}): """Write the given attribute (or __class__ for the type) of 'vinst'.""" if (attr in self.my_redirected_fields and not flags.get('access_directly')): @@ -149,5 +149,4 @@ ll_setter = self.get_setter(mangled_name) llops.gendirectcall(ll_setter, vinst, vvalue) else: - InstanceRepr.setfield(self, vinst, attr, vvalue, llops, force_cast, - opname) + InstanceRepr.setfield(self, vinst, attr, vvalue, llops, force_cast) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/test_lltype.py Thu Nov 22 15:56:01 2007 @@ -709,23 +709,6 @@ def test_pyobject(): p = pyobjectptr({42: 84}) assert typeOf(p) == Ptr(PyObject) - S1 = PyStruct('S1', ('head', PyObject), ('x', Signed)) - py.test.raises(TypeError, PyStruct, 'S2', ('y', Signed)) - py.test.raises(TypeError, PyStruct, 'S2', ('x', Struct('X'))) - py.test.raises(TypeError, PyStruct, 'S2', ('x', GcStruct('X'))) - - inttype = pyobjectptr(int) - s1 = malloc(S1, flavor='cpy', extra_args=(inttype,)) - s1.x = 17 - assert typeOf(s1.head) == Ptr(PyObject) - h1 = cast_pointer(Ptr(PyObject), s1) - assert s1.head == h1 - s2 = cast_pointer(Ptr(S1), h1) - assert s2 == s1 - - del s1, s2 - s3 = cast_pointer(Ptr(S1), h1) - assert s3.x == 17 def test_name_clash(): import re Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/boehm.py Thu Nov 22 15:56:01 2007 @@ -56,14 +56,6 @@ def pop_alive_nopyobj(self, var, llops): pass - def gct_gc_protect(self, hop): - """ for boehm it is enough to do nothing""" - pass - - def gct_gc_unprotect(self, hop): - """ for boehm it is enough to do nothing""" - pass - def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): # XXX same behavior for zero=True: in theory that's wrong if TYPE._is_atomic(): Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/framework.py Thu Nov 22 15:56:01 2007 @@ -1,6 +1,6 @@ -from pypy.rpython.memory.gctransform.transform import GCTransformer, var_ispyobj +from pypy.rpython.memory.gctransform.transform import GCTransformer from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ - get_rtti, ll_call_destructor, type_contains_pyobjs + get_rtti, ll_call_destructor, type_contains_pyobjs, var_ispyobj from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import rmodel from pypy.rpython.memory import gctypelayout Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py Thu Nov 22 15:56:01 2007 @@ -68,8 +68,6 @@ gcheader.refcount = refcount def ll_no_pointer_dealloc(adr): llop.gc_free(lltype.Void, adr) - def ll_no_pointer_cpydealloc(adr): - llop.cpy_free(lltype.Void, adr) mh = mallocHelpers() mh.allocate = llmemory.raw_malloc @@ -103,8 +101,6 @@ ll_decref_simple, [llmemory.Address], lltype.Void) self.no_pointer_dealloc_ptr = self.inittime_helper( ll_no_pointer_dealloc, [llmemory.Address], lltype.Void) - self.no_pointer_cpydealloc_ptr = self.inittime_helper( - ll_no_pointer_cpydealloc, [llmemory.Address], lltype.Void) self.malloc_fixedsize_ptr = self.inittime_helper( ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( @@ -152,14 +148,6 @@ lltype.typeOf(dealloc_fptr), dealloc_fptr) llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr]) - def gct_gc_protect(self, hop): - """ protect this object from gc (make it immortal) """ - self.push_alive(hop.spaceop.args[0], hop.llops) - - def gct_gc_unprotect(self, hop): - """ get this object back into gc control """ - self.pop_alive(hop.spaceop.args[0], hop.llops) - def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size], resulttype=llmemory.Address) @@ -189,7 +177,7 @@ def consider_constant(self, TYPE, value): if value is not lltype.top_container(value): - return + return if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): p = value._as_ptr() if not self.gcheaderbuilder.get_header(p): @@ -210,11 +198,8 @@ DESTR_ARG = None if destrptr is None and not find_gc_ptrs_in_type(TYPE): - #print repr(TYPE)[:80], 'is dealloc easy' - if TYPE._gckind == 'cpy': - p = self.no_pointer_cpydealloc_ptr.value - else: - p = self.no_pointer_dealloc_ptr.value + assert TYPE._gckind != 'cpy' + p = self.no_pointer_dealloc_ptr.value self.static_deallocator_funcptrs[TYPE] = p return p Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/stacklessframework.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/stacklessframework.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/stacklessframework.py Thu Nov 22 15:56:01 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.memory.gctransform.transform import var_ispyobj +from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer from pypy.rpython.lltypesystem import lltype, llmemory Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/transform.py Thu Nov 22 15:56:01 2007 @@ -13,6 +13,7 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc +from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList from pypy.rpython.rbuiltin import gen_cast @@ -21,16 +22,6 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.simplify import join_blocks, cleanup_graph -def var_ispyobj(var): - if hasattr(var, 'concretetype'): - if isinstance(var.concretetype, lltype.Ptr): - return var.concretetype.TO._gckind == 'cpy' - else: - return False - else: - # assume PyObjPtr - return True - PyObjPtr = lltype.Ptr(lltype.PyObject) class GcHighLevelOp(object): @@ -521,17 +512,11 @@ hop.genop("raw_memclear", [v_raw, c_size]) return v_raw - def gct_fv_cpy_malloc(self, hop, flags, TYPE, c_size): # xxx - op = hop.spaceop - args = op.args[:] - del args[1] - assert not flags.get('zero') - return hop.genop('cpy_malloc', args, resulttype=op.result.concretetype) - def gct_malloc_varsize(self, hop): flags = hop.spaceop.args[1].value flavor = flags['flavor'] + assert flavor != 'cpy', "cannot malloc CPython objects directly" meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None) assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) return self.varsize_malloc_helper(hop, flags, meth, []) @@ -606,10 +591,9 @@ op = hop.spaceop flavor = op.args[1].value v = op.args[0] + assert flavor != 'cpy', "cannot free CPython objects directly" if flavor == 'raw': v = hop.genop("cast_ptr_to_adr", [v], resulttype=llmemory.Address) hop.genop('raw_free', [v]) - elif flavor == 'cpy': - hop.genop('cpy_free', [v]) # xxx else: assert False, "%s has no support for free with flavor %r" % (self, flavor) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py Thu Nov 22 15:56:01 2007 @@ -322,19 +322,16 @@ BUILTIN_TYPER[object.__init__] = rtype_object__init__ # annotation of low-level types -def rtype_malloc(hop, i_flavor=None, i_extra_args=None, i_zero=None): +def rtype_malloc(hop, i_flavor=None, i_zero=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' - v_flavor, v_extra_args, v_zero = parse_kwds(hop, (i_flavor, lltype.Void), - (i_extra_args, None), - (i_zero, None)) + v_flavor, v_zero = parse_kwds(hop, (i_flavor, lltype.Void), (i_zero, None)) flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value if i_zero is not None: - assert i_extra_args is None flags['zero'] = v_zero.value vlist.append(hop.inputconst(lltype.Void, flags)) @@ -342,15 +339,6 @@ vlist.append(hop.inputarg(lltype.Signed, arg=1)) opname += '_varsize' - if v_extra_args is not None: - # items of the v_extra_args tuple become additional args to the op - from pypy.rpython.rtuple import AbstractTupleRepr - r_tup = hop.args_r[i_extra_args] - assert isinstance(r_tup, AbstractTupleRepr) - for n, r in enumerate(r_tup.items_r): - v = r_tup.getitem(hop.llops, v_extra_args, n) - vlist.append(v) - hop.has_implicit_exception(MemoryError) # record that we know about it hop.exception_is_here() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rmodel.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rmodel.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rmodel.py Thu Nov 22 15:56:01 2007 @@ -171,8 +171,8 @@ if (isinstance(T, lltype.Ptr) and isinstance(T.TO, (lltype.Struct, lltype.Array, - lltype.ForwardReference)) and - T.TO._gckind != 'cpy'): + lltype.ForwardReference))): + assert T.TO._gckind != 'cpy' return DummyValueBuilder(rtyper, T.TO) else: return None @@ -439,9 +439,6 @@ PyObjPtr = Ptr(PyObject) def getgcflavor(classdef): - for parentdef in classdef.getmro(): - if hasattr(parentdef, '_cpy_exported_type_'): - return 'cpy' classdesc = classdef.classdesc alloc_flavor = classdesc.read_attribute('_alloc_flavor_', Constant('gc')).value Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py Thu Nov 22 15:56:01 2007 @@ -573,17 +573,6 @@ esize = self.expr(op.args[0]) return "OP_STACK_MALLOC(%s, %s, void *);" % (esize, eresult) - def OP_CPY_MALLOC(self, op): - TYPE = self.lltypemap(op.result).TO - typename = self.db.gettype(TYPE) - erestype = cdecl(typename, '*') - eresult = self.expr(op.result) - cpytype = self.expr(op.args[1]) - return "OP_CPY_MALLOC(%s, %s, %s);" % (cpytype, eresult, erestype) - - def OP_CPY_FREE(self, op): - return "OP_CPY_FREE(%s);" % (self.expr(op.args[0]),) - def OP_DIRECT_FIELDPTR(self, op): return self.OP_GETFIELD(op, ampersand='&') Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Thu Nov 22 15:56:01 2007 @@ -1,7 +1,7 @@ import autopath import py import sys -from pypy.translator.c.node import PyObjectNode, PyObjHeadNode, FuncNode +from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.gensupp import uniquemodulename, NameManager @@ -38,7 +38,7 @@ self.libraries = libraries self.exports = {} - def build_database(self, exports=[], pyobj_options=None): + def build_database(self, pyobj_options=None): translator = self.translator gcpolicyclass = self.get_gcpolicyclass() @@ -76,20 +76,6 @@ pf = self.getentrypointptr() pfname = db.get(pf) self.exports[self.entrypoint.func_name] = pf - for obj in exports: - if type(obj) is tuple: - objname, obj = obj - elif hasattr(obj, '__name__'): - objname = obj.__name__ - else: - objname = None - po = self.getentrypointptr(obj) - poname = db.get(po) - objname = objname or poname - if objname in self.exports: - raise NameError, 'duplicate name in export: %s is %s and %s' % ( - objname, db.get(self.exports[objname]), poname) - self.exports[objname] = po db.complete() # add library dependencies @@ -777,21 +763,12 @@ print >> f print >> f, 'static globalobjectdef_t globalobjectdefs[] = {' for node in database.containerlist: - if isinstance(node, (PyObjectNode, PyObjHeadNode)): + if isinstance(node, PyObjectNode): for target in node.where_to_copy_me: print >> f, '\t{%s, "%s"},' % (target, node.exported_name) print >> f, '\t{ NULL, NULL }\t/* Sentinel */' print >> f, '};' print >> f - print >> f, 'static cpyobjheaddef_t cpyobjheaddefs[] = {' - for node in database.containerlist: - if isinstance(node, PyObjHeadNode): - print >> f, '\t{"%s", %s, %s},' % (node.exported_name, - node.ptrname, - node.get_setupfn_name()) - print >> f, '\t{ NULL, NULL, NULL }\t/* Sentinel */' - print >> f, '};' - print >> f print >> f, '/***********************************************************/' print >> f, '/*** Table of functions ***/' print >> f Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py Thu Nov 22 15:56:01 2007 @@ -1,9 +1,9 @@ from __future__ import generators from pypy.rpython.lltypesystem.lltype import \ Struct, Array, FixedSizeArray, FuncType, PyObjectType, typeOf, \ - GcStruct, GcArray, RttiStruct, PyStruct, ContainerType, \ + GcStruct, GcArray, RttiStruct, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ - RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray, _pyobjheader + RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray from pypy.rpython.lltypesystem import llmemory from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator @@ -546,13 +546,9 @@ data = data[0:1] for name, value in data: - if isinstance(value, _pyobjheader): # hack - node = self.db.getcontainernode(value) - lines = [node.pyobj_initexpr()] - else: - c_expr = defnode.access_expr(self.name, name) - lines = generic_initializationexpr(self.db, value, c_expr, - decoration + name) + c_expr = defnode.access_expr(self.name, name) + lines = generic_initializationexpr(self.db, value, c_expr, + decoration + name) for line in lines: yield '\t' + line if not lines[0].startswith('/*'): @@ -896,47 +892,6 @@ def implementation(self): return [] - -class PyObjHeadNode(ContainerNode): - nodekind = 'pyobj' - - def __init__(self, db, T, obj): - ContainerNode.__init__(self, db, T, obj) - self.where_to_copy_me = [] - self.exported_name = db.namespace.uniquename('cpyobj') - - def basename(self): - raise Exception("PyObjHead should always have a parent") - - def enum_dependencies(self): - yield self.obj.ob_type - if self.obj.setup_fnptr: - yield self.obj.setup_fnptr - - def get_setupfn_name(self): - if self.obj.setup_fnptr: - return self.db.get(self.obj.setup_fnptr) - else: - return 'NULL' - - def pyobj_initexpr(self): - parent, parentindex = parentlink(self.obj) - typenode = self.db.getcontainernode(self.obj.ob_type._obj) - typenode.where_to_copy_me.append('(PyObject **) & %s.ob_type' % ( - self.name,)) - if typeOf(parent)._hints.get('inline_head'): - return 'PyObject_HEAD_INIT(NULL)' - else: - return '{ PyObject_HEAD_INIT(NULL) },' - - -def objectnode_factory(db, T, obj): - if isinstance(obj, _pyobjheader): - return PyObjHeadNode(db, T, obj) - else: - return PyObjectNode(db, T, obj) - - def weakrefnode_factory(db, T, obj): assert isinstance(obj, llmemory._wref) ptarget = obj._dereference() @@ -948,12 +903,11 @@ ContainerNodeFactory = { Struct: StructNode, GcStruct: StructNode, - PyStruct: StructNode, Array: ArrayNode, GcArray: ArrayNode, FixedSizeArray: FixedSizeArrayNode, FuncType: FuncNode, OpaqueType: opaquenode_factory, - PyObjectType: objectnode_factory, + PyObjectType: PyObjectNode, llmemory._WeakRefType: weakrefnode_factory, } Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py Thu Nov 22 15:56:01 2007 @@ -8,7 +8,6 @@ from pypy.translator.gensupp import builtin_base, builtin_type_base from pypy.translator.c.support import log from pypy.translator.c.wrapper import gen_wrapper, new_method_graph -from pypy.translator.tool.raymond import should_expose from pypy.rlib.rarithmetic import r_int, r_uint from pypy.rpython.lltypesystem.lltype import pyobjectptr, LowLevelType @@ -53,16 +52,7 @@ stackentry = obj self.debugstack = (self.debugstack, stackentry) try: - try: - self.translator.rtyper # check for presence - entry = extregistry.lookup(obj) - getter = entry.get_ll_pyobjectptr - except (KeyError, AttributeError): - # common case: 'p' is a _pyobject - p = pyobjectptr(obj) - else: - # 'p' should be a PyStruct pointer, i.e. a _pyobjheader - p = getter(self.translator.rtyper) + p = pyobjectptr(obj) node = self.db.getcontainernode(p._obj) finally: self.debugstack, x = self.debugstack @@ -584,9 +574,6 @@ if isinstance(value, FunctionType): func = value fname = '%s.%s' % (cls.__name__, func.__name__) - if not should_expose(func): - log.REMARK('method %s hidden from wrapper' % fname) - continue if func.__name__ == '__init__': init_seen = True # there is the problem with exposed classes inheriting from Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/mem.h (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/mem.h Thu Nov 22 15:56:01 2007 @@ -118,17 +118,6 @@ #endif /* USING_BOEHM_GC */ /************************************************************/ -/* rcpy support */ - -#define OP_CPY_MALLOC(cpytype, r, restype) { \ - /* XXX add tp_itemsize later */ \ - r = (restype)((PyTypeObject *)cpytype)->tp_alloc( \ - (PyTypeObject *)cpytype, 0); \ - if (!r) RPyConvertExceptionFromCPython(); \ - } -#define OP_CPY_FREE(x) ((PyObject *)x)->ob_type->tp_free((PyObject *)x) - -/************************************************************/ /* weakref support */ #define OP_CAST_PTR_TO_WEAKREFPTR(x, r) r = x Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/module.h ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/module.h (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/module.h Thu Nov 22 15:56:01 2007 @@ -38,11 +38,9 @@ } \ if (setup_globalfunctions(globalfunctiondefs, #modname) < 0) \ return; \ - if (setup_exportglobalobjects(cpyobjheaddefs) < 0) \ - return; \ if (setup_initcode(frozen_initcode, FROZEN_INITCODE_SIZE) < 0) \ return; \ - if (setup_globalobjects(globalobjectdefs, cpyobjheaddefs) < 0) \ + if (setup_globalobjects(globalobjectdefs) < 0) \ return; /*** table of global objects ***/ @@ -55,12 +53,6 @@ } globalobjectdef_t; typedef struct { - char* name; - PyObject* cpyobj; - void (*setupfn)(PyObject *); -} cpyobjheaddef_t; - -typedef struct { PyObject** p; char* gfunc_name; PyMethodDef ml; @@ -77,33 +69,10 @@ #ifndef PYPY_NOT_MAIN_FILE -static int setup_exportglobalobjects(cpyobjheaddef_t* cpyheadtable) -{ - PyObject* obj; - cpyobjheaddef_t* cpydef; - - /* Store the object given by their heads into the module's dict. - Warning: these object heads might still be invalid, e.g. - typically their ob_type needs patching! - But PyDict_SetItemString() doesn't inspect them... - */ - for (cpydef = cpyheadtable; cpydef->name != NULL; cpydef++) { - obj = cpydef->cpyobj; - if (obj->ob_type == NULL) - obj->ob_type = &PyType_Type; - if (PyDict_SetItemString(this_module_globals, - cpydef->name, obj) < 0) - return -1; - } - return 0; -} - -static int setup_globalobjects(globalobjectdef_t* globtable, - cpyobjheaddef_t* cpyheadtable) +static int setup_globalobjects(globalobjectdef_t* globtable) { PyObject* obj; globalobjectdef_t* def; - cpyobjheaddef_t* cpydef; /* Patch all locations that need to contain a specific PyObject*. This must go after the previous loop, otherwise @@ -120,30 +89,6 @@ Py_INCREF(obj); *def->p = obj; /* store the object ref in the global var */ } - /* All objects should be valid at this point. Loop again and - make sure all types are ready. - */ - for (cpydef = cpyheadtable; cpydef->name != NULL; cpydef++) { - obj = cpydef->cpyobj; - if (PyType_Check(obj)) { - /* XXX hmmm */ - obj->ob_type = NULL; - if (PyType_Ready((PyTypeObject*) obj) < 0) - return -1; - } - } - /* call the user-defined setups *after* all types are ready - * in case of dependencies */ - for (cpydef = cpyheadtable; cpydef->name != NULL; cpydef++) { - obj = cpydef->cpyobj; - if (cpydef->setupfn) { - cpydef->setupfn(obj); - if (RPyExceptionOccurred()) { - RPyConvertExceptionToCPython(); - return -1; - } - } - } return 0; } Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py Thu Nov 22 15:56:01 2007 @@ -220,51 +220,6 @@ res = fn() assert res == 8765 - def test_pystruct(self): - PS1 = PyStruct('PS1', ('head', PyObject), ('x', Signed), - hints = {'inline_head': True}) - class mytype(object): - pass - mytype_ptr = pyobjectptr(mytype) - def llf(): - p = malloc(PS1, flavor='cpy', extra_args=(mytype_ptr,)) - return cast_pointer(Ptr(PyObject), p) - - fn = self.getcompiled(llf) - res = fn() - assert type(res).__name__.endswith('mytype') - - def test_pystruct_prebuilt(self): - PS1 = PyStruct('PS1', ('head', PyObject), ('x', Signed), - hints = {'inline_head': True}) - class mytype(object): - pass - - def llsetup(phead): - "Called when the CPython ext module is imported." - p = cast_pointer(Ptr(PS1), phead) - p.x = 27 - - mytype_ptr = pyobjectptr(mytype) - p = malloc(PS1, flavor='cpy', extra_args=(mytype_ptr,)) - p.x = -5 # overridden by llsetup() - - def llf(): - return p.x - - def process(t): - rtyper = t.buildrtyper() - rtyper.specialize() - llsetup_ptr = rtyper.annotate_helper_fn(llsetup, [Ptr(PyObject)]) - phead = cast_pointer(Ptr(PyObject), p) - phead._obj.setup_fnptr = llsetup_ptr - - self.process = process - fn = self.getcompiled(llf) - res = fn() - assert res == 27 - del self.process - def test_union(self): U = Struct('U', ('s', Signed), ('c', Char), hints={'union': True}) From cfbolz at codespeak.net Thu Nov 22 16:34:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 16:34:10 +0100 (CET) Subject: [pypy-svn] r48927 - pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform Message-ID: <20071122153410.A1A678202@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 16:34:09 2007 New Revision: 48927 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py Log: (pedronis, cfbolz): PyObjects don't need an RPython deallocator Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py Thu Nov 22 16:34:09 2007 @@ -189,6 +189,8 @@ return self.static_deallocator_funcptrs[TYPE] #print_call_chain(self) + if TYPE._gckind == 'cpy': + return # you don't really have an RPython deallocator for PyObjects rtti = get_rtti(TYPE) if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): destrptr = rtti._obj.destructor_funcptr @@ -198,7 +200,6 @@ DESTR_ARG = None if destrptr is None and not find_gc_ptrs_in_type(TYPE): - assert TYPE._gckind != 'cpy' p = self.no_pointer_dealloc_ptr.value self.static_deallocator_funcptrs[TYPE] = p return p From cfbolz at codespeak.net Thu Nov 22 16:37:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 16:37:00 +0100 (CET) Subject: [pypy-svn] r48928 - pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform Message-ID: <20071122153700.A90C78203@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 16:37:00 2007 New Revision: 48928 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py Log: (pedronis, cfbolz): there shouldn't be a dynamic deallocator pointer for CPy Objects at all. Assert that. Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/gctransform/refcounting.py Thu Nov 22 16:37:00 2007 @@ -254,6 +254,7 @@ return fptr def dynamic_deallocation_funcptr_for_type(self, TYPE): + assert TYPE._gckind != 'cpy' if TYPE in self.dynamic_deallocator_funcptrs: return self.dynamic_deallocator_funcptrs[TYPE] #print_call_chain(self) From cfbolz at codespeak.net Thu Nov 22 16:43:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 16:43:24 +0100 (CET) Subject: [pypy-svn] r48929 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test Message-ID: <20071122154324.813F68208@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 16:43:24 2007 New Revision: 48929 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Log: (pedronis, cfbolz): this test works nowadays. we should look at our skips. Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Thu Nov 22 16:43:24 2007 @@ -359,7 +359,6 @@ print f(C) def test_oswrite(): - py.test.skip("Example of single character string degenerating to SomeObject") def f(): import os os.write(1,"o") From arigo at codespeak.net Thu Nov 22 17:14:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Nov 2007 17:14:06 +0100 (CET) Subject: [pypy-svn] r48930 - pypy/dist/pypy/rpython/lltypesystem/module Message-ID: <20071122161406.7B311820D@code0.codespeak.net> Author: arigo Date: Thu Nov 22 17:14:06 2007 New Revision: 48930 Modified: pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py Log: This is sandboxsafe. I think it incidentally fixes the ValueError crashes of threaded pypy-c's, but I'm working on a proper fix. Modified: pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py Thu Nov 22 17:14:06 2007 @@ -9,7 +9,8 @@ sandboxsafe=True) math_modf = rffi.llexternal('modf', [rffi.DOUBLE, rffi.DOUBLEP], rffi.DOUBLE, sandboxsafe=True) -math_ldexp = rffi.llexternal('ldexp', [rffi.DOUBLE, rffi.INT], rffi.DOUBLE) +math_ldexp = rffi.llexternal('ldexp', [rffi.DOUBLE, rffi.INT], rffi.DOUBLE, + sandboxsafe=True) unary_math_functions = [ 'acos', 'asin', 'atan', 'ceil', 'cos', 'cosh', 'exp', 'fabs', From pypy-svn at codespeak.net Thu Nov 22 17:14:47 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Thu, 22 Nov 2007 17:14:47 +0100 (CET) Subject: [pypy-svn] October 78% OFF Message-ID: <20071122-11452.7005.qmail@ip-189.21.84-113.globalwave.com.br> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Thu Nov 22 17:48:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Nov 2007 17:48:29 +0100 (CET) Subject: [pypy-svn] r48931 - pypy/dist/pypy/module/thread Message-ID: <20071122164829.4111B820F@code0.codespeak.net> Author: arigo Date: Thu Nov 22 17:48:27 2007 New Revision: 48931 Modified: pypy/dist/pypy/module/thread/gil.py Log: Fragile code to preserve the errno... Modified: pypy/dist/pypy/module/thread/gil.py ============================================================================== --- pypy/dist/pypy/module/thread/gil.py (original) +++ pypy/dist/pypy/module/thread/gil.py Thu Nov 22 17:48:27 2007 @@ -12,6 +12,7 @@ from pypy.interpreter.miscutils import Action from pypy.module.thread.threadlocals import OSThreadLocals from pypy.rlib.objectmodel import invoke_around_extcall +from pypy.rlib.rposix import get_errno, set_errno class GILThreadLocals(OSThreadLocals): """A version of OSThreadLocals that enforces a GIL.""" @@ -83,10 +84,16 @@ pass spacestate = SpaceState() +# Fragile code below. We have to preserve the C-level errno manually... + def before_external_call(): # this function must not raise, in such a way that the exception # transformer knows that it cannot raise! + e = get_errno() spacestate.GIL.release() + set_errno(e) def after_external_call(): + e = get_errno() spacestate.GIL.acquire(True) + set_errno(e) From tverwaes at codespeak.net Thu Nov 22 17:52:10 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 17:52:10 +0100 (CET) Subject: [pypy-svn] r48932 - in pypy/dist/pypy/lang/smalltalk: . test Message-ID: <20071122165210.4E5338210@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 17:52:09 2007 New Revision: 48932 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/primitives.py pypy/dist/pypy/lang/smalltalk/squeakimage.py pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py pypy/dist/pypy/lang/smalltalk/test/test_model.py pypy/dist/pypy/lang/smalltalk/test/test_primitives.py pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Log: another step closer to compiling methods. a bit of a "cleanup" on the at0 and atput0 for compiledmethod, cleaning up issues with methodheaderss Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Nov 22 17:52:09 2007 @@ -4,6 +4,7 @@ from pypy.lang.smalltalk import constants from pypy.tool.pairtype import extendabletype from pypy.rlib.objectmodel import instantiate +from pypy.lang.smalltalk.tool.bitmanipulation import splitter class W_Object(object): __slots__ = () # no RPython-level instance variables allowed in W_Object @@ -299,14 +300,9 @@ The trailer has two variant formats. In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer). In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables. The number of bytes used for this purpose is the value of the last byte in the method. """ - def __init__(self, literalsize, bytes="", argsize=0, - tempsize=0, primitive=0, w_compiledin=None): - self.literals = [None] * literalsize - self.w_compiledin = w_compiledin - self.bytes = bytes - self.argsize = argsize - self.tempsize = tempsize - self.primitive = primitive + def __init__(self, bytecount=0, header=0): + self.setheader(header) + self.bytes = "\x00"*bytecount def compiledin(self): if self.w_compiledin is None: @@ -323,10 +319,11 @@ return w_CompiledMethod def getliteral(self, index): - return self.literals[index + constants.LITERAL_START] + return self.literals[index] #+ constants.LITERAL_START] def getliteralsymbol(self, index): w_literal = self.getliteral(index) + print "literals: %r" % self.literals assert isinstance(w_literal, W_BytesObject) return w_literal.as_string() # XXX performance issue here @@ -354,36 +351,73 @@ self.primitive is not None) def size(self): - return self.literalsize() + len(self.bytes) + return self.getliteralsize() + len(self.bytes) - def literalsize(self): - return len(self.literals) * constants.BYTES_PER_WORD + def getliteralsize(self): + return self.literalsize * constants.BYTES_PER_WORD def primsize(self): - return self.size() + self.headersize() + return self.size() def headersize(self): return constants.BYTES_PER_WORD + def getheader(self): + return self.header + + def setheader(self, header): + #(index 0) 9 bits: main part of primitive number (#primitive) + #(index 9) 8 bits: number of literals (#numLiterals) + #(index 17) 1 bit: whether a large frame size is needed (#frameSize) + #(index 18) 6 bits: number of temporary variables (#numTemps) + #(index 24) 4 bits: number of arguments to the method (#numArgs) + #(index 28) 1 bit: high-bit of primitive number (#primitive) + #(index 29) 1 bit: flag bit, ignored by the VM (#flag) + primitive, literalsize, islarge, tempsize, numargs, highbit = ( + splitter[9,8,1,6,4,1](header)) + primitive = primitive + (highbit << 10) ##XXX todo, check this + self.literalsize = literalsize + self.literals = [w_nil] * self.literalsize + self.header = header + self.argsize = numargs + self.tempsize = tempsize + self.primitive = primitive + self.w_compiledin = None + + def literalat0(self, index0): + if index0 == 0: + from pypy.lang.smalltalk import utility + return utility.wrap_int(self.getheader()) + else: + return self.literals[index0-1] + + def literalatput0(self, index0, w_value): + if index0 == 0: + from pypy.lang.smalltalk import utility + print "Going to save as header: %r" % w_value + header = utility.unwrap_int(w_value) + self.setheader(header) + else: + self.literals[index0-1] = w_value + def at0(self, index0): # XXX from pypy.lang.smalltalk import utility - index0 = index0 - self.literalsize() - if index0 < 0: - # XXX Do something useful with this.... we are not a block - # of memory as smalltalk expects but wrapped in py-os - raise NotImplementedError() - return utility.wrap_int(ord(self.bytes[index0])) + print "TRYING TO GET: %d %d" % (self.getliteralsize(), index0) + if index0 < self.getliteralsize(): + self.literalat0(index0) + else: + index0 = index0 - self.getliteralsize() + return utility.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility - index0 = index0 - self.literalsize() - if index0 < 0: - # XXX Do something useful with this.... we are not a block - # of memory as smalltalk expects but wrapped in py-os - raise NotImplementedError() + print "TRYING TO SET: %d %d %r" % (self.getliteralsize(), index0, w_value) + if index0 < self.getliteralsize(): + self.literalatput0(index0, w_value) else: # XXX use to-be-written unwrap_char + index0 = index0 - self.getliteralsize() self.setchar(index0, chr(utility.unwrap_int(w_value))) def setchar(self, index0, character): Modified: pypy/dist/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/primitives.py Thu Nov 22 17:52:09 2007 @@ -346,16 +346,15 @@ def func(interp, w_rcvr, n0): if not isinstance(w_rcvr, model.W_CompiledMethod): raise PrimitiveFailedError() - assert_bounds(n0, 0, len(w_rcvr.literals)) - return w_rcvr.literals[n0] + return w_rcvr.literalat0(n0) @expose_primitive(OBJECT_AT_PUT, unwrap_spec=[object, index1_0, object]) -def func(interp, w_rcvr, n0, w_val): +def func(interp, w_rcvr, n0, w_value): if not isinstance(w_rcvr, model.W_CompiledMethod): raise PrimitiveFailedError() - assert_bounds(n0, 0, len(w_rcvr.literals)) - w_rcvr.literals[n0] = w_val - return w_val + #assert_bounds(n0, 0, len(w_rcvr.literals)) + w_rcvr.literalatput0(n0, w_value) + return w_value @expose_primitive(NEW, unwrap_spec=[object]) def func(interp, w_cls): @@ -419,18 +418,10 @@ # it returns the "next" instance after w_obj. raise PrimitiveNotYetWrittenError() - at expose_primitive(NEW_METHOD, unwrap_spec=[object, int, object]) -def func(interp, w_class, bytecount, w_header): - # XXX untested - header = utility.unwrap_int(w_header) - literalcount = ((header >> 10) & 255) + 1 - w_method = w_class.as_class_get_shadow().new(literalcount) - # XXX not sure this is correct - assert isinstance(w_method, model.W_CompiledMethod) - w_method.literals[constants.METHOD_HEADER_INDEX] = w_header - for i0 in range(1, literalcount): - w_method.literals[i0] = objtable.w_nil - w_method.bytes = "\x00" * bytecount + at expose_primitive(NEW_METHOD, unwrap_spec=[object, int, int]) +def func(interp, w_class, bytecount, header): + # We ignore w_class because W_CompiledMethod is special + w_method = model.W_CompiledMethod(bytecount, header) return w_method # ___________________________________________________________________________ Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Thu Nov 22 17:52:09 2007 @@ -357,29 +357,13 @@ def fillin_compiledmethod(self, w_compiledmethod): header = self.chunk.data[0] - #---!!!---- 1 tagged pointer! - #(index 0) 9 bits: main part of primitive number (#primitive) - #(index 9) 8 bits: number of literals (#numLiterals) - #(index 17) 1 bit: whether a large frame size is needed (#frameSize) - #(index 18) 6 bits: number of temporary variables (#numTemps) - #(index 24) 4 bits: number of arguments to the method (#numArgs) - #(index 28) 1 bit: high-bit of primitive number (#primitive) - #(index 29) 1 bit: flag bit, ignored by the VM (#flag) - _, primitive, literalsize, islarge, tempsize, numargs, highbit = ( - splitter[1,9,8,1,6,4,1](header)) - primitive = primitive + (highbit << 10) ##XXX todo, check this - literals = [self.decode_pointer(pointer).w_object - for pointer in self.chunk.data[:literalsize+1]] - bbytes = self.get_bytes()[(literalsize + 1)*4:] + w_compiledmethod.setheader(header>>1) # We untag before giving header + for i in range(1,w_compiledmethod.literalsize+1): + w_compiledmethod.literalatput0(i, self.decode_pointer(self.chunk.data[i]).w_object) + bbytes = self.get_bytes()[(w_compiledmethod.literalsize + 1)*4:] # XXX assert mirrorcache.get_or_build(self.g_class.w_object) is # ct.m_CompiledMethod - w_compiledmethod.__init__( - literalsize = literalsize, - bytes = ''.join(bbytes), - argsize = numargs, - tempsize = tempsize, - primitive = primitive) - w_compiledmethod.literals = literals + w_compiledmethod.bytes = ''.join(bbytes) class ImageChunk(object): def __init__(self, size, format, classid, hash12): Modified: pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_interpreter.py Thu Nov 22 17:52:09 2007 @@ -30,8 +30,10 @@ # Install faked compiled methods that just invoke the primitive: for (w_class, primnum, argsize, methname) in methods: s_class = w_class.as_class_get_shadow() - prim_meth = model.W_CompiledMethod( - 0, "", argsize=argsize, primitive=primnum) + prim_meth = model.W_CompiledMethod(0) + prim_meth.primitive = primnum + prim_meth.w_compiledin = w_class + prim_meth.argsize = argsize s_class.installmethod(methname, prim_meth) try: @@ -62,20 +64,24 @@ res.storevarpointer(i, fakeliteral(lit[i])) return res return lit - return [fakesymbol("methodheader")]+[fakeliteral(lit) for lit in literals] + return [fakeliteral(lit) for lit in literals] def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) - w_method = model.W_CompiledMethod(0, bytes=bytes, - argsize=2, tempsize=1) + w_method = model.W_CompiledMethod(len(bytes)) + w_method.bytes = bytes + w_method.argsize=2 + w_method.tempsize=1 w_frame = w_method.create_frame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() interp.w_active_context = w_frame return interp def test_create_frame(): - w_method = model.W_CompiledMethod(0, bytes="hello", - argsize=2, tempsize=1) + w_method = model.W_CompiledMethod(len("hello")) + w_method.bytes="hello" + w_method.argsize=2 + w_method.tempsize=1 w_frame = w_method.create_frame("receiver", ["foo", "bar"]) assert w_frame.w_receiver == "receiver" assert w_frame.gettemp(0) == "foo" @@ -380,8 +386,9 @@ (returnNil, interpreter.Interpreter.NIL), (returnTopFromMethod, interpreter.Interpreter.ONE) ]: shadow = w_class.as_class_get_shadow() - shadow.installmethod("foo", - model.W_CompiledMethod(0, pushConstantOneBytecode + bytecode)) + w_method = model.W_CompiledMethod(2) + w_method.bytes = pushConstantOneBytecode + bytecode + shadow.installmethod("foo", w_method) interp = new_interpreter(bytecodes) interp.w_active_context.w_method().literals = fakeliterals("foo") interp.w_active_context.push(w_object) @@ -405,7 +412,10 @@ def test_fibWithArgument(): bytecode = ''.join(map(chr, [ 16, 119, 178, 154, 118, 164, 11, 112, 16, 118, 177, 224, 112, 16, 119, 177, 224, 176, 124 ])) shadow = mockclass(0).as_class_get_shadow() - method = model.W_CompiledMethod(1, bytecode, 1) + method = model.W_CompiledMethod(len(bytecode)) + method.literalsize = 1 + method.bytes = bytecode + method.argsize = 1 method.literals = fakeliterals("fib:") shadow.installmethod("fib:", method) w_object = shadow.new() @@ -529,7 +539,11 @@ def test_callPrimitiveAndPush_fallback(): interp = new_interpreter(bytecodePrimAdd) shadow = mockclass(0).as_class_get_shadow() - shadow.installmethod("+", model.W_CompiledMethod(1, "", 1)) + w_method = model.W_CompiledMethod(0) + w_method.argsize = 1 + w_method.literalsize = 1 + shadow.installmethod("+", w_method) + w_object = shadow.new() interp.w_active_context.push(w_object) interp.w_active_context.push(interp.ONE) @@ -567,12 +581,14 @@ # first call method installed in w_class bytecodes = singleExtendedSendBytecode + chr(0) # which does a call to its super - meth1 = model.W_CompiledMethod(0, pushReceiverBytecode + bytecode) + meth1 = model.W_CompiledMethod(2) + meth1.bytes = pushReceiverBytecode + bytecode w_class.as_class_get_shadow().installmethod("foo", meth1) # and that one again to its super - meth2 = model.W_CompiledMethod(0, pushReceiverBytecode + bytecode) + meth2 = model.W_CompiledMethod(2) + meth2.bytes = pushReceiverBytecode + bytecode w_super.as_class_get_shadow().installmethod("foo", meth2) - meth3 = model.W_CompiledMethod(0, "") + meth3 = model.W_CompiledMethod(0) w_supersuper.as_class_get_shadow().installmethod("foo", meth3) meth1.literals = fakeliterals("foo") meth2.literals = fakeliterals("foo") @@ -726,7 +742,7 @@ def test_bc_primBytecodeAt_with_instvars(): # ^ self at: 1 - py.test.skip("Broken, we are fixing it.") + py.test.skip("Broken, fix me") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -742,7 +758,7 @@ def test_bc_primBytecodeAtPut_with_instvars(): # ^ self at: 1 put: #b - py.test.skip("Broken, we are fixing it.") + py.test.skip("Broken, fix me") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -763,7 +779,8 @@ # ^ self objectAt: 2. yields the first literal (22) # ^ self objectAt: 2 put: 3. changes the first literal to 3 # ^ self objectAt: 2. yields the new first literal (3) - prim_meth = model.W_CompiledMethod(0, "") + py.test.skip("Broken, fix me") + prim_meth = model.W_CompiledMethod(0) prim_meth.literals = fakeliterals(22) mhs = fakesymbol("methodheader") oal = fakeliterals("objectAt:") Modified: pypy/dist/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_model.py Thu Nov 22 17:52:09 2007 @@ -66,14 +66,14 @@ w_super = mockclass(0) w_class = mockclass(0, w_superclass=w_super) supershadow = w_super.as_class_get_shadow() - supershadow.installmethod("foo", model.W_CompiledMethod(0, "")) + supershadow.installmethod("foo", model.W_CompiledMethod(0)) classshadow = w_class.as_class_get_shadow() assert classshadow.lookup("foo").w_compiledin is w_super def test_compiledmethod_setchar(): - w_method = model.W_CompiledMethod(0, "abc") + w_method = model.W_CompiledMethod(3) w_method.setchar(0, "c") - assert w_method.bytes == "cbc" + assert w_method.bytes == "c\x00\x00" def test_hashes(): w_five = model.W_SmallInteger(5) Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Nov 22 17:52:09 2007 @@ -207,10 +207,11 @@ assert prim(primitives.SIZE, [w_obj]).value == 5 def test_size_of_compiled_method(): - varsize = 3 - text = "abc" - w_cm = model.W_CompiledMethod(varsize, text, 1, 1) - assert prim(primitives.SIZE, [w_cm]).value == (varsize+1)*constants.BYTES_PER_WORD + len(text) + literalsize = 3 + bytecount = 3 + w_cm = model.W_CompiledMethod(bytecount) + w_cm.literalsize = literalsize + assert prim(primitives.SIZE, [w_cm]).value == (literalsize)*constants.BYTES_PER_WORD + bytecount def test_string_at(): assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") @@ -422,9 +423,9 @@ shadow = mockclass(0).as_class_get_shadow() w_method = prim(primitives.NEW_METHOD, [classtable.w_CompiledMethod, len(bytecode), 1025]) - assert w_method.literals[0].value == 1025 - assert len(w_method.literals) == 2 - assert w_method.literals[1] is objtable.w_nil + assert w_method.literalat0(0).value == 1025 + assert w_method.literalsize == 2 + assert w_method.literalat0(1) is objtable.w_nil assert w_method.bytes == "\x00" * len(bytecode) Modified: pypy/dist/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_shadow.py Thu Nov 22 17:52:09 2007 @@ -64,8 +64,8 @@ yield basicshape, "CompiledMeth", 0xE02, shadow.COMPILED_METHOD, True, 0 def test_methoddict(): - methods = {'foo': model.W_CompiledMethod(0, ""), - 'bar': model.W_CompiledMethod(1, "")} + methods = {'foo': model.W_CompiledMethod(0), + 'bar': model.W_CompiledMethod(0)} w_class = build_smalltalk_class("Demo", 0x90, methods=methods) classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods From tverwaes at codespeak.net Thu Nov 22 18:06:49 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 18:06:49 +0100 (CET) Subject: [pypy-svn] r48933 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071122170649.54B798215@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 18:06:49 2007 New Revision: 48933 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: Size has to include header apparently. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Nov 22 18:06:49 2007 @@ -319,11 +319,11 @@ return w_CompiledMethod def getliteral(self, index): + # We changed this part return self.literals[index] #+ constants.LITERAL_START] def getliteralsymbol(self, index): w_literal = self.getliteral(index) - print "literals: %r" % self.literals assert isinstance(w_literal, W_BytesObject) return w_literal.as_string() # XXX performance issue here @@ -351,7 +351,7 @@ self.primitive is not None) def size(self): - return self.getliteralsize() + len(self.bytes) + return self.getliteralsize() + len(self.bytes) + self.headersize() def getliteralsize(self): return self.literalsize * constants.BYTES_PER_WORD From tverwaes at codespeak.net Thu Nov 22 18:07:45 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 18:07:45 +0100 (CET) Subject: [pypy-svn] r48934 - pypy/dist/pypy/lang/smalltalk/test Message-ID: <20071122170745.BB5AD8216@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 18:07:45 2007 New Revision: 48934 Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Log: and this is the test for the previous commit Modified: pypy/dist/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/dist/pypy/lang/smalltalk/test/test_primitives.py Thu Nov 22 18:07:45 2007 @@ -211,7 +211,7 @@ bytecount = 3 w_cm = model.W_CompiledMethod(bytecount) w_cm.literalsize = literalsize - assert prim(primitives.SIZE, [w_cm]).value == (literalsize)*constants.BYTES_PER_WORD + bytecount + assert prim(primitives.SIZE, [w_cm]).value == (literalsize+1)*constants.BYTES_PER_WORD + bytecount def test_string_at(): assert prim(primitives.STRING_AT, ["foobar", 4]) == wrap("b") From cfbolz at codespeak.net Thu Nov 22 18:16:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 18:16:50 +0100 (CET) Subject: [pypy-svn] r48935 - pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/llvm/test Message-ID: <20071122171650.A4A23821B@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 18:16:50 2007 New Revision: 48935 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/llvm/test/test_llvmjit.py Log: skip this, rctypes is gone Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/llvm/test/test_llvmjit.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/llvm/test/test_llvmjit.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/llvm/test/test_llvmjit.py Thu Nov 22 18:16:50 2007 @@ -4,6 +4,8 @@ from pypy.jit.codegen.llvm import llvmjit from pypy.jit.codegen.llvm.compatibility import define, globalprefix, icmp, i1, i32 +py.test.skip("doesn't work right now since it is using rctypes") + try: from pypy.jit.codegen.llvm import llvmjit except OSError: From tverwaes at codespeak.net Thu Nov 22 18:17:03 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 18:17:03 +0100 (CET) Subject: [pypy-svn] r48936 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071122171703.D7426821C@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 18:17:03 2007 New Revision: 48936 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: primsize = size by default Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Nov 22 18:17:03 2007 @@ -15,6 +15,9 @@ def varsize(self): return self.size() + def primsize(self): + return self.size() + def getclass(self): raise NotImplementedError @@ -221,9 +224,6 @@ def size(self): return len(self.bytes) - def primsize(self): - return self.size() - def __str__(self): return self.as_string() @@ -356,9 +356,6 @@ def getliteralsize(self): return self.literalsize * constants.BYTES_PER_WORD - def primsize(self): - return self.size() - def headersize(self): return constants.BYTES_PER_WORD From rxe at codespeak.net Thu Nov 22 18:18:35 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 22 Nov 2007 18:18:35 +0100 (CET) Subject: [pypy-svn] r48937 - pypy/dist/pypy/translator/llvm Message-ID: <20071122171835.C3E148222@code0.codespeak.net> Author: rxe Date: Thu Nov 22 18:18:33 2007 New Revision: 48937 Modified: pypy/dist/pypy/translator/llvm/structnode.py Log: bug fix found by turning on optimizations for tests Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Thu Nov 22 18:18:33 2007 @@ -124,10 +124,21 @@ return ref def get_childref(self, index): + if isinstance(index, str): + pos = 0 + found = False + for name in self.structtype._names_without_voids(): + if name == index: + found = True + break + pos += 1 + else: + pos = index + return "getelementptr(%s* %s, i32 0, i32 %s)" % ( self.get_typerepr(), self.get_ref(), - index) + pos) def setup(self): if isinstance(self.value, lltype._subarray): From cfbolz at codespeak.net Thu Nov 22 18:20:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 18:20:32 +0100 (CET) Subject: [pypy-svn] r48938 - pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/test Message-ID: <20071122172032.E79FA8225@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 18:20:32 2007 New Revision: 48938 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/test/test_ll_cairo.py Log: (pedronis, cfbolz): skip too, rctypes is gone Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/test/test_ll_cairo.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/test/test_ll_cairo.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/test/test_ll_cairo.py Thu Nov 22 18:20:32 2007 @@ -9,6 +9,9 @@ import py.test import distutils.errors + +py.test.skip("this is using rctypes, so not working currently") + try: from pypy.rlib.rcairo._cairo import CAIRO_FORMAT_ARGB32 from pypy.rlib.rcairo.ll_cairo import * From cfbolz at codespeak.net Thu Nov 22 18:21:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 18:21:00 +0100 (CET) Subject: [pypy-svn] r48939 - in pypy/branch/remove-extcompiler-rctypes/pypy: rlib/rctypes rpython/rctypes Message-ID: <20071122172100.D65F78226@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 18:21:00 2007 New Revision: 48939 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rctypes/ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rctypes/ Log: (cfbolz, pedronis): kill rctypes From rxe at codespeak.net Thu Nov 22 18:28:09 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 22 Nov 2007 18:28:09 +0100 (CET) Subject: [pypy-svn] r48940 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071122172809.9EB2A822F@code0.codespeak.net> Author: rxe Date: Thu Nov 22 18:28:09 2007 New Revision: 48940 Added: pypy/dist/pypy/translator/llvm/test/test_standalone.py Modified: pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: add tests for standalone eventually Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Nov 22 18:28:09 2007 @@ -25,6 +25,7 @@ bk = translator.annotator.bookkeeper graph_entrypoint = bk.getdesc(entrypoint).getuniquegraph() s_result = translator.annotator.binding(graph_entrypoint.getreturnvar()) + get_argc = rffi.llexternal('_pypy_getargc', [], rffi.INT) get_argv = rffi.llexternal('_pypy_getargv', [], rffi.CCHARPP) @@ -36,11 +37,10 @@ entrypoint._annenforceargs_ = [s_list_of_strings] mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) + graph = mixlevelannotator.getgraph(new_entrypoint, [], s_result) mixlevelannotator.finish() - - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(translator) + mixlevelannotator.backend_optimize() return new_entrypoint Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Thu Nov 22 18:28:09 2007 @@ -100,6 +100,7 @@ isolate=True, # pass to compile + exe_name=None, optimize=True, extra_opts={}): @@ -120,7 +121,7 @@ options.update(extra_opts) config.set(**options) - driver = TranslationDriver(config=config) + driver = TranslationDriver(config=config, exe_name=exe_name) driver.setup(function, annotation) driver.annotate() if conftest.option.view: @@ -158,8 +159,12 @@ return compile_test(function, annotation, isolate_hint=isolate_hint, **kwds)[1] def compile_standalone(function, **kwds): + exe_name = kwds.pop('exe_name', function.func_name) optimize = kwds.pop('optimize', optimize_tests) - drvier = genllvm_compile(function, None, optimize=optimize, **kwds) + driver = genllvm_compile(function, None, + optimize=optimize, + exe_name=exe_name, + **kwds) #______________________________________________________________________________ # XXX Work in progress, this was mostly copied from cli Added: pypy/dist/pypy/translator/llvm/test/test_standalone.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_standalone.py Thu Nov 22 18:28:09 2007 @@ -0,0 +1,39 @@ +import py +from pypy.translator.llvm.test.runtest import compile_standalone + +def cmdexec(exe_name, *args): + from pypy.tool.udir import udir + exe_name = udir.join(exe_name) + args = ' '.join(args) + return py.process.cmdexec('%s %s' % (exe_name, args)) + +def test_print(): + def entry_point(argv): + print "argument count:", len(argv) + print "arguments:", argv + print "argument lengths:", + print [len(s) for s in argv] + return 0 + + exe_name = 'test_print' + compile_standalone(entry_point, exe_name=exe_name) + data = cmdexec(exe_name, 'abc', 'def') + assert data.startswith('argument count: 3') + data = cmdexec(exe_name) + assert data.startswith('argument count: 1') + +def test_hello_world(): + import os + def entry_point(argv): + os.write(1, "hello world\n") + argv = argv[1:] + os.write(1, "argument count: " + str(len(argv)) + "\n") + for s in argv: + os.write(1, " '" + str(s) + "'\n") + return 0 + + exe_name = 'test_hello_world' + compile_standalone(entry_point, exe_name=exe_name) + data = cmdexec(exe_name, 'hi', 'there') + assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''') + From tverwaes at codespeak.net Thu Nov 22 18:35:39 2007 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 22 Nov 2007 18:35:39 +0100 (CET) Subject: [pypy-svn] r48941 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071122173539.E45938219@code0.codespeak.net> Author: tverwaes Date: Thu Nov 22 18:35:39 2007 New Revision: 48941 Modified: pypy/dist/pypy/lang/smalltalk/model.py pypy/dist/pypy/lang/smalltalk/shadow.py Log: Contexts can not be created with basicNew -> raise Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Thu Nov 22 18:35:39 2007 @@ -391,16 +391,13 @@ def literalatput0(self, index0, w_value): if index0 == 0: from pypy.lang.smalltalk import utility - print "Going to save as header: %r" % w_value header = utility.unwrap_int(w_value) self.setheader(header) else: self.literals[index0-1] = w_value def at0(self, index0): - # XXX from pypy.lang.smalltalk import utility - print "TRYING TO GET: %d %d" % (self.getliteralsize(), index0) if index0 < self.getliteralsize(): self.literalat0(index0) else: @@ -409,7 +406,6 @@ def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility - print "TRYING TO SET: %d %d %r" % (self.getliteralsize(), index0, w_value) if index0 < self.getliteralsize(): self.literalatput0(index0, w_value) else: Modified: pypy/dist/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/shadow.py (original) +++ pypy/dist/pypy/lang/smalltalk/shadow.py Thu Nov 22 18:35:39 2007 @@ -120,8 +120,8 @@ if w_cls == classtable.w_BlockContext: return model.W_BlockContext(None, None, 0, 0) elif w_cls == classtable.w_MethodContext: - assert 0, "this seems nonsense" - return model.W_MethodContext(None, None, [], extrasize) + # From slang: Contexts must only be created with newForMethod: + raise error.PrimitiveFailedError if self.instance_kind == POINTERS: return model.W_PointersObject(w_cls, self.instance_size+extrasize) From cfbolz at codespeak.net Thu Nov 22 18:46:19 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 18:46:19 +0100 (CET) Subject: [pypy-svn] r48942 - in pypy/branch/remove-extcompiler-rctypes/pypy: jit/codegen/i386 jit/codegen/ppc module/signal rpython rpython/lltypesystem rpython/test translator/c Message-ID: <20071122174619.16DBD81C0@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 18:46:18 2007 New Revision: 48942 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/module/signal/ctypes_signal.py Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/ppc/codebuf.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/llmemory.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py pypy/branch/remove-extcompiler-rctypes/pypy/rpython/test/test_rbuiltin.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py Log: (pedronis, cfbolz): deal with the removal of rctypes Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py Thu Nov 22 18:46:18 2007 @@ -170,12 +170,11 @@ import py py.test.skip("must run in the main thread") try: - from ctypes import cast, c_void_p - from pypy.rpython.rctypes.tool import util + from ctypes import cast, c_void_p, util path = util.find_library('gc') if path is None: raise ImportError("Boehm (libgc) not found") - boehmlib = util.load_library(path) + boehmlib = ctypes.cdll.LoadLibrary(path) except ImportError, e: import py py.test.skip(str(e)) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/ppc/codebuf.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/ppc/codebuf.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/ppc/codebuf.py Thu Nov 22 18:46:18 2007 @@ -1,48 +1,15 @@ -import pypy.rpython.rctypes.implementation # register rctypes types -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc from ctypes import POINTER, cast, c_void_p, c_int, c_char - -class CConfig: - _includes_ = ("sys/types.h", "sys/mman.h") - size_t = ctypes_platform.SimpleType("size_t", c_int) - off_t = ctypes_platform.SimpleType("off_t", c_int) - - MAP_PRIVATE = ctypes_platform.DefinedConstantInteger("MAP_PRIVATE") - MAP_ANON = ctypes_platform.DefinedConstantInteger("MAP_ANON") - MAP_ANONYMOUS = ctypes_platform.DefinedConstantInteger("MAP_ANONYMOUS") - PROT_READ = ctypes_platform.DefinedConstantInteger("PROT_READ") - PROT_WRITE = ctypes_platform.DefinedConstantInteger("PROT_WRITE") - PROT_EXEC = ctypes_platform.DefinedConstantInteger("PROT_EXEC") - -globals().update(ctypes_platform.configure(CConfig)) -if MAP_ANONYMOUS is None: - MAP_ANONYMOUS = MAP_ANON - assert MAP_ANONYMOUS is not None -del MAP_ANON - -# ____________________________________________________________ - -PTR = POINTER(c_char) # cannot use c_void_p as return value of functions :-( - -mmap_ = libc.mmap -mmap_.argtypes = [PTR, size_t, c_int, c_int, c_int, off_t] -mmap_.restype = PTR -mmap_.includes = ("sys/mman.h",) -munmap_ = libc.munmap -munmap_.argtypes = [PTR, size_t] -munmap_.restype = c_int -munmap_.includes = ("sys/mman.h",) +from pypy.jit.codegen.i386 import codebuf_posix def alloc(map_size): - flags = MAP_PRIVATE | MAP_ANONYMOUS - prot = PROT_EXEC | PROT_READ | PROT_WRITE - res = mmap_(PTR(), map_size, prot, flags, -1, 0) + flags = codebuf_posix.MAP_PRIVATE | codebuf_posix.MAP_ANONYMOUS + prot = codebuf_posix.PROT_EXEC | codebuf_posix.PROT_READ | codebuf_posix.PROT_WRITE + res = codebuf_posix.mmap_(PTR(), map_size, prot, flags, -1, 0) if not res: raise MemoryError return res -free = munmap_ +free = codebuf_posix.munmap_ class CodeBlockOverflow(Exception): pass Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/llmemory.py Thu Nov 22 18:46:18 2007 @@ -84,7 +84,6 @@ repeat = self.repeat if repeat == 0: return - from pypy.rpython.rctypes.rmodel import reccopy if isinstance(self.TYPE, lltype.ContainerType): PTR = lltype.Ptr(self.TYPE) else: @@ -92,7 +91,7 @@ while True: src = cast_adr_to_ptr(srcadr, PTR) dst = cast_adr_to_ptr(dstadr, PTR) - reccopy(src, dst) + _reccopy(src, dst) repeat -= 1 if repeat <= 0: break @@ -133,8 +132,7 @@ PTR = lltype.Ptr(self.TYPE) src = cast_adr_to_ptr(srcadr, PTR) dst = cast_adr_to_ptr(dstadr, PTR) - from pypy.rpython.rctypes.rmodel import reccopy - reccopy(src, dst) + _reccopy(src, dst) class CompositeOffset(AddressOffset): @@ -257,8 +255,7 @@ return cast_ptr_to_adr(headerptr) def raw_memcopy(self, srcadr, dstadr): - from pypy.rpython.rctypes.rmodel import reccopy - reccopy(srcadr.ptr, dstadr.ptr) + _reccopy(srcadr.ptr, dstadr.ptr) class GCHeaderAntiOffset(AddressOffset): def __init__(self, gcheaderbuilder): @@ -638,3 +635,34 @@ else: # regular case return lltype.cast_pointer(EXPECTED_TYPE, ptr) + + +def _reccopy(source, dest): + # copy recursively a structure or array onto another. + T = lltype.typeOf(source).TO + assert T == lltype.typeOf(dest).TO + if isinstance(T, (lltype.Array, lltype.FixedSizeArray)): + assert source._obj.getlength() == dest._obj.getlength() + ITEMTYPE = T.OF + for i in range(source._obj.getlength()): + if isinstance(ITEMTYPE, lltype.ContainerType): + subsrc = source._obj.getitem(i)._as_ptr() + subdst = dest._obj.getitem(i)._as_ptr() + _reccopy(subsrc, subdst) + else: + # this is a hack XXX de-hack this + llvalue = source._obj.getitem(i, uninitialized_ok=True) + dest._obj.setitem(i, llvalue) + elif isinstance(T, lltype.Struct): + for name in T._names: + FIELDTYPE = getattr(T, name) + if isinstance(FIELDTYPE, lltype.ContainerType): + subsrc = source._obj._getattr(name)._as_ptr() + subdst = dest._obj._getattr(name)._as_ptr() + _reccopy(subsrc, subdst) + else: + # this is a hack XXX de-hack this + llvalue = source._obj._getattr(name, uninitialized_ok=True) + setattr(dest._obj, name, llvalue) + else: + raise TypeError(T) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py Thu Nov 22 18:46:18 2007 @@ -413,9 +413,9 @@ def __init__(self, args, result): for arg in args: assert isinstance(arg, LowLevelType) - # -- disabled the following check for the benefits of rctypes -- - #if isinstance(arg, ContainerType): - # raise TypeError, "function arguments can only be primitives or pointers" + # -- disable the following check for the benefits of rffi -- + if isinstance(arg, ContainerType): + raise TypeError, "function arguments can only be primitives or pointers" self.ARGS = tuple(args) assert isinstance(result, LowLevelType) if isinstance(result, ContainerType): Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py Thu Nov 22 18:46:18 2007 @@ -116,8 +116,7 @@ def rtype_call_args(self, hop): # calling a built-in function with keyword arguments: - # mostly for rpython.objectmodel.hint() and for constructing - # rctypes structures + # mostly for rpython.objectmodel.hint() hop, kwds_i = call_args_expand(hop) bltintyper = self.findbltintyper(hop.rtyper) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/rpython/test/test_rbuiltin.py Thu Nov 22 18:46:18 2007 @@ -7,7 +7,7 @@ from pypy.rlib.rarithmetic import r_uint, intmask from pypy.annotation.builtin import * from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rpython.rctypes.rcarithmetic import CShort +from pypy.rpython.lltypesystem.rffi import SHORT from pypy.rpython import extfunc import py @@ -473,7 +473,7 @@ res = self.interpret(llf, [ord('x')], policy=LowLevelAnnotatorPolicy()) assert res == u'x' def llf(v): - return lltype.cast_primitive(CShort, v) + return lltype.cast_primitive(SHORT, v) res = self.interpret(llf, [123], policy=LowLevelAnnotatorPolicy()) assert res == 123 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py Thu Nov 22 18:46:18 2007 @@ -388,6 +388,7 @@ continue # skip 'void' argument args.append(self.expr(v)) # special case for rctypes: by-value container args: + # XXX is this still needed now that rctypes is gone if isinstance(ARGTYPE, ContainerType): args[-1] = '*%s' % (args[-1],) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py Thu Nov 22 18:46:18 2007 @@ -114,7 +114,7 @@ return self.prefix + name def verbatim_field_name(self, name): - if name.startswith('c_'): # produced in this way by rctypes + if name.startswith('c_'): # produced in this way by rffi return name[2:] else: # field names have to start with 'c_' or be meant for names that From xoraxax at codespeak.net Thu Nov 22 20:48:59 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 22 Nov 2007 20:48:59 +0100 (CET) Subject: [pypy-svn] r48943 - in pypy/dist/pypy: rlib translator/c/src Message-ID: <20071122194859.8EB958230@code0.codespeak.net> Author: xoraxax Date: Thu Nov 22 20:48:57 2007 New Revision: 48943 Modified: pypy/dist/pypy/rlib/rstack.py pypy/dist/pypy/translator/c/src/g_include.h pypy/dist/pypy/translator/c/src/stack.h Log: Fix the stack_too_big test failure in llvm by shuffling around includes. Modified: pypy/dist/pypy/rlib/rstack.py ============================================================================== --- pypy/dist/pypy/rlib/rstack.py (original) +++ pypy/dist/pypy/rlib/rstack.py Thu Nov 22 20:48:57 2007 @@ -33,7 +33,7 @@ else: return len(inspect.stack()) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, _callable=lambda: 0) +stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, includes=['src/stack.h'], _callable=lambda: 0) def stack_check(): if stack_too_big(): Modified: pypy/dist/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/dist/pypy/translator/c/src/g_include.h (original) +++ pypy/dist/pypy/translator/c/src/g_include.h Thu Nov 22 20:48:57 2007 @@ -60,10 +60,6 @@ #endif #endif -#ifndef AVR -#include "src/stack.h" -#endif - #ifdef PYPY_STANDALONE # include "src/main.h" #endif Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Thu Nov 22 20:48:57 2007 @@ -2,6 +2,7 @@ /************************************************************/ /*** C header subsection: stack operations ***/ + #ifndef MAX_STACK_SIZE # define MAX_STACK_SIZE (1 << 19) #endif @@ -10,6 +11,7 @@ int LL_stack_too_big(void); #ifndef PYPY_NOT_MAIN_FILE +#include "thread.h" #ifndef PYPY_NOINLINE # if defined __GNUC__ From cfbolz at codespeak.net Thu Nov 22 21:12:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 21:12:09 +0100 (CET) Subject: [pypy-svn] r48944 - in pypy/dist/pypy: objspace/std objspace/std/test rlib rlib/test Message-ID: <20071122201209.07E158232@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 21:12:06 2007 New Revision: 48944 Added: pypy/dist/pypy/rlib/rope.py - copied unchanged from r48924, pypy/dist/pypy/objspace/std/rope.py pypy/dist/pypy/rlib/test/test_rope.py - copied, changed from r48924, pypy/dist/pypy/objspace/std/test/test_rope.py Removed: pypy/dist/pypy/objspace/std/rope.py pypy/dist/pypy/objspace/std/test/test_rope.py Modified: pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/ropeunicodeobject.py Log: move rope.py to rlib, where it really belongs Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Thu Nov 22 21:12:06 2007 @@ -10,7 +10,7 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.objspace.std.stringtype import wrapchar -from pypy.objspace.std import rope +from pypy.rlib import rope from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY,\ _upper, _lower Modified: pypy/dist/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeunicodeobject.py Thu Nov 22 21:12:06 2007 @@ -4,7 +4,7 @@ from pypy.objspace.std.unicodeobject import _normalize_index from pypy.objspace.std.ropeobject import W_RopeObject from pypy.objspace.std.noneobject import W_NoneObject -from pypy.objspace.std import rope +from pypy.rlib import rope from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std import slicetype from pypy.objspace.std.tupleobject import W_TupleObject From cfbolz at codespeak.net Thu Nov 22 21:27:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 21:27:52 +0100 (CET) Subject: [pypy-svn] r48945 - pypy/branch/new-genc-tests-wrapper Message-ID: <20071122202752.BE56F8166@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 21:27:52 2007 New Revision: 48945 Removed: pypy/branch/new-genc-tests-wrapper/ Log: kill aborted branch From cfbolz at codespeak.net Thu Nov 22 21:29:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 21:29:33 +0100 (CET) Subject: [pypy-svn] r48946 - pypy/dist/pypy/objspace/std Message-ID: <20071122202933.DEC6C816C@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 21:29:31 2007 New Revision: 48946 Modified: pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/unicodeobject.py Log: fix some tests Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Thu Nov 22 21:29:31 2007 @@ -830,7 +830,10 @@ def str_decode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ getdefaultencoding - from pypy.objspace.std.ropeunicodeobject import decode_string + if space.config.objspace.std.withropeunicode: + from pypy.objspace.std.ropeunicodeobject import decode_string + else: + from pypy.objspace.std.unicodetype import decode_object as decode_string encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None: encoding = getdefaultencoding(space) Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Thu Nov 22 21:29:31 2007 @@ -771,7 +771,7 @@ return space.newtuple([w_unistr, W_UnicodeObject.EMPTY, W_UnicodeObject.EMPTY]) else: - assert pos > 0 + assert pos >= 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, space.wrap(unistr[pos+len(unisub):])]) @@ -786,7 +786,7 @@ return space.newtuple([W_UnicodeObject.EMPTY, W_UnicodeObject.EMPTY, w_unistr]) else: - assert pos > 0 + assert pos >= 0 return space.newtuple([space.wrap(unistr[:pos]), w_unisub, space.wrap(unistr[pos+len(unisub):])]) From cfbolz at codespeak.net Thu Nov 22 21:42:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 21:42:13 +0100 (CET) Subject: [pypy-svn] r48947 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071122204213.BC3EB8220@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 21:42:13 2007 New Revision: 48947 Added: pypy/dist/pypy/objspace/std/test/test_ropeunicodeobject.py (contents, props changed) Modified: pypy/dist/pypy/objspace/std/ropeunicodeobject.py Log: oops, forget to add the rope unicode tests to svn :-(. do that and fix a test Modified: pypy/dist/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeunicodeobject.py Thu Nov 22 21:42:13 2007 @@ -22,10 +22,18 @@ from pypy.objspace.std.unicodetype import getdefaultencoding assert isinstance(w_str, W_RopeObject) encoding = getdefaultencoding(space) - return decode_string(space, w_str, encoding, "strict") + w_retval = decode_string(space, w_str, encoding, "strict") + if not space.is_true(space.isinstance(w_retval, space.w_unicode)): + raise OperationError( + space.w_TypeError, + space.wrap( + "decoder did not return an unicode object (type=%s)" % + space.type(w_retval).getname(space, '?'))) + assert isinstance(w_retval, W_RopeUnicodeObject) + return w_retval def decode_string(space, w_str, encoding, errors): - from pypy.objspace.std.unicodetype import unicode_from_encoded_object + from pypy.objspace.std.unicodetype import decode_object if errors is None or errors == "strict": node = w_str._node if encoding == 'ascii': @@ -39,9 +47,8 @@ result = rope.str_decode_utf8(node) if result is not None: return W_RopeUnicodeObject(result) - result = unicode_from_encoded_object(space, w_str, encoding, errors) - assert isinstance(result, W_RopeUnicodeObject) - return result + w_result = decode_object(space, w_str, encoding, errors) + return w_result def encode_unicode(space, w_unistr, encoding, errors): from pypy.objspace.std.unicodetype import getdefaultencoding, \ Added: pypy/dist/pypy/objspace/std/test/test_ropeunicodeobject.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/std/test/test_ropeunicodeobject.py Thu Nov 22 21:42:13 2007 @@ -0,0 +1,29 @@ +import py + +from pypy.objspace.std.test import test_stringobject, test_unicodeobject +from pypy.conftest import gettestobjspace + +class AppTestRopeObject(test_stringobject.AppTestStringObject): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withropeunicode": True}) + + def test_hash(self): + # doesn't make sense, since ropes hash differently than cpython's + # strings + pass + +class AppTestRopeUnicode(object): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withropeunicode": True}) + +class AppTestUnicodeRopeStdOnly(test_unicodeobject.AppTestUnicodeStringStdOnly): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withropeunicode": True}) + +class AppTestUnicodeRope(test_unicodeobject.AppTestUnicodeString): + + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withropeunicode": True}) From fijal at codespeak.net Thu Nov 22 21:42:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 22 Nov 2007 21:42:44 +0100 (CET) Subject: [pypy-svn] r48948 - pypy/dist/pypy/doc Message-ID: <20071122204244.7155D8222@code0.codespeak.net> Author: fijal Date: Thu Nov 22 21:42:44 2007 New Revision: 48948 Modified: pypy/dist/pypy/doc/_ref.txt Log: Fix reference to extfunctable Modified: pypy/dist/pypy/doc/_ref.txt ============================================================================== --- pypy/dist/pypy/doc/_ref.txt (original) +++ pypy/dist/pypy/doc/_ref.txt Thu Nov 22 21:42:44 2007 @@ -87,7 +87,6 @@ .. _`pypy/rpython/`: .. _`rpython/`: ../../pypy/rpython .. _`pypy/rpython/controllerentry.py`: ../../pypy/rpython/controllerentry.py -.. _`pypy/rpython/extfunctable.py`: ../../pypy/rpython/extfunctable.py .. _`rpython/lltypesystem/`: ../../pypy/rpython/lltypesystem .. _`pypy/rpython/lltypesystem/lltype.py`: .. _`rpython/lltypesystem/lltype.py`: ../../pypy/rpython/lltypesystem/lltype.py From xoraxax at codespeak.net Thu Nov 22 21:44:30 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 22 Nov 2007 21:44:30 +0100 (CET) Subject: [pypy-svn] r48949 - in pypy/dist/pypy: config translator/llvm Message-ID: <20071122204430.84EB68227@code0.codespeak.net> Author: xoraxax Date: Thu Nov 22 21:44:30 2007 New Revision: 48949 Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/translator/llvm/buildllvm.py Log: Add an option that can be used to change the opt options. If you use this new option with the argument "", you can actually build the prolog target without any abort() in opt. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Thu Nov 22 21:44:30 2007 @@ -214,7 +214,9 @@ OptionDescription("llvm", "GenLLVM options", [ BoolOption("debug", "Include the llops in the source as comments", default=False), BoolOption("logging", "Log how long the various parts of llvm generation take", default=False), - BoolOption("isolate", "Peform an isolated import", default=True), + BoolOption("isolate", "Perform an isolated import", default=True), + StrOption("opt_options", "Options passed to opt (influences level of optimization in LLVM)", + default="-std-compile-opts", cmdline="--translation-llvm-opt-options"), ]), OptionDescription("cli", "GenCLI options", [ Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Thu Nov 22 21:44:30 2007 @@ -53,7 +53,7 @@ self.cmds = [] def optimizations(self): - return '-std-compile-opts' + return self.genllvm.config.translation.llvm.opt_options def execute_cmds(self): c = stdoutcapture.Capture(mixed_out_err=True) From fijal at codespeak.net Thu Nov 22 21:56:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 22 Nov 2007 21:56:55 +0100 (CET) Subject: [pypy-svn] r48950 - pypy/dist/pypy/translator/llvm Message-ID: <20071122205655.BB878822B@code0.codespeak.net> Author: fijal Date: Thu Nov 22 21:56:55 2007 New Revision: 48950 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: Really tell what was wrong. Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Thu Nov 22 21:56:55 2007 @@ -49,7 +49,7 @@ include_path, includes, emit_llvm, plain, plain) if os.system(cmd) != 0: - raise Exception("Failed to run '%s'") + raise Exception("Failed to run '%s'" % cmd) llcode = open(plain + '.ll').read() From xoraxax at codespeak.net Thu Nov 22 22:02:04 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 22 Nov 2007 22:02:04 +0100 (CET) Subject: [pypy-svn] r48951 - pypy/dist/pypy/config Message-ID: <20071122210204.4D70B8235@code0.codespeak.net> Author: xoraxax Date: Thu Nov 22 22:02:03 2007 New Revision: 48951 Modified: pypy/dist/pypy/config/translationoption.py Log: Disable weakrefs for llvm and use the canonical option format for the opt options option. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Thu Nov 22 22:02:03 2007 @@ -29,7 +29,8 @@ "c": [("translation.type_system", "lltype")], "llvm": [("translation.type_system", "lltype"), ("translation.gc", "boehm"), - ("translation.backendopt.raisingop2direct_call", True)], + ("translation.backendopt.raisingop2direct_call", True), + ("translation.rweakref", False)], "cli": [("translation.type_system", "ootype")], "jvm": [("translation.type_system", "ootype")], "js": [("translation.type_system", "ootype")], @@ -216,7 +217,7 @@ BoolOption("logging", "Log how long the various parts of llvm generation take", default=False), BoolOption("isolate", "Perform an isolated import", default=True), StrOption("opt_options", "Options passed to opt (influences level of optimization in LLVM)", - default="-std-compile-opts", cmdline="--translation-llvm-opt-options"), + default="-std-compile-opts"), ]), OptionDescription("cli", "GenCLI options", [ From cfbolz at codespeak.net Thu Nov 22 23:08:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 22 Nov 2007 23:08:53 +0100 (CET) Subject: [pypy-svn] r48956 - pypy/dist/pypy/doc/config Message-ID: <20071122220853.900C7823E@code0.codespeak.net> Author: cfbolz Date: Thu Nov 22 23:08:53 2007 New Revision: 48956 Added: pypy/dist/pypy/doc/config/objspace.std.withropeunicode.txt (contents, props changed) Log: add documentation for unicode ropes. Added: pypy/dist/pypy/doc/config/objspace.std.withropeunicode.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.std.withropeunicode.txt Thu Nov 22 23:08:53 2007 @@ -0,0 +1,7 @@ +Use ropes to implement unicode strings (and also normal strings). + +See the section in `Standard Interpreter Optimizations`_ for more details. + +.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#ropes + + From antocuni at codespeak.net Fri Nov 23 00:05:33 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 23 Nov 2007 00:05:33 +0100 (CET) Subject: [pypy-svn] r48962 - pypy/dist/pypy/translator/microbench Message-ID: <20071122230533.428D0823C@code0.codespeak.net> Author: antocuni Date: Fri Nov 23 00:05:30 2007 New Revision: 48962 Modified: pypy/dist/pypy/translator/microbench/microbench.py Log: print the unexpected output in case of error Modified: pypy/dist/pypy/translator/microbench/microbench.py ============================================================================== --- pypy/dist/pypy/translator/microbench/microbench.py (original) +++ pypy/dist/pypy/translator/microbench/microbench.py Fri Nov 23 00:05:30 2007 @@ -52,7 +52,11 @@ data = [s for s in os.popen(exe + ' microbench.py %s 2>&1' % limit).readlines() if not s.startswith('debug:')] benchdata = {} for d in data: - testcase, took, duration, seconds = d.split() + try: + testcase, took, duration, seconds = d.split() + except ValueError: + print >> sys.stderr, 'Unexpected output:\n%s' % d + sys.exit(1) benchdata[testcase] = float(duration) if n == 0: benchdata_ref = benchdata From tismer at codespeak.net Fri Nov 23 00:52:55 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Fri, 23 Nov 2007 00:52:55 +0100 (CET) Subject: [pypy-svn] r48968 - pypy/branch/windows48396/pypy/translator/goal Message-ID: <20071122235255.AFAAC8160@code0.codespeak.net> Author: tismer Date: Fri Nov 23 00:52:55 2007 New Revision: 48968 Modified: pypy/branch/windows48396/pypy/translator/goal/app_main.py pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py Log: partial solution to the os import problem. The next version will avoid importing os, altogether. I think of producing these functions using the gateway. Modified: pypy/branch/windows48396/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/windows48396/pypy/translator/goal/app_main.py (original) +++ pypy/branch/windows48396/pypy/translator/goal/app_main.py Fri Nov 23 00:52:55 2007 @@ -146,8 +146,8 @@ # faking os, until we really are able to import it from source """ -Why fake_os ? -------------- +Why reload_os ? +--------------- When pypy is starting, the builtin modules are there, but os.path is a Python module. The current startup code depends on os.py, @@ -156,36 +156,16 @@ Os.py is a wrapper for system specific built-in modules and tries to unify the interface. One problem is os.environ which looks very differently per os. -Due to the way os.py initializes its environ variable on Windows, -it s hard to get a correct os.getenv that uses the actual environment -variables. In order to fix that without modifying os.py, we need to -re-import it several times, before and after compilation. - -When compiling the source, we first create a fake_os class instance. -fake_os contains all of os, when compiling.reduction seems to be not -appropriate, since lots of other modules are sucked in, at the same time. -A complete reduction is possible, but would require much more work. - -In the course of creating fake_os, we import the real os. We keep a record -of which modules were imported, before. - -During start of the entry_point, we call os.setup(). -This repeats an import of the specific parts in os.name. -Depending of the underlying os, this might or might not -cause a redefinition of certain os variables, but this happens -exactly like in os.py's initialization. -We then capture the variable environ, which may or may not come from the -specific os. -After that, we re-initialize our dict to what is expected in standard -os. The effect of all this magic is that we have captured the environ -cariable, again, and we now can redefine os.getenv to use this fresh -variable, being accurate and not frozen. - -At this point, we finally can compute the location of our import. - -As a side effect, the involved modules are re-imported, although they had -been compiled in, so PyPy behaves really conformant, making all .py -modules changeable, again. +Access to os.environ is not possible on Windows, due to caching +problems. We solve that by passing the PATH envirnment setting +in from a low-level interface. + +As an intermediate hack, the os.path functions are used until +os.py can be imported. The module is then re-imported. This +intermediate, since it will still leave os.environ unusable. +(the last version solved that by hairy import tricks) + +The next version will avoid early import of os, alltogether. """ def we_are_translated(): @@ -195,51 +175,23 @@ # which I did. return hasattr(sys, 'pypy_translation_info') -class fake_os: +class reload_os: def __init__(self): - import sys self.pre_import = sys.modules.keys() - import os - self.os = os - # make ourselves a clone of os - self.__dict__.update(self.os.__dict__) - - def setup(self): - # we now repeat the os-specific initialization, which - # must be done before importing os, since os hides - # variables after initialization. - specifics = __import__(self.os.name) - self.__dict__.update(specifics.__dict__) - # depending on the os, we now might or might not have - # a new environ variable. However, we can now - # repeat the environ initialisation from os.py - environ = self.os._Environ(self.environ) - # to be safe, reset our dict to be like os - self.__dict__.update(self.os.__dict__) - # but now we insert the fresh environ - self.environ = environ - del self.getenv - # use our method, instead of the os's one's - assert self.getenv - - def teardown(self): + + def reload(self): # re-load modules instead of using the pre-compiled ones # note that this gives trouble if we are not translated, # since some exithandler in threading.py complains - # XXX check if this is a bug to be fixed for cpython if we_are_translated(): for mod in sys.modules.keys(): if mod not in self.pre_import: del sys.modules[mod] global os import os - - def getenv(self, key, default=None): - """Get an environment variable, return None if it doesn't exist. - The optional second argument can specify an alternate default.""" - return self.environ.get(key, default) -os = fake_os() +reload_os = reload_os() +import os AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' @@ -249,16 +201,16 @@ else: IS_WINDOWS = False -def entry_point(executable, argv): +def entry_point(executable, argv, path): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH - os.setup() # this is the faked one if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): executable += '.exe' if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable): pass # the path is already more than just an executable name else: - path = os.getenv('PATH') + # path = os.getenv('PATH') + # we avoid getenv if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) @@ -286,7 +238,7 @@ sys.path = newpath # found! break - os.teardown() # from now on this is the real one + reload_os.reload() # from now on use a fresh import go_interactive = False run_command = False @@ -507,5 +459,5 @@ from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path - sys.exit(entry_point(sys.argv[0], sys.argv[1:])) + sys.exit(entry_point(sys.argv[0], sys.argv[1:], os.getenv("PATH"))) #sys.exit(entry_point('app_main.py', sys.argv[1:])) Modified: pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py Fri Nov 23 00:52:55 2007 @@ -43,7 +43,9 @@ space.call_function(w_run_toplevel, w_call_startup_gateway) w_executable = space.wrap(argv[0]) w_argv = space.newlist([space.wrap(s) for s in argv[1:]]) - w_exitcode = space.call_function(w_entry_point, w_executable, w_argv) + # we avoid using app-level environ due to the weird initialization of os + w_path = space.wrap(os.environ.get("PATH")) + w_exitcode = space.call_function(w_entry_point, w_executable, w_argv, w_path) exitcode = space.int_w(w_exitcode) # try to pull it all in ## from pypy.interpreter import main, interactive, error From xoraxax at codespeak.net Fri Nov 23 10:29:02 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 10:29:02 +0100 (CET) Subject: [pypy-svn] r48971 - pypy/dist/pypy/doc/config Message-ID: <20071123092902.170C681FE@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 10:29:00 2007 New Revision: 48971 Added: pypy/dist/pypy/doc/config/translation.llvm.opt_options.txt Log: Add translation.llvm.opt_options.txt. Added: pypy/dist/pypy/doc/config/translation.llvm.opt_options.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/translation.llvm.opt_options.txt Fri Nov 23 10:29:00 2007 @@ -0,0 +1,2 @@ +The options passed to the LLVM tool opt which is resposible for optimizations. By default, only "-std-compile-opts" is used. + From antocuni at codespeak.net Fri Nov 23 11:02:53 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 23 Nov 2007 11:02:53 +0100 (CET) Subject: [pypy-svn] r48972 - in pypy/dist/pypy: rpython/test translator/cli/src translator/jvm/src/pypy Message-ID: <20071123100253.1417B81F0@code0.codespeak.net> Author: antocuni Date: Fri Nov 23 11:02:53 2007 New Revision: 48972 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/translator/cli/src/ll_math.cs pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: - add a test for math.ldexp - implement math.ldexp for jvm - fix math.ldexp for cli (what is not tested etc. etc.) Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Fri Nov 23 11:02:53 2007 @@ -161,6 +161,13 @@ mantissa, exponent = math.frexp(10/3.0) assert self.float_eq(res.item0, mantissa) and self.float_eq(res.item1, exponent) + def test_builtin_math_ldexp(self): + import math + def fn(a, b): + return math.ldexp(a, b) + assert self.interpret(fn, [1, 2]) == 4 + self.interpret_raises(OverflowError, fn, [1, 100000]) + def test_builtin_math_modf(self): import math def fn(f): Modified: pypy/dist/pypy/translator/cli/src/ll_math.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_math.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_math.cs Fri Nov 23 11:02:53 2007 @@ -108,7 +108,7 @@ // Handle overflow if (e >= MEXP) - return 2.0*MAXNUM; + Helpers.raise_OverflowError(); if (e < 1) { Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Fri Nov 23 11:02:53 2007 @@ -249,7 +249,7 @@ try { return Double.parseDouble(s); } catch(NumberFormatException ex) { - interlink.throwValueError(); + throwValueError(); return 0.0; // not reached } } @@ -976,6 +976,18 @@ return x % y; } + public static double ll_math_ldexp(double v, int w) { + return check(v * Math.pow(2.0, w)); + } + + private static double check(double v) { + if (Double.isNaN(v)) + throwValueError(); + if (Double.isInfinite(v)) + throwOverflowError(); + return v; + } + // ---------------------------------------------------------------------- // Convenient Helpers for throwing exceptions // From cfbolz at codespeak.net Fri Nov 23 12:10:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:10:16 +0100 (CET) Subject: [pypy-svn] r48973 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071123111016.438FD81ED@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:10:15 2007 New Revision: 48973 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Fri Nov 23 12:10:15 2007 @@ -8,21 +8,30 @@ - Review the cpython regression test suite (skips/failures) - think about/remove orphaned parts -Wednesday afternoon: - - Start slowly thinking about 1.1 plans -Thursday afternoon: +Discussion topics (dynamically scheduled): - discuss callback mechanism for rffi/lltype/ll2ctypes - -Saturday: - discuss approaches for extension modules + (probably Armin, Maciek, Alexander) + + - discuss Q/A (Holger, Samuele, Carl Friedrich) + - discuss how to deal with the windows situation + Translation toolchain +++++++++++++++++++++ - - refactor rffi header, sources, includes management + - refactor rffi header, sources, includes management IN-PROGRESS + (maciek, alexander) + + - py-lib 0.9.1 release? (holger, maciek) - - fix windows compilation (christian) + - tried to translate PyPy with LLVM IN-PROGRESS (found a bug in LLVM opt) + + - some more work on cleaning up the startup process IN-PROGRESS (christian) + + - fix windows compilation: rffi-platform checks have problem, which are being + approached now - find out why our memcpy behaves so erratically GCC behaves very erratically, so we won't work on this for now @@ -36,21 +45,19 @@ for rtti, simplify translator/c/gc.py - kill opaque hacks in the C backend - - cleanup pyobj.py and rclass stuff and remove the ability to make cpython extensions, just dll - update gencapicall impl (cfbolz, samuele) + - cleanup pyobj.py and rclass stuff and remove the ability to make cpython + extensions, just dll + update gencapicall impl (cfbolz, samuele) IN-PROGRESS - clean up the tangle of including headers in the C backend - kill half concrete wrapper - - implement global variables in rffi (anto, maciek) DONE - - review pdbplus, especially the graph commands, also in the light of https://codespeak.net/issue/pypy-dev/issue303 and the fact that we can have more than one translator/annotator around (with the timeshifter) - - fix test_thread etc Interpreter +++++++++++ @@ -65,9 +72,10 @@ performance-critical IN-PROGRESS: moved super, classmethod, statimethod to interplevel, some dict methods - - remove prebuilt costate vars (might happen before) + - remove prebuilt costate vars (christian, armin?) + + - look at the new failing test in test_descr.py - - think about the __name__ of MixedModules, because *.py makes CLR unhappy Modules +++++++ From cfbolz at codespeak.net Fri Nov 23 12:15:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:15:57 +0100 (CET) Subject: [pypy-svn] r48974 - pypy/branch/kill-keepalives Message-ID: <20071123111557.1D83F815C@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:15:56 2007 New Revision: 48974 Removed: pypy/branch/kill-keepalives/ Log: the interesting ideas of this branch are on the trunk now, killing it. From cfbolz at codespeak.net Fri Nov 23 12:16:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:16:14 +0100 (CET) Subject: [pypy-svn] r48975 - pypy/branch/prolog-bytecode Message-ID: <20071123111614.2B192815C@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:16:13 2007 New Revision: 48975 Removed: pypy/branch/prolog-bytecode/ Log: this is merged, kill the branch From cfbolz at codespeak.net Fri Nov 23 12:16:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:16:39 +0100 (CET) Subject: [pypy-svn] r48976 - pypy/branch/unicode-objspace Message-ID: <20071123111639.B12668162@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:16:39 2007 New Revision: 48976 Removed: pypy/branch/unicode-objspace/ Log: this branch is merged, kill it From cfbolz at codespeak.net Fri Nov 23 12:17:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:17:25 +0100 (CET) Subject: [pypy-svn] r48977 - pypy/branch/pypy-rpython-unicode Message-ID: <20071123111725.5B9B8815F@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:17:24 2007 New Revision: 48977 Removed: pypy/branch/pypy-rpython-unicode/ Log: another merged branch, kill kill kill From cfbolz at codespeak.net Fri Nov 23 12:18:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:18:17 +0100 (CET) Subject: [pypy-svn] r48978 - pypy/branch/zlib-rffi-impl Message-ID: <20071123111817.149558190@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:18:17 2007 New Revision: 48978 Removed: pypy/branch/zlib-rffi-impl/ Log: this is merged as well From cfbolz at codespeak.net Fri Nov 23 12:18:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:18:38 +0100 (CET) Subject: [pypy-svn] r48979 - pypy/branch/rdict-index-based2 Message-ID: <20071123111838.99F798190@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:18:38 2007 New Revision: 48979 Removed: pypy/branch/rdict-index-based2/ Log: merged as well From cfbolz at codespeak.net Fri Nov 23 12:19:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:19:34 +0100 (CET) Subject: [pypy-svn] r48980 - pypy/branch/doc10 Message-ID: <20071123111934.50BAC8190@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:19:34 2007 New Revision: 48980 Removed: pypy/branch/doc10/ Log: this was merged, or if it wasn't the changes don't matter today From cfbolz at codespeak.net Fri Nov 23 12:20:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:20:58 +0100 (CET) Subject: [pypy-svn] r48981 - pypy/branch/temp Message-ID: <20071123112058.428EC81ED@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:20:57 2007 New Revision: 48981 Removed: pypy/branch/temp/ Log: a branch with the name temp can by definition be removed From cfbolz at codespeak.net Fri Nov 23 12:39:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:39:58 +0100 (CET) Subject: [pypy-svn] r48982 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/goal Message-ID: <20071123113958.7D6498200@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:39:57 2007 New Revision: 48982 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/translator/goal/targetrpystonex.py Log: this target is broken (and, incidentally, uses the equally broken symboltable feature) From cfbolz at codespeak.net Fri Nov 23 12:40:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:40:18 +0100 (CET) Subject: [pypy-svn] r48983 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . test Message-ID: <20071123114018.A9D2A8200@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:40:17 2007 New Revision: 48983 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/symboltable.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_symboltable.py Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Log: remove the symboltable feature, it's disabled anyway Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Fri Nov 23 12:40:17 2007 @@ -20,7 +20,6 @@ class CBuilder(object): c_source_filename = None _compiled = False - symboltable = None modulename = None def __init__(self, translator, entrypoint, config, libraries=None, @@ -128,14 +127,10 @@ CBuilder.have___thread = check_under_under_thread() if not self.standalone: assert not self.config.translation.instrument - from pypy.translator.c.symboltable import SymbolTable - # XXX fix symboltable - #self.symboltable = SymbolTable() cfile, extra, include_dirs, library_dirs = \ gen_source(db, modulename, targetdir, defines = defines, exports = self.exports, - symboltable = self.symboltable, libraries = self.libraries, extra_info = extra_info) else: @@ -197,8 +192,6 @@ mod = import_module_from_directory(self.c_source_filename.dirpath(), self.c_source_filename.purebasename) self.c_ext_module = mod - if self.symboltable: - self.symboltable.attach(mod) # hopefully temporary hack return mod def isolated_import(self): @@ -686,7 +679,7 @@ def gen_source(database, modulename, targetdir, defines={}, exports={}, - symboltable=None, libraries=[], extra_info={}): + libraries=[], extra_info={}): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -733,27 +726,6 @@ sg.gen_readable_parts_of_source(f) # - # Debugging info - # - if symboltable: - print >> f - print >> f, '/*******************************************************/' - print >> f, '/*** Debugging info ***/' - print >> f - print >> f, 'static int debuginfo_offsets[] = {' - for node in database.structdefnodes.values(): - for expr in symboltable.generate_type_info(database, node): - print >> f, '\t%s,' % expr - print >> f, '\t0 };' - print >> f, 'static void *debuginfo_globals[] = {' - for node in database.globalcontainers(): - if not isinstance(node, PyObjectNode): - result = symboltable.generate_global_info(database, node) - print >> f, '\t%s,' % (result,) - print >> f, '\tNULL };' - print >> f, '#include "src/debuginfo.h"' - - # # PyObject support (strange) code # pyobjmaker = database.pyobjmaker From cfbolz at codespeak.net Fri Nov 23 12:46:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 12:46:21 +0100 (CET) Subject: [pypy-svn] r48984 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/test Message-ID: <20071123114621.3B8F081FE@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 12:46:20 2007 New Revision: 48984 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py Log: use the new interface Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py Fri Nov 23 12:46:20 2007 @@ -40,7 +40,8 @@ backend_optimizations(t) if view: t.viewcg() - return getattr(cbuilder.import_module(), func.__name__) + cbuilder.import_module() + return cbuilder.get_entry_point() def example_int_long(arg=int): return long(arg+42) From pedronis at codespeak.net Fri Nov 23 13:13:54 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 23 Nov 2007 13:13:54 +0100 (CET) Subject: [pypy-svn] r48985 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool Message-ID: <20071123121354.324A58187@code0.codespeak.net> Author: pedronis Date: Fri Nov 23 13:13:53 2007 New Revision: 48985 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py Log: have a so_ext constant that specify the platform dll file extension Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py Fri Nov 23 13:13:53 2007 @@ -11,6 +11,11 @@ debug = 0 +if sys.platform == 'win32': + so_ext = '.dll' +else: + so_ext = '.so' + def compiler_command(): # e.g. for tcc, you might set this to # "tcc -shared -o %s.so %s.c" @@ -158,7 +163,7 @@ modname = str(cache_dir.join(modname)) compile_c_module(cfiles, modname, include_dirs=include_dirs, libraries=libraries) - return modname + '.so' + return modname + so_ext def make_module_from_c(cfile, include_dirs=None, libraries=[]): cfile = py.path.local(cfile) From fijal at codespeak.net Fri Nov 23 13:15:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Nov 2007 13:15:59 +0100 (CET) Subject: [pypy-svn] r48986 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool Message-ID: <20071123121559.6E73A8187@code0.codespeak.net> Author: fijal Date: Fri Nov 23 13:15:58 2007 New Revision: 48986 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py Log: Check in ExternalCompilationInfo just to find conflicts Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/cbuild.py Fri Nov 23 13:15:58 2007 @@ -11,6 +11,55 @@ debug = 0 +class ExternalCompilationInfo(object): + + _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs', + 'post_include_lines', 'libraries', 'library_dirs', + 'separate_module_sources', 'separate_module_files'] + + def __init__(self, + pre_include_lines = [], + includes = [], + include_dirs = [], + post_include_lines = [], + libraries = [], + library_dirs = [], + separate_module_sources = [], + separate_module_files = []): + """ + pre_include_lines: list of lines that should be put at the top + of the generated .c files, before any #include. They shouldn't + contain an #include themselves. + + includes: list of .h file names to be #include'd from the + generated .c files. + + include_dirs: list of dir names that is passed to the C compiler + + post_include_lines: list of lines that should be put at the top + of the generated .c files, after the #includes. + + libraries: list of library names that is passed to the linker + + library_dirs: list of dir names that is passed to the linker + + separate_module_sources: list of multiline strings that are + each written to a .c file and compiled separately and linked + later on. (If function prototypes are needed for other .c files + to access this, they can be put in post_include_lines.) + + separate_module_files: list of .c file names that are compiled + separately and linked later on. (If an .h file is needed for + other .c files to access this, it can be put in includes.) + """ + for name in self._ATTRIBUTES: + value = locals()[name] + assert isinstance(value, (list, tuple)) + setattr(self, name, tuple(value)) + + # XXX custom hash and eq functions, they should be compared + # by contents + if sys.platform == 'win32': so_ext = '.dll' else: From pedronis at codespeak.net Fri Nov 23 13:56:20 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 23 Nov 2007 13:56:20 +0100 (CET) Subject: [pypy-svn] r48987 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator: . c c/test test Message-ID: <20071123125620.7BE238187@code0.codespeak.net> Author: pedronis Date: Fri Nov 23 13:56:19 2007 New Revision: 48987 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_boehm.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/driver.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py Log: (cfbolz, pedronis) don't depend on the so generated to be proper c extensions Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Fri Nov 23 13:56:19 2007 @@ -5,6 +5,7 @@ from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.gensupp import uniquemodulename, NameManager +from pypy.translator.tool.cbuild import so_ext from pypy.translator.tool.cbuild import compile_c_module from pypy.translator.tool.cbuild import build_executable, CCompiler, ProfOpt from pypy.translator.tool.cbuild import import_module_from_directory @@ -26,6 +27,7 @@ gcpolicy=None): self.translator = translator self.entrypoint = entrypoint + self.entrypoint_name = self.entrypoint.func_name self.originalentrypoint = entrypoint self.gcpolicy = gcpolicy if gcpolicy is not None and gcpolicy.requires_stackless: @@ -74,7 +76,8 @@ # build entrypoint and eventually other things to expose pf = self.getentrypointptr() pfname = db.get(pf) - self.exports[self.entrypoint.func_name] = pf + self.exports[self.entrypoint_name] = pf + self.c_entrypoint_name = pfname db.complete() # add library dependencies @@ -168,7 +171,7 @@ class CExtModuleBuilder(CBuilder): standalone = False - c_ext_module = None + _module = None def getentrypointptr(self, obj=None): if obj is None: @@ -186,30 +189,58 @@ libraries=self.libraries) self._compiled = True - def import_module(self): + def _make_wrapper_module(self): + modfile = self.c_source_filename.new(ext=".py") + CODE = """ +import ctypes + +_lib = ctypes.PyDLL("%(so_name)s") + +_entry_point = getattr(_lib, "%(c_entrypoint_name)s") +_entry_point.restype = ctypes.py_object +_entry_point.argtypes = 3*(ctypes.py_object,) + +def %(entrypoint_name)s(*args, **kwds): + return _entry_point(None, args, kwds) + +_malloc_counters = _lib.malloc_counters +_malloc_counters.restype = ctypes.py_object +_malloc_counters.argtypes = 2*(ctypes.py_object,) + +def malloc_counters(): + return _malloc_counters(None, None) +""" % {'so_name': self.c_source_filename.new(ext=so_ext), + 'entrypoint_name': self.entrypoint_name, + 'c_entrypoint_name': self.c_entrypoint_name} + modfile.write(CODE) + self._module_path = modfile + + def _import_module(self, isolated=False): + if self._module is not None: + return self._module assert self._compiled - assert not self.c_ext_module - mod = import_module_from_directory(self.c_source_filename.dirpath(), - self.c_source_filename.purebasename) - self.c_ext_module = mod + assert not self._module + self._make_wrapper_module() + if not isolated: + mod = self._module_path.pyimport() + else: + mod = isolate.Isolate((str(self._module_path.dirpath()), + self._module_path.purebasename)) + self._module = mod return mod - - def isolated_import(self): - assert self._compiled - assert not self.c_ext_module - self.c_ext_module = isolate.Isolate((str(self.c_source_filename.dirpath()), - self.c_source_filename.purebasename)) - return self.c_ext_module - def get_entry_point(self): - assert self.c_ext_module - return getattr(self.c_ext_module, - self.entrypoint.func_name) - + def get_entry_point(self, isolated=False): + self._import_module(isolated=isolated) + return getattr(self._module, self.entrypoint_name) + + def get_malloc_counters(self, isolated=False): + self._import_module(isolated=isolated) + return self._module.malloc_counters + def cleanup(self): - assert self.c_ext_module - if isinstance(self.c_ext_module, isolate.Isolate): - isolate.close_isolate(self.c_ext_module) + assert self._module + if isinstance(self._module, isolate.Isolate): + isolate.close_isolate(self._module) class CStandaloneBuilder(CBuilder): standalone = True Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_boehm.py Fri Nov 23 13:56:19 2007 @@ -41,9 +41,8 @@ if conftest.option.view: t.view() cbuilder.compile() - mod = cbuilder.isolated_import() self._cleanups.append(cbuilder.cleanup) # schedule cleanup after test - return cbuilder.get_entry_point() + return cbuilder.get_entry_point(isolated=True) return compile() Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Fri Nov 23 13:56:19 2007 @@ -42,15 +42,14 @@ compiled_fn = t.compile_c() if conftest.option.view: t.view() - # XXX fish fish fish some more - module = t.driver.cbuilder.c_ext_module + malloc_counters = t.driver.cbuilder.get_malloc_counters() def checking_fn(*args, **kwds): if 'expected_extra_mallocs' in kwds: expected_extra_mallocs = kwds.pop('expected_extra_mallocs') else: expected_extra_mallocs = 0 res = compiled_fn(*args, **kwds) - mallocs, frees = module.malloc_counters() + mallocs, frees = malloc_counters() if isinstance(expected_extra_mallocs, int): assert mallocs - frees == expected_extra_mallocs else: Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py Fri Nov 23 13:56:19 2007 @@ -23,16 +23,15 @@ builder = genc.CExtModuleBuilder(t, fn, config=config) builder.generate_source(defines={'COUNT_OP_MALLOCS': 1}) builder.compile() - builder.import_module() if conftest.option.view: t.view() - module = builder.c_ext_module compiled_fn = builder.get_entry_point() + malloc_counters = builder.get_malloc_counters() def checking_fn(*args, **kwds): try: return compiled_fn(*args, **kwds) finally: - mallocs, frees = module.malloc_counters() + mallocs, frees = malloc_counters() assert mallocs == frees return checking_fn Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py Fri Nov 23 13:56:19 2007 @@ -35,7 +35,6 @@ builder = genc.CExtModuleBuilder(t, func, config=t.config) builder.generate_source() builder.compile() - builder.import_module() return builder.get_entry_point() def getcompiled(self, func, argtypes=None, view=False): Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/driver.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/driver.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/driver.py Fri Nov 23 13:56:19 2007 @@ -514,7 +514,6 @@ self.c_entryp = cbuilder.executable_name self.create_exe() else: - cbuilder.import_module() self.c_entryp = cbuilder.get_entry_point() # task_compile_c = taskdef(task_compile_c, ['source_c'], "Compiling c source") Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py Fri Nov 23 13:56:19 2007 @@ -40,7 +40,6 @@ backend_optimizations(t) if view: t.viewcg() - cbuilder.import_module() return cbuilder.get_entry_point() def example_int_long(arg=int): From cfbolz at codespeak.net Fri Nov 23 15:48:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 15:48:11 +0100 (CET) Subject: [pypy-svn] r48988 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071123144811.A8A718207@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 15:48:10 2007 New Revision: 48988 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Log: kill this function (again), it's not used Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Fri Nov 23 15:48:10 2007 @@ -355,13 +355,6 @@ f.close() -def translator2database(translator, entrypoint): - pf = lltype.pyobjectptr(entrypoint) - db = LowLevelDatabase(translator) - db.get(pf) - db.complete() - return db, pf - # ____________________________________________________________ SPLIT_CRITERIA = 65535 # support VC++ 7.2 From hpk at codespeak.net Fri Nov 23 16:55:28 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 23 Nov 2007 16:55:28 +0100 (CET) Subject: [pypy-svn] r48989 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071123155528.A2E218209@code0.codespeak.net> Author: hpk Date: Fri Nov 23 16:55:27 2007 New Revision: 48989 Added: pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt Log: outcome of cfbolz, sameuele, holger discussion Added: pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt Fri Nov 23 16:55:27 2007 @@ -0,0 +1,60 @@ +discussion notes about QA issues (cf, pedronis, hpk) + +stability & testing + +status: + +- automated nightly test runs of pypy tests on 32bit linux machine + (wyvern) + +- automated nightly builds on tuatara (PowerPC OSX) for builds + and benchmarks + +- nightly run of cpython regression tests on top of several pypy-c's + (running on samuele's linux Desktop machine) + +- maciej (XXX) runs nightly tests of pypy-c, py.test -A on a private + machine + + +missing: + +- automated windows tests + - a dedicated remotely accessible windows machine + (cfbolz investigates at HHU) + - setting up nightly runs + +- aggregation of information from all these automated runs + goal: present summary of all results from automated runs at one place + + +releases + +goal: + - what does "making a release" mean? + + - in addition to tests, run pypy with some real applications + - have pervasive testing (see above) on demand (and not only + nightly) and on branches + + - write small-to-medium automatically tested example applications + for the special features of PyPy (thunk, taint, stackless, greenlets, tp, + sandboxing?, ropes) + + - how to publish new features/enhancements/cool stuff? + - through Blog Posts + - maybe dist which snapshots a usable state of that feature + +documentation +- goal: try to sort out current pypy/doc mess and bring it to more + maintainable state + +- write "books" with many subsections about topics, e.g. + "The complete guide to the Python Standard Interpreter" + "Building and Programming with RPython" + "Architecture and Implementation" + + in any case, the idea is to keep the number of such "books" around 3, + first think about sections, then migrate existing docs to it + + From hpk at codespeak.net Fri Nov 23 17:38:57 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 23 Nov 2007 17:38:57 +0100 (CET) Subject: [pypy-svn] r48990 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071123163857.BDF8E81ED@code0.codespeak.net> Author: hpk Date: Fri Nov 23 17:38:56 2007 New Revision: 48990 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt Log: small tweaks Modified: pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/qa-notes.txt Fri Nov 23 17:38:56 2007 @@ -13,9 +13,6 @@ - nightly run of cpython regression tests on top of several pypy-c's (running on samuele's linux Desktop machine) -- maciej (XXX) runs nightly tests of pypy-c, py.test -A on a private - machine - missing: @@ -26,6 +23,7 @@ - aggregation of information from all these automated runs goal: present summary of all results from automated runs at one place + consider existing tools releases @@ -34,6 +32,7 @@ - what does "making a release" mean? - in addition to tests, run pypy with some real applications + - have pervasive testing (see above) on demand (and not only nightly) and on branches From hpk at codespeak.net Fri Nov 23 17:59:22 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 23 Nov 2007 17:59:22 +0100 (CET) Subject: [pypy-svn] r48991 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071123165922.9C31781EF@code0.codespeak.net> Author: hpk Date: Fri Nov 23 17:59:22 2007 New Revision: 48991 Added: pypy/extradoc/sprintinfo/gothenburg-2007/extensions-notes.txt Log: (cfbolz notetaking, all discussing): notes about how to approach extension modules Added: pypy/extradoc/sprintinfo/gothenburg-2007/extensions-notes.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2007/extensions-notes.txt Fri Nov 23 17:59:22 2007 @@ -0,0 +1,26 @@ +How to support the notion of extension modules +============================================== + +participants: Maciek, Armin, Alexander, Richard, Jacob, Christian + +two approaches + - app-level ctypes + - idea is to have pygame-ctypes as an intermediate goal + - the hard bit is callbacks + - only applevel code, which is good + - making ctypes_platform a generally useful thing (even for CPython) + + - separate compilation, mixed modules + - two step process: + - prototyping for faster module development + - really loading the module later + - boehm only + - would allow to wrap C libs with rffi + - annotations on the object space method fixed + - fixed names for those in the C library + - duplication of low-level helpers, rlib + +the first one is easier to implement, to use, more useful for external people, +which is why it is a priority (but we will need both eventually) + +far future idea: can we support CPython C extensions? From rxe at codespeak.net Fri Nov 23 18:05:42 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 23 Nov 2007 18:05:42 +0100 (CET) Subject: [pypy-svn] r48992 - in pypy/dist/pypy/translator: . c Message-ID: <20071123170542.9FC0B80E8@code0.codespeak.net> Author: rxe Date: Fri Nov 23 18:05:42 2007 New Revision: 48992 Modified: pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/exceptiontransform.py Log: (arigo, rxe) rewrite fetch/restore operations in exception transformer Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Fri Nov 23 18:05:42 2007 @@ -129,15 +129,6 @@ args = [funcgen.expr(v) for v in op.args] return 'OP_FREE(%s);' % (args[0], ) - def OP_GC_FETCH_EXCEPTION(self, funcgen, op): - result = funcgen.expr(op.result) - return ('%s = RPyFetchExceptionValue();\n' - 'RPyClearException();') % (result, ) - - def OP_GC_RESTORE_EXCEPTION(self, funcgen, op): - argh = funcgen.expr(op.args[0]) - return 'if (%s != NULL) RPyRaiseException(RPYTHON_TYPE_OF_EXC_INST(%s), %s);' % (argh, argh, argh) - def OP_GC__COLLECT(self, funcgen, op): return '' @@ -216,15 +207,6 @@ def convert_weakref_to(self, ptarget): return boehm.convert_weakref_to(ptarget) - def OP_GC_FETCH_EXCEPTION(self, funcgen, op): - result = funcgen.expr(op.result) - return ('%s = RPyFetchExceptionValue();\n' - 'RPyClearException();') % (result, ) - - def OP_GC_RESTORE_EXCEPTION(self, funcgen, op): - argh = funcgen.expr(op.args[0]) - return 'if (%s != NULL) RPyRaiseException(RPYTHON_TYPE_OF_EXC_INST(%s), %s);' % (argh, argh, argh) - def OP_GC__COLLECT(self, funcgen, op): return 'GC_gcollect(); GC_invoke_finalizers();' Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Fri Nov 23 18:05:42 2007 @@ -86,6 +86,15 @@ exc_data.exc_type = etype exc_data.exc_value = evalue + def rpyexc_fetch_exception(): + evalue = rpyexc_fetch_value() + rpyexc_clear() + return evalue + + def rpyexc_restore_exception(evalue): + if evalue: + rpyexc_raise(rclass.ll_inst_type(evalue), evalue) + def rpyexc_raise_runtime_error(): rpyexc_raise(runtime_error_ll_exc_type, runtime_error_ll_exc) @@ -127,6 +136,16 @@ rpyexc_raise_runtime_error, [], lltype.Void) + self.rpyexc_fetch_exception_ptr = self.build_func( + "RPyFetchException", + rpyexc_fetch_exception, + [], self.lltype_of_exception_value) + + self.rpyexc_restore_exception_ptr = self.build_func( + "RPyRestoreException", + rpyexc_restore_exception, + [self.lltype_of_exception_value], lltype.Void) + self.mixlevelannotator.finish() self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() @@ -162,6 +181,7 @@ n_gen_exc_checks = 0 for block in list(graph.iterblocks()): self.replace_stack_unwind(block) + self.replace_fetch_restore_operations(block) need_exc_matching, gen_exc_checks = self.transform_block(graph, block) n_need_exc_matching_blocks += need_exc_matching n_gen_exc_checks += gen_exc_checks @@ -180,6 +200,16 @@ block.operations[i].opname = "direct_call" block.operations[i].args = [self.rpyexc_raise_runtime_error_ptr] + def replace_fetch_restore_operations(self, block): + for i in range(len(block.operations)): + if block.operations[i].opname == 'gc_fetch_exception': + block.operations[i].opname = "direct_call" + block.operations[i].args = [self.rpyexc_fetch_exception_ptr] + + if block.operations[i].opname == 'gc_restore_exception': + block.operations[i].opname = "direct_call" + block.operations[i].args.insert(0, self.rpyexc_restore_exception_ptr) + def transform_block(self, graph, block): need_exc_matching = False n_gen_exc_checks = 0 From rxe at codespeak.net Fri Nov 23 18:12:43 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 23 Nov 2007 18:12:43 +0100 (CET) Subject: [pypy-svn] r48993 - pypy/dist/pypy/translator Message-ID: <20071123171243.DEAF681FB@code0.codespeak.net> Author: rxe Date: Fri Nov 23 18:12:43 2007 New Revision: 48993 Modified: pypy/dist/pypy/translator/exceptiontransform.py Log: (arigo, rxe) add a comment as to why these replace operations work Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Fri Nov 23 18:12:43 2007 @@ -201,6 +201,9 @@ block.operations[i].args = [self.rpyexc_raise_runtime_error_ptr] def replace_fetch_restore_operations(self, block): + # the gctransformer will create these operations. It looks as if the + # order of transformations is important - but the gctransformer will + # put them in a new graph, so all transformations will run again. for i in range(len(block.operations)): if block.operations[i].opname == 'gc_fetch_exception': block.operations[i].opname = "direct_call" From rxe at codespeak.net Fri Nov 23 18:44:27 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 23 Nov 2007 18:44:27 +0100 (CET) Subject: [pypy-svn] r48995 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071123174427.9075781DA@code0.codespeak.net> Author: rxe Date: Fri Nov 23 18:44:26 2007 New Revision: 48995 Modified: pypy/dist/pypy/translator/llvm/module/boehm.h pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/test_genllvm.py Log: (fijal, rxe) make finalizer ha ha work on genllvm Modified: pypy/dist/pypy/translator/llvm/module/boehm.h ============================================================================== --- pypy/dist/pypy/translator/llvm/module/boehm.h (original) +++ pypy/dist/pypy/translator/llvm/module/boehm.h Fri Nov 23 18:44:26 2007 @@ -31,6 +31,10 @@ GC_invoke_finalizers(); } +void pypy_register_finalizer(void *whatever, void *proc) { + GC_REGISTER_FINALIZER(whatever, (GC_finalization_proc)proc, NULL, NULL, NULL); +} + extern GC_all_interior_pointers; // startup specific code for boehm Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Fri Nov 23 18:44:26 2007 @@ -306,8 +306,9 @@ self.db.gcpolicy._zeromalloc(self.codewriter, opr.retref, opr.argrefs[0], atomic=True) def boehm_register_finalizer(self, opr): - # ha, ha - pass + tmpvar = self._tmp() + self.codewriter.cast(tmpvar, opr.argtypes[1], opr.argrefs[1], 'i8 *') + self.codewriter.call(None, 'void', '@pypy_register_finalizer', ['i8 *', 'i8 *'], [opr.argrefs[0], tmpvar]) def call_boehm_gc_alloc(self, opr): word = self.db.get_machine_word() Modified: pypy/dist/pypy/translator/llvm/test/test_genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_genllvm.py Fri Nov 23 18:44:26 2007 @@ -4,6 +4,7 @@ import py from pypy.rlib.rarithmetic import r_uint +from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.test.runtest import * @@ -471,3 +472,39 @@ f = compile_function(f, [int]) res = f(4) assert res == 4*(4+2) + +def test__del__(): + from pypy.rpython.lltypesystem.lloperation import llop + class State: + pass + s = State() + class A(object): + def __del__(self): + s.a_dels += 1 + class B(A): + def __del__(self): + s.b_dels += 1 + class C(A): + pass + def f(): + s.a_dels = 0 + s.b_dels = 0 + A() + B() + C() + A() + B() + C() + llop.gc__collect(lltype.Void) + return s.a_dels * 10 + s.b_dels + fn = compile_function(f, []) + # we can't demand that boehm has collected all of the objects, + # even with the gc__collect call. calling the compiled + # function twice seems to help, though. + res = 0 + res += fn() + res += fn() + + # if res is still 0, then we haven't tested anything so fail. + assert 0 < res <= 84 + From tismer at codespeak.net Fri Nov 23 19:05:18 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Fri, 23 Nov 2007 19:05:18 +0100 (CET) Subject: [pypy-svn] r48996 - in pypy/branch/windows48396/pypy/translator/goal: . test2 Message-ID: <20071123180518.B74558202@code0.codespeak.net> Author: tismer Date: Fri Nov 23 19:05:18 2007 New Revision: 48996 Added: pypy/branch/windows48396/pypy/translator/goal/nanos.py (contents, props changed) pypy/branch/windows48396/pypy/translator/goal/test2/test_nanos.py (contents, props changed) Modified: pypy/branch/windows48396/pypy/translator/goal/app_main.py pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py Log: a final solution to the import problem in app_main.py Actually quite elegant and minimal Modified: pypy/branch/windows48396/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/windows48396/pypy/translator/goal/app_main.py (original) +++ pypy/branch/windows48396/pypy/translator/goal/app_main.py Fri Nov 23 19:05:18 2007 @@ -144,64 +144,27 @@ # ____________________________________________________________ # Main entry point -# faking os, until we really are able to import it from source -""" -Why reload_os ? ---------------- - -When pypy is starting, the builtin modules are there, but os.path -is a Python module. The current startup code depends on os.py, -which had the side effect that os.py got frozen into the binary. - -Os.py is a wrapper for system specific built-in modules and tries -to unify the interface. One problem is os.environ which looks -very differently per os. -Access to os.environ is not possible on Windows, due to caching -problems. We solve that by passing the PATH envirnment setting -in from a low-level interface. - -As an intermediate hack, the os.path functions are used until -os.py can be imported. The module is then re-imported. This -intermediate, since it will still leave os.environ unusable. -(the last version solved that by hairy import tricks) - -The next version will avoid early import of os, alltogether. -""" +# see nanos.py for explainment why we do not import os here +# CAUTION! +# remember to update nanos.py if you are using more functions +# from os or os.path! +# Running test/test_nanos.py might be helpful as well. def we_are_translated(): - # this function does not exist on app-level. - # Don't confuse it with - # from pypy.rlib.objectmodel import we_are_translated - # which I did. + # app-level, very different from pypy.rlib.objectmodel.we_are_translated return hasattr(sys, 'pypy_translation_info') -class reload_os: - def __init__(self): - self.pre_import = sys.modules.keys() - - def reload(self): - # re-load modules instead of using the pre-compiled ones - # note that this gives trouble if we are not translated, - # since some exithandler in threading.py complains - if we_are_translated(): - for mod in sys.modules.keys(): - if mod not in self.pre_import: - del sys.modules[mod] - global os - import os - -reload_os = reload_os() -import os - -AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' - if 'nt' in sys.builtin_module_names: IS_WINDOWS = True DRIVE_LETTER_SEP = ':' else: IS_WINDOWS = False -def entry_point(executable, argv, path): +def entry_point(executable, argv, nanos): + # a substituted os if we are translated + global os + os = nanos + AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): @@ -209,8 +172,7 @@ if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable): pass # the path is already more than just an executable name else: - # path = os.getenv('PATH') - # we avoid getenv + path = os.getenv('PATH') if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) @@ -238,8 +200,6 @@ sys.path = newpath # found! break - reload_os.reload() # from now on use a fresh import - go_interactive = False run_command = False import_site = True @@ -441,6 +401,7 @@ if __name__ == '__main__': + import os import autopath # obscure! try removing the following line, see how it crashes, and # guess why... @@ -459,5 +420,5 @@ from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path - sys.exit(entry_point(sys.argv[0], sys.argv[1:], os.getenv("PATH"))) + sys.exit(entry_point(sys.argv[0], sys.argv[1:], os)) #sys.exit(entry_point('app_main.py', sys.argv[1:])) Added: pypy/branch/windows48396/pypy/translator/goal/nanos.py ============================================================================== --- (empty file) +++ pypy/branch/windows48396/pypy/translator/goal/nanos.py Fri Nov 23 19:05:18 2007 @@ -0,0 +1,53 @@ +""" +Not An os or Nano os :-) + +Implementation of a few methods needed for starting up +PyPy in app_main without importing os there. + +At startup time, app_main wants to find out about itself, +sys.executable, the path to its library etc. +This is done the easiest using os.py, but since this +is a python module, we have a recurrence problem. + +Importing it at compile time would work, partially, +but the way os is initialized will cause os.getenv +to malfunction, due to caching problems. + +The solution taken here implements a minimal os using +app-level code that is created at compile-time. Only +the few needed functions are implemented in a tiny os module +that contains a tiny path module. + +os.getenv got a direct implementation to overcome the caching +problem. + +Please adjust the applevel code below, if you need to support +more from os and os.path. +""" + +from pypy.interpreter.gateway import applevel, ObjSpace, W_Root, interp2app +import os + +app_os_path = applevel(r''' + from os.path import dirname, join, abspath, isfile, islink +''', filename=__file__) + +app_os = applevel(r''' + from os import sep, pathsep + try: + from os import readlink + except ImportError: + pass +''', filename=__file__) + +def getenv(space, w_name): + name = space.str_w(w_name) + return space.wrap(os.environ.get(name)) +getenv_w = interp2app(getenv, unwrap_spec=[ObjSpace, W_Root]) + +def setup_nanos(space): + w_os = space.wrap(app_os.buildmodule(space, 'os')) + w_os_path = space.wrap(app_os_path.buildmodule(space, 'path')) + space.setattr(w_os, space.wrap('path'), w_os_path) + space.setattr(w_os, space.wrap('getenv'), space.wrap(getenv_w)) + return w_os Modified: pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py Fri Nov 23 19:05:18 2007 @@ -8,6 +8,7 @@ from pypy.translator.goal.ann_override import PyPyAnnotatorPolicy from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE from pypy.tool.option import make_objspace +from pypy.translator.goal.nanos import setup_nanos thisdir = py.magic.autopath().dirpath() @@ -26,6 +27,7 @@ w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel')) w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish)) w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) + w_os = setup_nanos(space) def entry_point(argv): #debug("entry point starting") @@ -43,9 +45,7 @@ space.call_function(w_run_toplevel, w_call_startup_gateway) w_executable = space.wrap(argv[0]) w_argv = space.newlist([space.wrap(s) for s in argv[1:]]) - # we avoid using app-level environ due to the weird initialization of os - w_path = space.wrap(os.environ.get("PATH")) - w_exitcode = space.call_function(w_entry_point, w_executable, w_argv, w_path) + w_exitcode = space.call_function(w_entry_point, w_executable, w_argv, w_os) exitcode = space.int_w(w_exitcode) # try to pull it all in ## from pypy.interpreter import main, interactive, error Added: pypy/branch/windows48396/pypy/translator/goal/test2/test_nanos.py ============================================================================== --- (empty file) +++ pypy/branch/windows48396/pypy/translator/goal/test2/test_nanos.py Fri Nov 23 19:05:18 2007 @@ -0,0 +1,20 @@ +""" +Tests for the entry point of pypy-c, whether nanos.py is supplying +the needed names for app_main.py. +""" +import os + +from pypy.translator.goal import app_main +this_dir = os.path.dirname(app_main.__file__) + +from pypy.objspace.std import Space +from pypy.translator.goal.targetpypystandalone import create_entry_point + +def test_nanos(): + space = Space() + # manually imports app_main.py + filename = os.path.join(this_dir, 'app_main.py') + w_dict = space.newdict() + space.exec_(open(filename).read(), w_dict, w_dict) + entry_point = create_entry_point(space, w_dict) + entry_point(['', '-c', 'print 42']) From cfbolz at codespeak.net Fri Nov 23 19:14:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 19:14:37 +0100 (CET) Subject: [pypy-svn] r48997 - pypy/branch/njriley-trans Message-ID: <20071123181437.73E8C821C@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 19:14:37 2007 New Revision: 48997 Removed: pypy/branch/njriley-trans/ Log: nicholas riley tells me that he doesn't need this branch anymore From xoraxax at codespeak.net Fri Nov 23 19:19:36 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 19:19:36 +0100 (CET) Subject: [pypy-svn] r48998 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071123181936.7E76A8205@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 19:19:34 2007 New Revision: 48998 Modified: pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/test/test_genllvm.py Log: (xoraxax, algorithm by fijal): Fixed name mangling of LLVM backend. Before it generated two functions with the same name. Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Fri Nov 23 19:19:34 2007 @@ -1,22 +1,26 @@ from pypy.rpython.lltypesystem import lltype + class Node(object): __slots__ = "name".split() prefix = '%' nodename_count = {} + def mangle(self, name): + if name not in self.nodename_count: + result = name + self.nodename_count[name] = 1 + return result + else: + result = '%s_%d' % (name, self.nodename_count[name]) + self.nodename_count[name] += 1 + return self.mangle(result) + def make_name(self, name=''): " helper for creating names" name = self.prefix + name - if name in self.nodename_count: - postfix = '_%d' % self.nodename_count[name] - self.nodename_count[name] += 1 - else: - postfix = '' - self.nodename_count[name] = 1 - name += postfix - + name = self.mangle(name) if " " in name or "<" in name: name = '"%s"' % name Modified: pypy/dist/pypy/translator/llvm/test/test_genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_genllvm.py Fri Nov 23 19:19:34 2007 @@ -473,6 +473,31 @@ res = f(4) assert res == 4*(4+2) + +def func_with_dup_name(): + return 1 + +foo_func_with_dup_name = func_with_dup_name + +def test_dup_func(): + def func_with_dup_name(): + return 2 + def func_with_dup_name_1(): + return 3 + def call_func(x): + if x > 10: + return foo_func_with_dup_name() + elif x > 5: + return func_with_dup_name_1() + elif x > 0: + return func_with_dup_name() + return 0 + f = compile_function(call_func, [int]) + assert f(1) == 2 + assert f(6) == 3 + assert f(11) == 1 + + def test__del__(): from pypy.rpython.lltypesystem.lloperation import llop class State: From xoraxax at codespeak.net Fri Nov 23 19:20:03 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 19:20:03 +0100 (CET) Subject: [pypy-svn] r48999 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071123182003.26FFA8205@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 19:20:02 2007 New Revision: 48999 Modified: pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/test_lltype.py Log: Implement ullong shifts. Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Fri Nov 23 19:20:02 2007 @@ -49,6 +49,8 @@ 'llong_lshift': 'shl', 'llong_rshift': 'lshr', + 'ullong_lshift': 'shl', + 'ullong_rshift': 'lshr', } binary_operations = { Modified: pypy/dist/pypy/translator/llvm/test/test_lltype.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lltype.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lltype.py Fri Nov 23 19:20:02 2007 @@ -946,7 +946,6 @@ # XXX what does this do? def test_cast_primitive(self): - py.test.skip("ullong_lshift operation") def f(x): x = cast_primitive(UnsignedLongLong, x) x <<= 60 From xoraxax at codespeak.net Fri Nov 23 19:20:21 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 19:20:21 +0100 (CET) Subject: [pypy-svn] r49000 - pypy/dist/pypy/translator/llvm Message-ID: <20071123182021.DC5978223@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 19:20:21 2007 New Revision: 49000 Modified: pypy/dist/pypy/translator/llvm/gc.py Log: Add NOOP op_set_max_heap_size. Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Fri Nov 23 19:20:21 2007 @@ -128,6 +128,9 @@ cconv='ccc') + def op_set_max_heap_size(self, codewriter, opr): + pass + def op__collect(self, codewriter, opr): codewriter.call(opr.retref, opr.rettype, "@pypy_gc__collect", opr.argtypes, opr.argrefs) From xoraxax at codespeak.net Fri Nov 23 19:21:14 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 19:21:14 +0100 (CET) Subject: [pypy-svn] r49001 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071123182114.4E036821B@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 19:21:14 2007 New Revision: 49001 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: (xoraxax, fijal and rxe around): Wrap the os.W* macros in functions, add a test for them. In the end, it allows LLVM to use these. Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Fri Nov 23 19:21:14 2007 @@ -17,6 +17,8 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rlib import rposix +from pypy.tool.udir import udir + posix = __import__(os.name) if sys.platform.startswith('win'): @@ -52,6 +54,7 @@ ('tms_cstime', rffi.INT)]) else: _includes_ += ['sys/utime.h'] + _include_dirs_ = [str(udir)] SEEK_SET = platform.DefinedConstantInteger('SEEK_SET') SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') @@ -67,6 +70,25 @@ def __init__(self): self.configure(CConfig) + # we need an indirection via c functions to get macro calls working on llvm + decl_snippet = """ + %(ret_type)s pypy_%(name)s (int status); + """ + def_snippet = """ + %(ret_type)s pypy_%(name)s (int status) { + return %(name)s(status); + } + """ + decls = [] + defs = [] + for name in self.w_star: + data = {'ret_type': 'int', 'name': name} + decls.append(decl_snippet % data) + defs.append(def_snippet % data) + h_source = ['#include "sys/wait.h"'] + decls + ["#ifndef PYPY_NOT_MAIN_FILE"] + defs + ["#endif"] + h_file = udir.join("pypy_os_macros.h") + h_file.write("\n".join(h_source)) + # a simple, yet usefull factory def extdef_for_os_function_returning_int(self, name, **kwds): c_func = self.llexternal(name, [], rffi.INT, **kwds) @@ -1069,16 +1091,20 @@ # last 3 are returning int w_star_returning_int = dict.fromkeys(w_star[-3:]) + + def declare_new_w_star(self, name): """ stupid workaround for the python late-binding 'feature' """ + def fake(status): return int(getattr(os, name)(status)) fake.func_name = 'fake_' + name - os_c_func = self.llexternal(name, [lltype.Signed], - lltype.Signed, _callable=fake) + os_c_func = self.llexternal("pypy_" + name, [lltype.Signed], + lltype.Signed, includes=['pypy_os_macros.h'], + _callable=fake) if name in self.w_star_returning_int: def llimpl(status): @@ -1089,7 +1115,7 @@ return bool(os_c_func(status)) resulttype = bool llimpl.func_name = name + '_llimpl' - return extdef([int], resulttype, "ll_os."+name, + return extdef([int], resulttype, "ll_os." + name, llimpl=llimpl) for name in w_star: Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Fri Nov 23 19:21:14 2007 @@ -122,17 +122,6 @@ return os.getgid() assert self.interpret(f, []) == f() - def test_os_wstar(self): - from pypy.rpython.module.ll_os import RegisterOs - for name in RegisterOs.w_star: - if not hasattr(os, name): - continue - def fun(s): - return getattr(os, name)(s) - - for value in [0, 1, 127, 128, 255]: - res = self.interpret(fun, [value]) - assert res == fun(value) if hasattr(os, 'setuid'): def test_os_setuid(self): @@ -148,3 +137,18 @@ def test_fstat(self): py.test.skip("ootypesystem does not support os.fstat") + +def test_os_wstar(): + from pypy.rpython.module.ll_os import RegisterOs + from pypy.translator.c.test.test_genc import compile + for name in RegisterOs.w_star: + if not hasattr(os, name): + continue + def fun(s): + return getattr(os, name)(s) + + fun_c = compile(fun, [int]) + for value in [0, 1, 127, 128, 255]: + res = fun_c(value) + assert res == fun(value) + From rxe at codespeak.net Fri Nov 23 19:31:26 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 23 Nov 2007 19:31:26 +0100 (CET) Subject: [pypy-svn] r49002 - pypy/dist/pypy/translator/llvm Message-ID: <20071123183126.55024820F@code0.codespeak.net> Author: rxe Date: Fri Nov 23 19:31:25 2007 New Revision: 49002 Modified: pypy/dist/pypy/translator/llvm/gc.py Log: these are transformed now Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Fri Nov 23 19:31:25 2007 @@ -33,12 +33,6 @@ def op_free(self, codewriter, opr): raise Exception, 'GcPolicy should not be used directly' - def op_fetch_exception(self, codewriter, opr): - raise Exception, 'GcPolicy should not be used directly' - - def op_restore_exception(self, codewriter, opr): - raise Exception, 'GcPolicy should not be used directly' - def op_collect(self, codewriter, opr): raise Exception, 'GcPolicy should not be used directly' From rxe at codespeak.net Fri Nov 23 19:31:54 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 23 Nov 2007 19:31:54 +0100 (CET) Subject: [pypy-svn] r49003 - pypy/dist/pypy/translator/llvm Message-ID: <20071123183154.B8355820B@code0.codespeak.net> Author: rxe Date: Fri Nov 23 19:31:54 2007 New Revision: 49003 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/genllvm.py Log: cleanup predeclare externs Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Fri Nov 23 19:31:54 2007 @@ -7,24 +7,12 @@ from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.buildllvm import llvm_gcc_version - from pypy.tool.udir import udir -def predeclare_stuff(c_db): - exctransformer = c_db.exctransformer - yield ('_rpyexc_occured_ptr', exctransformer._rpyexc_occured_ptr.value) - yield ('rpyexc_fetch_type_ptr', exctransformer.rpyexc_fetch_type_ptr.value) - yield ('rpyexc_clear_ptr', exctransformer.rpyexc_clear_ptr.value) - support_functions = [ "@LLVM_RPython_StartupCode", ] -skip_lines = [ - "%RPyString = type opaque", - "__main", - ] - def get_module_file(name): return os.path.join(get_llvm_cpath(), name) @@ -63,12 +51,6 @@ # get rid of any of the structs that llvm-gcc introduces to struct types line = line.replace("%struct.", "%") - # XXX slowwwwwww - for x in skip_lines: - if line.find(x) >= 0: - line = '' - break - # strip comments comment = line.find(';') if comment >= 0: @@ -117,43 +99,6 @@ ll_lines2.append("declare ccc void @abort()") return'\n'.join(ll_lines2) -def find_list_of_str(rtyper): - from pypy.rpython.lltypesystem import rlist - for r in rtyper.reprs.itervalues(): - if isinstance(r, rlist.ListRepr) and r.item_repr is rstr.string_repr: - return r.lowleveltype.TO - return None - -def setup_externs(c_db, db): - # hacks to make predeclare_all work - - rtyper = db.translator.rtyper - decls = list(predeclare_stuff(c_db)) - - for c_name, obj in decls: - if isinstance(obj, lltype.LowLevelType): - db.prepare_type(obj) - elif isinstance(obj, FunctionGraph): - funcptr = rtyper.getcallable(obj) - c = inputconst(lltype.typeOf(funcptr), funcptr) - db.prepare_arg_value(c) - elif isinstance(lltype.typeOf(obj), lltype.Ptr): - db.prepare_constant(lltype.typeOf(obj), obj) - elif type(c_name) is str and type(obj) is int: - pass #define c_name obj - else: - assert False, "unhandled predeclare %s %s %s" % (c_name, type(obj), obj) - - def annotatehelper(func, *argtypes): - graph = db.translator.rtyper.annotate_helper(func, argtypes) - fptr = rtyper.getcallable(graph) - c = inputconst(lltype.typeOf(fptr), fptr) - db.prepare_arg_value(c) - decls.append(("ll_" + func.func_name, graph)) - return graph.name - - return decls - def get_c_cpath(): from pypy.translator.c import genc return os.path.dirname(genc.__file__) @@ -175,7 +120,7 @@ includestr += "-I %s " % ii return includestr -def generate_llfile(db, extern_decls, entrynode, c_includes, c_sources, standalone, default_cconv): +def generate_llfile(db, entrynode, c_includes, c_sources, standalone, default_cconv): ccode = [] function_names = [] @@ -190,32 +135,10 @@ predeclarefn("__ENTRY_POINT__", entrynode.get_ref()) ccode.append('#define ENTRY_POINT_DEFINED 1\n\n') - for c_name, obj in extern_decls: - if isinstance(obj, lltype.LowLevelType): - s = "#define %s struct %s\n%s;\n" % (c_name, c_name, c_name) - ccode.append(s) - - elif isinstance(obj, FunctionGraph): - funcptr = db.translator.rtyper.getcallable(obj) - c = inputconst(lltype.typeOf(funcptr), funcptr) - predeclarefn(c_name, db.repr_arg(c)) - - elif isinstance(lltype.typeOf(obj), lltype.Ptr): - pass - - elif type(c_name) is str and type(obj) is int: - ccode.append("#define\t%s\t%d\n" % (c_name, obj)) - - else: - assert False, "unhandled extern_decls %s %s %s" % (c_name, type(obj), obj) - - - # include this early to get constants and macros for any further includes - ccode.append('#include \n') - # ask gcpolicy for any code needed ccode.append('%s\n' % db.gcpolicy.genextern_code()) + # ask rffi for includes/source for c_include in c_includes: ccode.append('#include <%s>\n' % c_include) Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Fri Nov 23 19:31:54 2007 @@ -12,7 +12,7 @@ from pypy.translator.llvm import extfuncnode from pypy.translator.llvm.module.support import extfunctions from pypy.translator.llvm.node import Node -from pypy.translator.llvm.externs2ll import setup_externs, generate_llfile +from pypy.translator.llvm.externs2ll import generate_llfile from pypy.translator.llvm.gc import GcPolicy from pypy.translator.llvm.log import log from pypy.rlib.nonconst import NonConstant @@ -133,7 +133,8 @@ self._checkpoint('setup_all all nodes') # set up externs nodes - self.extern_decls = setup_externs(c_db, self.db) + self.setup_externs(c_db, self.db) + self.translator.rtyper.specialize_more_blocks() self.db.setup_all() self._checkpoint('setup_all externs') @@ -149,10 +150,17 @@ # create ll file from c code self.generate_ll_externs(codewriter) - self._checkpoint('setup_externs') return codewriter + def setup_externs(self, c_db, db): + # XXX + exctransformer = c_db.exctransformer + for obj in [exctransformer._rpyexc_occured_ptr.value, + exctransformer.rpyexc_fetch_type_ptr.value, + exctransformer.rpyexc_clear_ptr.value]: + db.prepare_constant(lltype.typeOf(obj), obj) + def get_entry_point(self, func): assert func is not None self.entrypoint = func @@ -188,7 +196,6 @@ c_sources[source] = True self.llcode = generate_llfile(self.db, - self.extern_decls, self.entrynode, c_includes, c_sources, From arigo at codespeak.net Fri Nov 23 19:41:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Nov 2007 19:41:24 +0100 (CET) Subject: [pypy-svn] r49004 - in pypy/dist/pypy/translator/goal: . test2 Message-ID: <20071123184124.9E89D820F@code0.codespeak.net> Author: arigo Date: Fri Nov 23 19:41:23 2007 New Revision: 49004 Added: pypy/dist/pypy/translator/goal/app_main.py - copied unchanged from r49003, pypy/branch/windows48396/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/nanos.py - copied unchanged from r49003, pypy/branch/windows48396/pypy/translator/goal/nanos.py pypy/dist/pypy/translator/goal/targetpypystandalone.py - copied unchanged from r49003, pypy/branch/windows48396/pypy/translator/goal/targetpypystandalone.py pypy/dist/pypy/translator/goal/test2/ - copied from r49003, pypy/branch/windows48396/pypy/translator/goal/test2/ Log: merged the changes to app_main back into dist From xoraxax at codespeak.net Fri Nov 23 19:57:30 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 23 Nov 2007 19:57:30 +0100 (CET) Subject: [pypy-svn] r49005 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20071123185730.80DD5820F@code0.codespeak.net> Author: xoraxax Date: Fri Nov 23 19:57:30 2007 New Revision: 49005 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: Fix naming of the macro wrappers. Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Fri Nov 23 19:57:30 2007 @@ -72,10 +72,10 @@ # we need an indirection via c functions to get macro calls working on llvm decl_snippet = """ - %(ret_type)s pypy_%(name)s (int status); + %(ret_type)s pypy_macro_wrapper_%(name)s (int status); """ def_snippet = """ - %(ret_type)s pypy_%(name)s (int status) { + %(ret_type)s pypy_macro_wrapper_%(name)s (int status) { return %(name)s(status); } """ @@ -1102,7 +1102,7 @@ return int(getattr(os, name)(status)) fake.func_name = 'fake_' + name - os_c_func = self.llexternal("pypy_" + name, [lltype.Signed], + os_c_func = self.llexternal("pypy_macro_wrapper_" + name, [lltype.Signed], lltype.Signed, includes=['pypy_os_macros.h'], _callable=fake) Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Fri Nov 23 19:57:30 2007 @@ -1,6 +1,8 @@ import py from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.tool.udir import udir +from pypy.translator.llvm.test.runtest import compile_function as compile_llvm +from pypy.translator.c.test.test_genc import compile import os exec 'import %s as posix' % os.name @@ -138,9 +140,8 @@ py.test.skip("ootypesystem does not support os.fstat") -def test_os_wstar(): +def os_wstar_tester(compile): from pypy.rpython.module.ll_os import RegisterOs - from pypy.translator.c.test.test_genc import compile for name in RegisterOs.w_star: if not hasattr(os, name): continue @@ -152,3 +153,6 @@ res = fun_c(value) assert res == fun(value) +def test_os_wstar(): + yield os_wstar_tester, compile_llvm + yield os_wstar_tester, compile From cfbolz at codespeak.net Fri Nov 23 20:28:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 20:28:46 +0100 (CET) Subject: [pypy-svn] r49006 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071123192846.C0D878213@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 20:28:45 2007 New Revision: 49006 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py Log: use_true_methods is never true any more. kill it. Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py Fri Nov 23 20:28:45 2007 @@ -43,7 +43,6 @@ # leaving the import support intact, doesn't hurt. self.name_for_meth = {} # get nicer wrapper names self.is_method = {} - self.use_true_methods = False # may be overridden def nameof(self, obj, debug=None): if debug: @@ -586,13 +585,8 @@ if ann.binding(graph.getargs()[0]).classdef is not clsdef: value = new_method_graph(graph, clsdef, fname, self.translator) self.name_for_meth[value] = fname - if self.use_true_methods: - self.is_method[value] = True elif isinstance(value, property): fget, fset, fdel, doc = value.fget, value.fset, value.fdel, value.__doc__ - for f in fget, fset, fdel: - if f and self.use_true_methods: - self.is_method[f] = True stuff = [self.nameof(x) for x in fget, fset, fdel, doc] yield '%s.%s = property(%s, %s, %s, %s)' % ((name, key) + tuple(stuff)) @@ -625,9 +619,6 @@ def nameof_property(self, p): fget, fset, fdel, doc = p.fget, p.fset, p.fdel, p.__doc__ - for f in fget, fset, fdel: - if f and self.use_true_methods: - self.is_method[f] = True stuff = [self.nameof(x) for x in fget, fset, fdel, doc] name = self.uniquename('gprop') expr = 'property(%s, %s, %s, %s)' % (tuple(stuff)) From cfbolz at codespeak.net Fri Nov 23 20:30:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 20:30:40 +0100 (CET) Subject: [pypy-svn] r49007 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071123193040.850598213@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 20:30:40 2007 New Revision: 49007 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py Log: the is_method dictionary is also always empty now, kill it too. Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py Fri Nov 23 20:30:40 2007 @@ -42,7 +42,6 @@ self.import_hints = {} # I don't seem to need it any longer. # leaving the import support intact, doesn't hurt. self.name_for_meth = {} # get nicer wrapper names - self.is_method = {} def nameof(self, obj, debug=None): if debug: @@ -221,8 +220,7 @@ try: fwrapper = gen_wrapper(func, self.translator, - newname=self.name_for_meth.get(func, func.__name__), - as_method=func in self.is_method) + newname=self.name_for_meth.get(func, func.__name__)) except NoStandardGraph: return self.skipped_function(func) pycfunctionobj = self.uniquename('gfunc_' + func.__name__) @@ -611,8 +609,7 @@ def nameof_graph(self, g): newname=self.name_for_meth.get(g, g.func.__name__) - fwrapper = gen_wrapper(g, self.translator, newname=newname, - as_method=g in self.is_method) + fwrapper = gen_wrapper(g, self.translator, newname=newname) pycfunctionobj = self.uniquename('gfunc_' + newname) self.wrappers[pycfunctionobj] = g.func.__name__, self.db.get(fwrapper), g.func.__doc__ return pycfunctionobj From cfbolz at codespeak.net Fri Nov 23 20:32:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 20:32:55 +0100 (CET) Subject: [pypy-svn] r49008 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071123193255.2A75C8218@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 20:32:54 2007 New Revision: 49008 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Log: as_method is always False now Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Fri Nov 23 20:32:54 2007 @@ -13,7 +13,7 @@ ALWAYS_INLINE = False -def gen_wrapper(func, translator, newname=None, as_method=False): +def gen_wrapper(func, translator, newname=None): """generate a wrapper function for 'func' that can be put in a PyCFunction object. The wrapper has signature @@ -63,9 +63,6 @@ varguments = [] varnames = func.func_code.co_varnames func_defaults = func.func_defaults or () - if as_method: - nb_positional_args -= 1 - varnames = varnames[1:] for i in range(nb_positional_args): # "argument_i = decode_arg(fname, i, name, vargs, vkwds)" or # "argument_i = decode_arg_def(fname, i, name, vargs, vkwds, default)" @@ -105,10 +102,6 @@ # use the rtyper to produce the conversions inputargs = f._obj.graph.getargs() - if as_method: - varguments.insert(0, vself) - vlist = [vfname, vself] - newops.genop('check_self_nonzero', vlist, resulttype=Signed) for i in range(len(varguments)): if FUNCTYPE.ARGS[i] != PyObjPtr: # "argument_i = type_conversion_operations(argument_i)" From cfbolz at codespeak.net Fri Nov 23 20:36:35 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 23 Nov 2007 20:36:35 +0100 (CET) Subject: [pypy-svn] r49009 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071123193635.C8EFD815A@code0.codespeak.net> Author: cfbolz Date: Fri Nov 23 20:36:35 2007 New Revision: 49009 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Log: we always have an annotator now Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Fri Nov 23 20:36:35 2007 @@ -26,21 +26,15 @@ # get the fully typed low-level pointer to the function, if available do_inline = ALWAYS_INLINE - if translator.annotator is None: - # get the graph from the translator, "push it back" so that it's - # still available for further buildflowgraph() calls - graph = translator.buildflowgraph(func) - translator._prebuilt_graphs[func] = graph + if isinstance(func, FunctionGraph): + graph = func + func = graph.func + # in this case we want to inline for sure, because we + # created this extra graph with a single call-site. + do_inline = True else: - if isinstance(func, FunctionGraph): - graph = func - func = graph.func - # in this case we want to inline for sure, because we - # created this extra graph with a single call-site. - do_inline = True - else: - bk = translator.annotator.bookkeeper - graph = bk.getdesc(func).getuniquegraph() + bk = translator.annotator.bookkeeper + graph = bk.getdesc(func).getuniquegraph() f = getfunctionptr(graph) FUNCTYPE = typeOf(f).TO From tismer at codespeak.net Sat Nov 24 02:37:33 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 24 Nov 2007 02:37:33 +0100 (CET) Subject: [pypy-svn] r49011 - pypy/branch/windows48396/pypy/lib Message-ID: <20071124013733.649E1822B@code0.codespeak.net> Author: tismer Date: Sat Nov 24 02:37:31 2007 New Revision: 49011 Modified: pypy/branch/windows48396/pypy/lib/_exceptions.py Log: forgot an omission to _exceptions Modified: pypy/branch/windows48396/pypy/lib/_exceptions.py ============================================================================== --- pypy/branch/windows48396/pypy/lib/_exceptions.py (original) +++ pypy/branch/windows48396/pypy/lib/_exceptions.py Sat Nov 24 02:37:31 2007 @@ -210,6 +210,12 @@ class OSError(EnvironmentError): """OS system call failed.""" +try: + WindowsError + class WindowsError(OSError): + """MS-Windows OS system call failed.""" +except NameError:pass + class DeprecationWarning(Warning): """Base class for warnings about deprecated features.""" From rxe at codespeak.net Sat Nov 24 03:02:25 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 24 Nov 2007 03:02:25 +0100 (CET) Subject: [pypy-svn] r49012 - pypy/dist/pypy/translator/llvm Message-ID: <20071124020225.135E38228@code0.codespeak.net> Author: rxe Date: Sat Nov 24 03:02:25 2007 New Revision: 49012 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/node.py Log: an attempt to support to include dirs .... Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sat Nov 24 03:02:25 2007 @@ -16,7 +16,7 @@ def get_module_file(name): return os.path.join(get_llvm_cpath(), name) -def get_ll(ccode, function_names, default_cconv): +def get_ll(ccode, function_names, default_cconv, c_include_dirs): function_names += support_functions filename = str(udir.join("ccode.c")) f = open(filename, "w") @@ -24,17 +24,14 @@ f.close() plain = filename[:-2] - includes = get_incdirs() - + includes = get_incdirs(c_include_dirs) if llvm_gcc_version() < 4.0: emit_llvm = '' else: emit_llvm = '-emit-llvm -O0' - # XXX localize this - include_path = '-I/sw/include' - cmd = "llvm-gcc %s %s %s -S %s.c -o %s.ll 2>&1" % ( - include_path, includes, emit_llvm, plain, plain) + cmd = "llvm-gcc %s %s -S %s.c -o %s.ll 2>&1" % ( + includes, emit_llvm, plain, plain) if os.system(cmd) != 0: raise Exception("Failed to run '%s'" % cmd) @@ -106,10 +103,14 @@ def get_llvm_cpath(): return os.path.join(os.path.dirname(__file__), "module") -def get_incdirs(): +def get_incdirs(c_include_dirs): + + c_include_dirs import distutils.sysconfig - includes = (distutils.sysconfig.EXEC_PREFIX + "/include", + + includes = tuple(c_include_dirs) + ("/sw/include", + distutils.sysconfig.EXEC_PREFIX + "/include", distutils.sysconfig.EXEC_PREFIX + "/include/gc", distutils.sysconfig.get_python_inc(), get_c_cpath(), @@ -120,7 +121,7 @@ includestr += "-I %s " % ii return includestr -def generate_llfile(db, entrynode, c_includes, c_sources, standalone, default_cconv): +def generate_llfile(db, entrynode, c_include_dirs, c_includes, c_sources, standalone, default_cconv): ccode = [] function_names = [] @@ -149,5 +150,5 @@ # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) - llcode = get_ll("".join(ccode), function_names, default_cconv) + llcode = get_ll("".join(ccode), function_names, default_cconv, c_include_dirs) return llcode Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Sat Nov 24 03:02:25 2007 @@ -10,16 +10,20 @@ def external_c_source(self): # return a list of unique includes and sources in C + c_include_dirs = [] c_includes = [] c_sources = [] + if hasattr(self.value, 'include_dirs'): + c_include_dirs = list(self.value.include_dirs) + if hasattr(self.value, 'includes'): c_includes = list(self.value.includes) if hasattr(self.value, 'sources'): c_sources = list(self.value.sources) - return c_includes, c_sources + return c_include_dirs, c_includes, c_sources def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sat Nov 24 03:02:25 2007 @@ -185,11 +185,14 @@ return c.value._obj def generate_ll_externs(self, codewriter): + c_include_dirs = {} c_includes = {} c_sources = {} for node in self.db.getnodes(): - includes, sources = node.external_c_source() + include_dirs, includes, sources = node.external_c_source() + for incdirin in include_dirs: + c_include_dirs[incdir] = True for include in includes: c_includes[include] = True for source in sources: @@ -197,6 +200,7 @@ self.llcode = generate_llfile(self.db, self.entrynode, + c_include_dirs, c_includes, c_sources, self.standalone, Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Sat Nov 24 03:02:25 2007 @@ -34,7 +34,7 @@ def external_c_source(self): " return a list of unique includes and sources in C " - return [], [] + return [], [], [] def writesetupcode(self, codewriter): " pre entry-point setup " From tismer at codespeak.net Sat Nov 24 03:05:54 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 24 Nov 2007 03:05:54 +0100 (CET) Subject: [pypy-svn] r49013 - pypy/dist/pypy/lib Message-ID: <20071124020554.722418228@code0.codespeak.net> Author: tismer Date: Sat Nov 24 03:05:54 2007 New Revision: 49013 Modified: pypy/dist/pypy/lib/_exceptions.py Log: omitted something that was needed, too Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Sat Nov 24 03:05:54 2007 @@ -210,6 +210,12 @@ class OSError(EnvironmentError): """OS system call failed.""" +try: + WindowsError + class WindowsError(OSError): + """MS-Windows OS system call failed.""" +except NameError:pass + class DeprecationWarning(Warning): """Base class for warnings about deprecated features.""" @@ -438,3 +444,18 @@ self.encoding, self.start, self.end - 1, self.reason) + +#-- Logic object space specific stuff +#XXX conditionalize me on '-o logic' + +class LogicError(Exception): pass + +class UnificationError(LogicError): pass +class RebindingError(UnificationError): pass +class FutureBindingError(LogicError): pass + +class AllBlockedError(LogicError): pass + +# constraints + +class ConsistencyError(LogicError): pass From cfbolz at codespeak.net Sat Nov 24 11:38:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 11:38:52 +0100 (CET) Subject: [pypy-svn] r49020 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . test Message-ID: <20071124103852.D72F981F8@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 11:38:49 2007 New Revision: 49020 Added: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py (contents, props changed) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Log: (pedronis, cfbolz): create a new wrapper generator that does not do all the argument parsing of the old one. Added: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py ============================================================================== --- (empty file) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py Sat Nov 24 11:38:49 2007 @@ -0,0 +1,56 @@ +import py +from pypy import conftest +from pypy.translator.translator import TranslationContext +from pypy.translator.c.wrapper import new_wrapper +from pypy.rpython.rmodel import PyObjPtr +from pypy.rpython.llinterp import LLInterpreter, LLException, log +from pypy.rpython.lltypesystem import lltype + + +class TestMakeWrapper: + + def getgraph(self, func, argtypes=None): + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.gc = "ref" + config.translation.simplifying = True + t = TranslationContext(config=config) + if argtypes is None: + argtypes = [] + a = t.buildannotator() + a.build_types(func, argtypes) + a.simplify() + t.buildrtyper().specialize() + wrapperptr = new_wrapper(func, t) + wrappergraph = wrapperptr._obj.graph + for inputarg in wrappergraph.startblock.inputargs: + assert inputarg.concretetype == PyObjPtr + assert wrappergraph.getreturnvar().concretetype == PyObjPtr + return t.graphs[0], wrappergraph, t + + def interpret(self, t, graph, *args): + interp = LLInterpreter(t.rtyper) + result = interp.eval_graph(graph, [lltype.pyobjectptr(arg) + for arg in args]) + return result._obj.value + + def test_simple(self): + def f(x): + return x * 3 + graph, wrappergraph, t = self.getgraph(f, [int]) + res = self.interpret(t, wrappergraph, 3) + assert res == 9 + + def test_manyargs(self): + def f(x, y, z): + return x * y + z + graph, wrappergraph, t = self.getgraph(f, [int, int, int]) + res = self.interpret(t, wrappergraph, 3, 4, 5) + assert res == 3 * 4 + 5 + + def test_returnnone(self): + def f(): + pass + graph, wrappergraph, t = self.getgraph(f) + res = self.interpret(t, wrappergraph) + assert res is None Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Sat Nov 24 11:38:49 2007 @@ -195,3 +195,74 @@ if ALWAYS_INLINE: simple_inline_function(translator, graph, newgraph) return newgraph + + + +def new_wrapper(func, translator, newname=None): + # The basic idea is to produce a flow graph from scratch, using the + # help of the rtyper for the conversion of the arguments after they + # have been decoded. + + bk = translator.annotator.bookkeeper + graph = bk.getdesc(func).getuniquegraph() + + f = getfunctionptr(graph) + FUNCTYPE = typeOf(f).TO + + newops = LowLevelOpList(translator.rtyper) + + varguments = [] + for var in graph.startblock.inputargs: + v = Variable(var) + v.concretetype = PyObjPtr + varguments.append(v) + + wrapper_inputargs = varguments[:] + # use the rtyper to produce the conversions + inputargs = f._obj.graph.getargs() + for i in range(len(varguments)): + if FUNCTYPE.ARGS[i] != PyObjPtr: + rtyper = translator.rtyper + r_arg = rtyper.bindingrepr(inputargs[i]) + # give the rtyper a chance to know which function we are wrapping + rtyper.set_wrapper_context(func) + varguments[i] = newops.convertvar(varguments[i], + r_from = pyobj_repr, + r_to = r_arg) + rtyper.set_wrapper_context(None) + + vlist = [inputconst(typeOf(f), f)] + varguments + vresult = newops.genop('direct_call', vlist, resulttype=FUNCTYPE.RESULT) + + if FUNCTYPE.RESULT != PyObjPtr: + # convert "result" back to a PyObject + rtyper = translator.rtyper + assert rtyper is not None, ( + "needs the rtyper to perform function result conversions") + r_result = rtyper.bindingrepr(f._obj.graph.getreturnvar()) + vresult = newops.convertvar(vresult, + r_from = r_result, + r_to = pyobj_repr) + + # "return result" + block = Block(wrapper_inputargs) + wgraph = FunctionGraph('pyfn_' + (newname or func.func_name), block) + translator.update_call_graph(wgraph, graph, object()) + translator.graphs.append(wgraph) + block.operations[:] = newops + block.closeblock(Link([vresult], wgraph.returnblock)) + wgraph.getreturnvar().concretetype = PyObjPtr + checkgraph(wgraph) + + # the above convertvar()s may have created and annotated new helpers + # that need to be specialized now + translator.rtyper.specialize_more_blocks() + + return functionptr(FuncType([PyObjPtr, + PyObjPtr, + PyObjPtr], + PyObjPtr), + wgraph.name, + graph = wgraph, + exception_policy = "CPython") + From cfbolz at codespeak.net Sat Nov 24 11:42:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 11:42:27 +0100 (CET) Subject: [pypy-svn] r49021 - pypy/dist/pypy/lib Message-ID: <20071124104227.2AE848204@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 11:42:26 2007 New Revision: 49021 Modified: pypy/dist/pypy/lib/_exceptions.py Log: we don't have a logic object space any more Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Sat Nov 24 11:42:26 2007 @@ -445,17 +445,3 @@ -#-- Logic object space specific stuff -#XXX conditionalize me on '-o logic' - -class LogicError(Exception): pass - -class UnificationError(LogicError): pass -class RebindingError(UnificationError): pass -class FutureBindingError(LogicError): pass - -class AllBlockedError(LogicError): pass - -# constraints - -class ConsistencyError(LogicError): pass From pedronis at codespeak.net Sat Nov 24 11:59:02 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 24 Nov 2007 11:59:02 +0100 (CET) Subject: [pypy-svn] r49022 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . test Message-ID: <20071124105902.CA1608177@code0.codespeak.net> Author: pedronis Date: Sat Nov 24 11:59:02 2007 New Revision: 49022 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Log: (cfbolz, pedronis) oops Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py Sat Nov 24 11:59:02 2007 @@ -3,7 +3,7 @@ from pypy.translator.translator import TranslationContext from pypy.translator.c.wrapper import new_wrapper from pypy.rpython.rmodel import PyObjPtr -from pypy.rpython.llinterp import LLInterpreter, LLException, log +from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype @@ -23,7 +23,11 @@ t.buildrtyper().specialize() wrapperptr = new_wrapper(func, t) wrappergraph = wrapperptr._obj.graph - for inputarg in wrappergraph.startblock.inputargs: + F = lltype.typeOf(wrapperptr).TO + assert F.ARGS == (PyObjPtr,) * len(wrappergraph.getargs()) + assert F.RESULT == PyObjPtr + + for inputarg in wrappergraph.getargs(): assert inputarg.concretetype == PyObjPtr assert wrappergraph.getreturnvar().concretetype == PyObjPtr return t.graphs[0], wrappergraph, t Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Sat Nov 24 11:59:02 2007 @@ -258,9 +258,7 @@ # that need to be specialized now translator.rtyper.specialize_more_blocks() - return functionptr(FuncType([PyObjPtr, - PyObjPtr, - PyObjPtr], + return functionptr(FuncType([PyObjPtr] * len(wrapper_inputargs), PyObjPtr), wgraph.name, graph = wgraph, From cfbolz at codespeak.net Sat Nov 24 12:53:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 12:53:00 +0100 (CET) Subject: [pypy-svn] r49024 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071124115300.753F78196@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 12:52:59 2007 New Revision: 49024 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sat Nov 24 12:52:59 2007 @@ -21,14 +21,14 @@ Translation toolchain +++++++++++++++++++++ - - refactor rffi header, sources, includes management IN-PROGRESS - (maciek, alexander) + - refactor rffi header, sources, includes management IN-PROGRESS (maciek, + alexander, armin) - py-lib 0.9.1 release? (holger, maciek) - - tried to translate PyPy with LLVM IN-PROGRESS (found a bug in LLVM opt) + - tried to translate PyPy with LLVM DONE - - some more work on cleaning up the startup process IN-PROGRESS (christian) + - some more work on cleaning up the startup process DONE - fix windows compilation: rffi-platform checks have problem, which are being approached now @@ -36,7 +36,7 @@ - find out why our memcpy behaves so erratically GCC behaves very erratically, so we won't work on this for now - - finish rctypes removal + - finish rctypes removal DONE on a branch - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) @@ -47,7 +47,7 @@ - cleanup pyobj.py and rclass stuff and remove the ability to make cpython extensions, just dll - update gencapicall impl (cfbolz, samuele) IN-PROGRESS + update gencapicall impl (cfbolz, samuele) MORE-PROGRESS - clean up the tangle of including headers in the C backend @@ -58,6 +58,7 @@ can have more than one translator/annotator around (with the timeshifter) + - llvm profiling (richard) Interpreter +++++++++++ @@ -71,11 +72,13 @@ - review the things implemented at applevel whether they are performance-critical IN-PROGRESS: moved super, classmethod, statimethod to interplevel, some dict methods + to be done: property, exceptions? - remove prebuilt costate vars (christian, armin?) - look at the new failing test in test_descr.py + - fix __future__ imports IN-PROGRESS (arre) Modules +++++++ From pedronis at codespeak.net Sat Nov 24 13:02:17 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 24 Nov 2007 13:02:17 +0100 (CET) Subject: [pypy-svn] r49025 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . src test Message-ID: <20071124120217.460D681E0@code0.codespeak.net> Author: pedronis Date: Sat Nov 24 13:02:16 2007 New Revision: 49025 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/g_include.h pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py Log: (cfbolz, pedronis) WIP: not making c extension anymore Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Sat Nov 24 13:02:16 2007 @@ -4,6 +4,7 @@ from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines +from pypy.translator.c.wrapper import new_wrapper from pypy.translator.gensupp import uniquemodulename, NameManager from pypy.translator.tool.cbuild import so_ext from pypy.translator.tool.cbuild import compile_c_module @@ -61,6 +62,7 @@ stacklesstransformer=stacklesstransformer, thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) + self.db = db # pass extra options into pyobjmaker if pyobj_options: for key, value in pyobj_options.items(): @@ -171,12 +173,13 @@ class CExtModuleBuilder(CBuilder): standalone = False - _module = None + _module = None + _wrapper = None - def getentrypointptr(self, obj=None): - if obj is None: - obj = self.entrypoint - return lltype.pyobjectptr(obj) + def getentrypointptr(self): # xxx + if self._wrapper is None: + self._wrapper = new_wrapper(self.entrypoint, self.translator) + return self._wrapper def compile(self): assert self.c_source_filename @@ -190,7 +193,12 @@ self._compiled = True def _make_wrapper_module(self): - modfile = self.c_source_filename.new(ext=".py") + fname = 'wrap_' + self.c_source_filename.purebasename + modfile = self.c_source_filename.new(purebasename=fname, ext=".py") + + entrypoint_ptr = self.getentrypointptr() + wrapped_entrypoint_c_name = self.db.get(entrypoint_ptr) + CODE = """ import ctypes @@ -198,20 +206,25 @@ _entry_point = getattr(_lib, "%(c_entrypoint_name)s") _entry_point.restype = ctypes.py_object -_entry_point.argtypes = 3*(ctypes.py_object,) +_entry_point.argtypes = %(nargs)d*(ctypes.py_object,) -def %(entrypoint_name)s(*args, **kwds): - return _entry_point(None, args, kwds) +def %(entrypoint_name)s(*args): + return _entry_point(*args) -_malloc_counters = _lib.malloc_counters -_malloc_counters.restype = ctypes.py_object -_malloc_counters.argtypes = 2*(ctypes.py_object,) +try: + _malloc_counters = _lib.malloc_counters +except AttributeError: + pass +else: + _malloc_counters.restype = ctypes.py_object + _malloc_counters.argtypes = 2*(ctypes.py_object,) -def malloc_counters(): - return _malloc_counters(None, None) + def malloc_counters(): + return _malloc_counters(None, None) """ % {'so_name': self.c_source_filename.new(ext=so_ext), 'entrypoint_name': self.entrypoint_name, - 'c_entrypoint_name': self.c_entrypoint_name} + 'c_entrypoint_name': wrapped_entrypoint_c_name, + 'nargs': len(lltype.typeOf(entrypoint_ptr).TO.ARGS)} modfile.write(CODE) self._module_path = modfile @@ -238,7 +251,7 @@ return self._module.malloc_counters def cleanup(self): - assert self._module + #assert self._module if isinstance(self._module, isolate.Isolate): isolate.close_isolate(self._module) @@ -749,78 +762,78 @@ sg.set_strategy(targetdir) sg.gen_readable_parts_of_source(f) - # - # PyObject support (strange) code - # - pyobjmaker = database.pyobjmaker - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Table of global PyObjects ***/' - print >> f - print >> f, 'static globalobjectdef_t globalobjectdefs[] = {' - for node in database.containerlist: - if isinstance(node, PyObjectNode): - for target in node.where_to_copy_me: - print >> f, '\t{%s, "%s"},' % (target, node.exported_name) - print >> f, '\t{ NULL, NULL }\t/* Sentinel */' - print >> f, '};' - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Table of functions ***/' - print >> f - print >> f, 'static globalfunctiondef_t globalfunctiondefs[] = {' - wrappers = pyobjmaker.wrappers.items() - wrappers.sort() - for globalobject_name, (base_name, wrapper_name, func_doc) in wrappers: - print >> f, ('\t{&%s, "%s", {"%s", (PyCFunction)%s, ' - 'METH_VARARGS|METH_KEYWORDS, %s}},' % ( - globalobject_name, - globalobject_name, - base_name, - wrapper_name, - func_doc and c_string_constant(func_doc) or 'NULL')) - print >> f, '\t{ NULL }\t/* Sentinel */' - print >> f, '};' - print >> f, 'static globalfunctiondef_t *globalfunctiondefsptr = &globalfunctiondefs[0];' - print >> f - print >> f, '/***********************************************************/' - print >> f, '/*** Frozen Python bytecode: the initialization code ***/' - print >> f - print >> f, 'static char *frozen_initcode[] = {"\\' - bytecode, originalsource = database.pyobjmaker.getfrozenbytecode() - g = targetdir.join('frozen.py').open('w') - g.write(originalsource) - g.close() - def char_repr(c): - if c in '\\"': return '\\' + c - if ' ' <= c < '\x7F': return c - return '\\%03o' % ord(c) - for i in range(0, len(bytecode), 32): - print >> f, ''.join([char_repr(c) for c in bytecode[i:i+32]])+'\\' - if (i+32) % 1024 == 0: - print >> f, '", "\\' - print >> f, '"};' - print >> f, "#define FROZEN_INITCODE_SIZE %d" % len(bytecode) - print >> f - - # - # Module initialization function - # - print >> f, '/***********************************************************/' - print >> f, '/*** Module initialization function ***/' - print >> f - gen_startupcode(f, database) - print >> f - print >> f, 'MODULE_INITFUNC(%s)' % modulename - print >> f, '{' - print >> f, '\tSETUP_MODULE(%s);' % modulename - for publicname, pyobjptr in exports.items(): - # some fishing needed to find the name of the obj - pyobjnode = database.containernodes[pyobjptr._obj] - print >> f, '\tPyModule_AddObject(m, "%s", %s);' % (publicname, - pyobjnode.name) - print >> f, '\tcall_postsetup(m);' - print >> f, '}' +## # +## # PyObject support (strange) code +## # +## pyobjmaker = database.pyobjmaker +## print >> f +## print >> f, '/***********************************************************/' +## print >> f, '/*** Table of global PyObjects ***/' +## print >> f +## print >> f, 'static globalobjectdef_t globalobjectdefs[] = {' +## for node in database.containerlist: +## if isinstance(node, PyObjectNode): +## for target in node.where_to_copy_me: +## print >> f, '\t{%s, "%s"},' % (target, node.exported_name) +## print >> f, '\t{ NULL, NULL }\t/* Sentinel */' +## print >> f, '};' +## print >> f +## print >> f, '/***********************************************************/' +## print >> f, '/*** Table of functions ***/' +## print >> f +## print >> f, 'static globalfunctiondef_t globalfunctiondefs[] = {' +## wrappers = pyobjmaker.wrappers.items() +## wrappers.sort() +## for globalobject_name, (base_name, wrapper_name, func_doc) in wrappers: +## print >> f, ('\t{&%s, "%s", {"%s", (PyCFunction)%s, ' +## 'METH_VARARGS|METH_KEYWORDS, %s}},' % ( +## globalobject_name, +## globalobject_name, +## base_name, +## wrapper_name, +## func_doc and c_string_constant(func_doc) or 'NULL')) +## print >> f, '\t{ NULL }\t/* Sentinel */' +## print >> f, '};' +## print >> f, 'static globalfunctiondef_t *globalfunctiondefsptr = &globalfunctiondefs[0];' +## print >> f +## print >> f, '/***********************************************************/' +## print >> f, '/*** Frozen Python bytecode: the initialization code ***/' +## print >> f +## print >> f, 'static char *frozen_initcode[] = {"\\' +## bytecode, originalsource = database.pyobjmaker.getfrozenbytecode() +## g = targetdir.join('frozen.py').open('w') +## g.write(originalsource) +## g.close() +## def char_repr(c): +## if c in '\\"': return '\\' + c +## if ' ' <= c < '\x7F': return c +## return '\\%03o' % ord(c) +## for i in range(0, len(bytecode), 32): +## print >> f, ''.join([char_repr(c) for c in bytecode[i:i+32]])+'\\' +## if (i+32) % 1024 == 0: +## print >> f, '", "\\' +## print >> f, '"};' +## print >> f, "#define FROZEN_INITCODE_SIZE %d" % len(bytecode) +## print >> f + +## # +## # Module initialization function +## # +## print >> f, '/***********************************************************/' +## print >> f, '/*** Module initialization function ***/' +## print >> f +## gen_startupcode(f, database) +## print >> f +## print >> f, 'MODULE_INITFUNC(%s)' % modulename +## print >> f, '{' +## print >> f, '\tSETUP_MODULE(%s);' % modulename +## for publicname, pyobjptr in exports.items(): +## # some fishing needed to find the name of the obj +## pyobjnode = database.containernodes[pyobjptr._obj] +## print >> f, '\tPyModule_AddObject(m, "%s", %s);' % (publicname, +## pyobjnode.name) +## print >> f, '\tcall_postsetup(m);' +## print >> f, '}' f.close() # Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py Sat Nov 24 13:02:16 2007 @@ -876,15 +876,30 @@ self.db = db self.T = T self.obj = obj - self.name = db.pyobjmaker.computenameof(obj.value) + value = obj.value + self.name = self._python_c_name(value) self.ptrname = self.name self.exported_name = self.name # a list of expressions giving places where this constant PyObject # must be copied. Normally just in the global variable of the same # name, but see also StructNode.initializationexpr() :-( self.where_to_copy_me = [] - if self.name not in db.pyobjmaker.wrappers: - self.where_to_copy_me.append('&%s' % self.name) + + def _python_c_name(self, value): + # just some minimal cases: None and builtin exceptions + if value is None: + return 'Py_None' + import types, py + if isinstance(value, (type, types.ClassType)): + if (issubclass(value, Exception) and + (value.__module__ == 'exceptions' + or value is py.magic.AssertionError)): + return 'PyExc_' + value.__name__ + raise Exception("don't know how to simply render py object: %r" % + value) + + def forward_declaration(self): + return [] def enum_dependencies(self): return [] Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/g_include.h Sat Nov 24 13:02:16 2007 @@ -24,7 +24,6 @@ #endif #ifndef PYPY_STANDALONE -# include "src/module.h" # include "src/pyobj.h" #endif Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Sat Nov 24 13:02:16 2007 @@ -58,6 +58,7 @@ return checking_fn def test_func_as_pyobject(): + py.test.skip("") def f(x): return x*2 t = TranslationContext() @@ -88,7 +89,7 @@ return l[0] * l[-1] f1 = compile(f, [int]) assert f1(5) == 30 - assert f1(x=5) == 30 + #assert f1(x=5) == 30 def test_rptr(): @@ -105,7 +106,7 @@ return -42 f1 = compile(f, [int]) assert f1(5) == 10 - assert f1(i=5) == 10 + #assert f1(i=5) == 10 assert f1(1) == 2 assert f1(0) == -42 assert f1(-1) == -42 @@ -124,6 +125,7 @@ def test_runtime_type_info(): + py.test.skip("") S = GcStruct('s', ('is_actually_s1', Bool)) S1 = GcStruct('s1', ('sub', S)) attachRuntimeTypeInfo(S) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_newgc.py Sat Nov 24 13:02:16 2007 @@ -112,6 +112,7 @@ fn(0) == 3 def test_pyobj(): + py.test.skip("unsupported") def f(x): if x: a = 1 From cfbolz at codespeak.net Sat Nov 24 13:40:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 13:40:44 +0100 (CET) Subject: [pypy-svn] r49030 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator: c/test test Message-ID: <20071124124044.5D36E81AF@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 13:40:43 2007 New Revision: 49030 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/snippet.py Log: fix this test, this is no longer allowed Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_typed.py Sat Nov 24 13:40:43 2007 @@ -60,7 +60,7 @@ def test_inheritance2(self): inheritance2 = self.getcompiled(snippet.inheritance2) - assert inheritance2() == ((-12, -12), (3, "world")) + assert inheritance2() == ((-12, -12.0), (3, 12.3)) def test_factorial2(self): factorial2 = self.getcompiled(snippet.factorial2, [int]) Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/snippet.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/snippet.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/snippet.py Sat Nov 24 13:40:43 2007 @@ -371,7 +371,7 @@ d = D() d.stuff = (-12, -12) e = E() - e.stuff = (3, "world") + e.stuff = (3, 12.3) return _getstuff(d), _getstuff(e) class F: From rxe at codespeak.net Sat Nov 24 13:47:33 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 24 Nov 2007 13:47:33 +0100 (CET) Subject: [pypy-svn] r49031 - pypy/dist/pypy/translator/llvm Message-ID: <20071124124733.81A4A81AF@code0.codespeak.net> Author: rxe Date: Sat Nov 24 13:47:33 2007 New Revision: 49031 Modified: pypy/dist/pypy/translator/llvm/genllvm.py Log: alcohol fueled typo Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sat Nov 24 13:47:33 2007 @@ -191,7 +191,7 @@ for node in self.db.getnodes(): include_dirs, includes, sources = node.external_c_source() - for incdirin in include_dirs: + for incdir in include_dirs: c_include_dirs[incdir] = True for include in includes: c_includes[include] = True From rxe at codespeak.net Sat Nov 24 13:53:03 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 24 Nov 2007 13:53:03 +0100 (CET) Subject: [pypy-svn] r49032 - pypy/dist/pypy/translator/llvm Message-ID: <20071124125303.D45BC8151@code0.codespeak.net> Author: rxe Date: Sat Nov 24 13:53:03 2007 New Revision: 49032 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: add python.h in again Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sat Nov 24 13:53:03 2007 @@ -136,6 +136,9 @@ predeclarefn("__ENTRY_POINT__", entrynode.get_ref()) ccode.append('#define ENTRY_POINT_DEFINED 1\n\n') + # include python.h early + ccode.append('#include \n') + # ask gcpolicy for any code needed ccode.append('%s\n' % db.gcpolicy.genextern_code()) From cfbolz at codespeak.net Sat Nov 24 14:14:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 14:14:58 +0100 (CET) Subject: [pypy-svn] r49033 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071124131458.10861818F@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 14:14:57 2007 New Revision: 49033 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Log: call the rpython startup code, which makes more tests pass Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Sat Nov 24 14:14:57 2007 @@ -221,6 +221,9 @@ def malloc_counters(): return _malloc_counters(None, None) + +_rpython_startup = _lib.RPython_StartupCode +_rpython_startup() """ % {'so_name': self.c_source_filename.new(ext=so_ext), 'entrypoint_name': self.entrypoint_name, 'c_entrypoint_name': wrapped_entrypoint_c_name, @@ -822,7 +825,6 @@ ## print >> f, '/***********************************************************/' ## print >> f, '/*** Module initialization function ***/' ## print >> f -## gen_startupcode(f, database) ## print >> f ## print >> f, 'MODULE_INITFUNC(%s)' % modulename ## print >> f, '{' @@ -834,6 +836,7 @@ ## pyobjnode.name) ## print >> f, '\tcall_postsetup(m);' ## print >> f, '}' + gen_startupcode(f, database) f.close() # From xoraxax at codespeak.net Sat Nov 24 14:31:55 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 24 Nov 2007 14:31:55 +0100 (CET) Subject: [pypy-svn] r49034 - in pypy/dist/pypy: rpython/module translator/c Message-ID: <20071124133155.9719B81A7@code0.codespeak.net> Author: xoraxax Date: Sat Nov 24 14:31:54 2007 New Revision: 49034 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/translator/c/genc.py Log: Add a new line to the pypy macro file, fix the Makefile by including all necessary include dirs. Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Nov 24 14:31:54 2007 @@ -85,7 +85,7 @@ data = {'ret_type': 'int', 'name': name} decls.append(decl_snippet % data) defs.append(def_snippet % data) - h_source = ['#include "sys/wait.h"'] + decls + ["#ifndef PYPY_NOT_MAIN_FILE"] + defs + ["#endif"] + h_source = ['#include "sys/wait.h"'] + decls + ["#ifndef PYPY_NOT_MAIN_FILE"] + defs + ["#endif", ""] h_file = udir.join("pypy_os_macros.h") h_file.write("\n".join(h_source)) Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sat Nov 24 14:31:54 2007 @@ -292,7 +292,7 @@ print >> f prefix = ' ' * len(prefix) - compiler = self.getccompiler(extra_includes=['.']) + compiler = self.getccompiler(extra_includes=['.', str(self.targetdir)] + self.include_dirs) if sys.platform == 'darwin': compiler.compile_extra.append('-mdynamic-no-pic') if self.config.translation.compilerflags: From xoraxax at codespeak.net Sat Nov 24 14:32:33 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 24 Nov 2007 14:32:33 +0100 (CET) Subject: [pypy-svn] r49035 - pypy/dist/pypy/translator/llvm Message-ID: <20071124133233.99DC081A7@code0.codespeak.net> Author: xoraxax Date: Sat Nov 24 14:32:33 2007 New Revision: 49035 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: Beautify code. Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sat Nov 24 14:32:33 2007 @@ -105,8 +105,6 @@ def get_incdirs(c_include_dirs): - c_include_dirs - import distutils.sysconfig includes = tuple(c_include_dirs) + ("/sw/include", @@ -114,7 +112,7 @@ distutils.sysconfig.EXEC_PREFIX + "/include/gc", distutils.sysconfig.get_python_inc(), get_c_cpath(), - get_llvm_cpath()) + get_llvm_cpath(),) includestr = "" for ii in includes: From cfbolz at codespeak.net Sat Nov 24 14:34:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 14:34:00 +0100 (CET) Subject: [pypy-svn] r49036 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test Message-ID: <20071124133400.9175181A7@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 14:34:00 2007 New Revision: 49036 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_database.py Log: kill these tests that are using now-unsupported features Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_database.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_database.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_database.py Sat Nov 24 14:34:00 2007 @@ -151,26 +151,6 @@ db.complete() dump_on_stdout(db) -def test_untyped_func(): - def f(x): - return x+1 - graph = TranslationContext().buildflowgraph(f) - - F = FuncType([Ptr(PyObject)], Ptr(PyObject)) - f = functionptr(F, "f", graph=graph) - db = LowLevelDatabase() - db.get(f) - db.complete() - dump_on_stdout(db) - - S = GcStruct('testing', ('fptr', Ptr(F))) - s = malloc(S) - s.fptr = f - db = LowLevelDatabase() - db.get(s) - db.complete() - dump_on_stdout(db) - # ____________________________________________________________ def makegraph(func, argtypes): @@ -195,15 +175,6 @@ db.complete() dump_on_stdout(db) -def test_func_as_pyobject(): - def f(x): - return x+1 - t, graph = makegraph(f, [int]) - - db = LowLevelDatabase(t) - db.get(pyobjectptr(f)) - db.complete() - dump_on_stdout(db) def test_malloc(): S = GcStruct('testing', ('x', Signed), ('y', Signed)) @@ -240,15 +211,6 @@ db.complete() dump_on_stdout(db) -def test_array(): - A = GcArray(('obj', Ptr(PyObject))) - a = malloc(A, 10) - a[3].obj = pyobjectptr(5) - db = LowLevelDatabase() - db.get(a) - db.complete() - dump_on_stdout(db) - def test_array_of_char(): A = GcArray(Char) a = malloc(A, 11) @@ -259,15 +221,6 @@ db.complete() dump_on_stdout(db) -def test_array_of_pyobj(): - A = GcArray(Ptr(PyObject)) - a = malloc(A, 10) - a[3] = pyobjectptr(5) - db = LowLevelDatabase() - db.get(a) - db.complete() - dump_on_stdout(db) - def test_recursive_struct(): S = GcForwardReference() S.become(GcStruct('testing', ('p', Ptr(S)))) From cfbolz at codespeak.net Sat Nov 24 14:53:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 14:53:44 +0100 (CET) Subject: [pypy-svn] r49037 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test Message-ID: <20071124135344.636F9818F@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 14:53:44 2007 New Revision: 49037 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_extfunc.py Log: fix test_execv Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_extfunc.py Sat Nov 24 14:53:44 2007 @@ -825,9 +825,9 @@ if hasattr(posix, 'execv') and hasattr(posix, 'fork'): def test_execv(): + progname = str(sys.executable) filename = str(udir.join('test_execv.txt')) def does_stuff(): - progname = str(sys.executable) l = [progname, '-c', 'open(%r,"w").write("1")' % filename] pid = os.fork() if pid == 0: From fijal at codespeak.net Sat Nov 24 14:55:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 14:55:24 +0100 (CET) Subject: [pypy-svn] r49038 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071124135524.BA3ED80BE@code0.codespeak.net> Author: fijal Date: Sat Nov 24 14:55:24 2007 New Revision: 49038 Modified: pypy/dist/pypy/module/__builtin__/__init__.py pypy/dist/pypy/module/__builtin__/descriptor.py pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Log: Move property to interp-level + port tests from cpython Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Sat Nov 24 14:55:24 2007 @@ -55,8 +55,6 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', - 'property' : 'app_descriptor.property', - 'complex' : 'app_complex.complex', 'buffer' : 'app_buffer.buffer', @@ -127,6 +125,8 @@ 'super' : 'descriptor.W_Super', 'staticmethod' : 'descriptor.StaticMethod', 'classmethod' : 'descriptor.W_ClassMethod', + 'property' : 'descriptor.W_Property', + } def pick_builtin(self, w_globals): Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Sat Nov 24 14:55:24 2007 @@ -6,6 +6,8 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.callmethod import object_getattribute from pypy.interpreter.function import StaticMethod, Method +from pypy.interpreter.typedef import GetSetProperty, descr_get_dict, \ + descr_set_dict class W_Super(Wrappable): def __init__(self, space, w_selftype, w_starttype, w_type, w_self): @@ -129,3 +131,89 @@ If a class method is called for a derived class, the derived class object is passed as the implied first argument.""", ) + +class W_Property(Wrappable): + def __init__(self, space, w_fget, w_fset, w_fdel, doc): + self.w_fget = w_fget + self.w_fset = w_fset + self.w_fdel = w_fdel + self.doc = doc + # eh... + w = space.wrap + self.w_dict = space.newdict() + for w_item, w_value in [(w('fget'), w_fget), (w('fset'), w_fset), + (w('fdel'), w_fdel)]: + space.setitem(self.w_dict, w_item, w_value) + + def new(space, w_type, w_fget=None, w_fset=None, w_fdel=None, doc=''): + return W_Property(space, w_fget, w_fset, w_fdel, doc) + new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, str] + + def get(self, space, w_obj, w_objtype=None): + if space.is_w(w_obj, space.w_None): + return space.wrap(self) + if space.is_w(self.w_fget, space.w_None): + raise OperationError(space.w_AttributeError, space.wrap( + "unreadable attribute")) + return space.call_function(self.w_fget, w_obj) + get.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def set(self, space, w_obj, w_value): + if space.is_w(self.w_fset, space.w_None): + raise OperationError(space.w_AttributeError, space.wrap( + "can't set attribute")) + space.call_function(self.w_fset, w_obj, w_value) + return space.w_None + set.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def delete(self, space, w_obj): + if space.is_w(self.w_fdel, space.w_None): + raise OperationError(space.w_AttributeError, space.wrap( + "can't delete attribute")) + space.call_function(self.w_fdel, w_obj) + return space.w_None + delete.unwrap_spec = ['self', ObjSpace, W_Root] + + def getattribute(self, space, attr): + if attr == '__doc__': + return space.wrap(self.doc) + # shortcuts + elif attr == 'fget': + return self.w_fget + elif attr == 'fset': + return self.w_fset + elif attr == 'fdel': + return self.w_fdel + return space.call_function(object_getattribute(space), + space.wrap(self), space.wrap(attr)) + getattribute.unwrap_spec = ['self', ObjSpace, str] + + def setattr(self, space, attr, w_value): + raise OperationError(space.w_TypeError, space.wrap( + "Trying to set readonly attribute %s on property" % (attr,))) + setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] + + def descr_get_dict(space, self): + return self.w_dict + +W_Property.typedef = TypeDef( + 'property', + __doc__ = '''property(fget=None, fset=None, fdel=None, doc=None) -> property attribute + +fget is a function to be used for getting an attribute value, and likewise +fset is a function for setting, and fdel a function for deleting, an +attribute. Typical use is to define a managed attribute x: +class C(object): + def getx(self): return self.__x + def setx(self, value): self.__x = value + def delx(self): del self.__x + x = property(getx, setx, delx, "I am the 'x' property.")''', + __new__ = interp2app(W_Property.new.im_func), + __get__ = interp2app(W_Property.get), + __set__ = interp2app(W_Property.set), + __delete__ = interp2app(W_Property.delete), + __getattribute__ = interp2app(W_Property.getattribute), + __dict__ = GetSetProperty(W_Property.descr_get_dict), + __setattr__ = interp2app(W_Property.setattr), +) + Modified: pypy/dist/pypy/module/__builtin__/test/test_descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_descriptor.py Sat Nov 24 14:55:24 2007 @@ -224,3 +224,71 @@ raises(TypeError, "classmethod(1).__get__(1)") + def test_property_docstring(self): + assert property.__doc__.startswith('property') + + class A: + pass + + A.x = property(lambda x: x, lambda x, y: x, lambda x:x, 'xxx') + assert A.x.__doc__ == 'xxx' + + def test_property(self): + class C(object): + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = property(getx, setx, delx, doc="I'm the x property.") + a = C() + assert not hasattr(a, "x") + a.x = 42 + assert a._C__x == 42 + assert a.x == 42 + del a.x + assert not hasattr(a, "x") + assert not hasattr(a, "_C__x") + C.x.__set__(a, 100) + assert C.x.__get__(a) == 100 + C.x.__delete__(a) + assert not hasattr(a, "x") + + raw = C.__dict__['x'] + assert isinstance(raw, property) + + attrs = dir(raw) + assert "__doc__" in attrs + assert "fget" in attrs + assert "fset" in attrs + assert "fdel" in attrs + + assert raw.__doc__ == "I'm the x property." + assert raw.fget is C.__dict__['getx'] + assert raw.fset is C.__dict__['setx'] + assert raw.fdel is C.__dict__['delx'] + + for attr in "__doc__", "fget", "fset", "fdel": + try: + setattr(raw, attr, 42) + except TypeError, msg: + if str(msg).find('readonly') < 0: + raise Exception("when setting readonly attr %r on a " + "property, got unexpected TypeError " + "msg %r" % (attr, str(msg))) + else: + raise Exception("expected TypeError from trying to set " + "readonly %r attr on a property" % attr) + + class D(object): + __getitem__ = property(lambda s: 1/0) + + d = D() + try: + for i in d: + str(i) + except ZeroDivisionError: + pass + else: + raise Exception, "expected ZeroDivisionError from bad property" From fijal at codespeak.net Sat Nov 24 14:55:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 14:55:44 +0100 (CET) Subject: [pypy-svn] r49039 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071124135544.6158980D2@code0.codespeak.net> Author: fijal Date: Sat Nov 24 14:55:44 2007 New Revision: 49039 Removed: pypy/dist/pypy/module/__builtin__/app_descriptor.py Log: This file is not needed any more. From fijal at codespeak.net Sat Nov 24 15:01:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 15:01:21 +0100 (CET) Subject: [pypy-svn] r49040 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071124140121.8624A81A7@code0.codespeak.net> Author: fijal Date: Sat Nov 24 15:01:19 2007 New Revision: 49040 Modified: pypy/dist/pypy/module/__builtin__/descriptor.py Log: Another hacks. Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Sat Nov 24 15:01:19 2007 @@ -138,12 +138,6 @@ self.w_fset = w_fset self.w_fdel = w_fdel self.doc = doc - # eh... - w = space.wrap - self.w_dict = space.newdict() - for w_item, w_value in [(w('fget'), w_fget), (w('fset'), w_fset), - (w('fdel'), w_fdel)]: - space.setitem(self.w_dict, w_item, w_value) def new(space, w_type, w_fget=None, w_fset=None, w_fdel=None, doc=''): return W_Property(space, w_fget, w_fset, w_fdel, doc) @@ -178,24 +172,24 @@ if attr == '__doc__': return space.wrap(self.doc) # shortcuts - elif attr == 'fget': - return self.w_fget - elif attr == 'fset': - return self.w_fset - elif attr == 'fdel': - return self.w_fdel return space.call_function(object_getattribute(space), space.wrap(self), space.wrap(attr)) getattribute.unwrap_spec = ['self', ObjSpace, str] + def fget(space, self): + return self.w_fget + + def fset(space, self): + return self.w_fset + + def fdel(space, self): + return self.w_fdel + def setattr(self, space, attr, w_value): raise OperationError(space.w_TypeError, space.wrap( "Trying to set readonly attribute %s on property" % (attr,))) setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] - def descr_get_dict(space, self): - return self.w_dict - W_Property.typedef = TypeDef( 'property', __doc__ = '''property(fget=None, fset=None, fdel=None, doc=None) -> property attribute @@ -213,7 +207,9 @@ __set__ = interp2app(W_Property.set), __delete__ = interp2app(W_Property.delete), __getattribute__ = interp2app(W_Property.getattribute), - __dict__ = GetSetProperty(W_Property.descr_get_dict), __setattr__ = interp2app(W_Property.setattr), + fdel = GetSetProperty(W_Property.fdel), + fget = GetSetProperty(W_Property.fget), + fset = GetSetProperty(W_Property.fset), ) From fijal at codespeak.net Sat Nov 24 15:05:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 15:05:19 +0100 (CET) Subject: [pypy-svn] r49041 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071124140519.C1FB081AF@code0.codespeak.net> Author: fijal Date: Sat Nov 24 15:05:19 2007 New Revision: 49041 Modified: pypy/dist/pypy/module/__builtin__/descriptor.py Log: Default doc is None, not '' Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Sat Nov 24 15:05:19 2007 @@ -7,7 +7,7 @@ from pypy.interpreter.callmethod import object_getattribute from pypy.interpreter.function import StaticMethod, Method from pypy.interpreter.typedef import GetSetProperty, descr_get_dict, \ - descr_set_dict + descr_set_dict, interp_attrproperty_w class W_Super(Wrappable): def __init__(self, space, w_selftype, w_starttype, w_type, w_self): @@ -133,15 +133,15 @@ ) class W_Property(Wrappable): - def __init__(self, space, w_fget, w_fset, w_fdel, doc): + def __init__(self, space, w_fget, w_fset, w_fdel, w_doc): self.w_fget = w_fget self.w_fset = w_fset self.w_fdel = w_fdel - self.doc = doc + self.w_doc = w_doc - def new(space, w_type, w_fget=None, w_fset=None, w_fdel=None, doc=''): - return W_Property(space, w_fget, w_fset, w_fdel, doc) - new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, str] + def new(space, w_type, w_fget=None, w_fset=None, w_fdel=None, w_doc=None): + return W_Property(space, w_fget, w_fset, w_fdel, w_doc) + new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, W_Root] def get(self, space, w_obj, w_objtype=None): if space.is_w(w_obj, space.w_None): @@ -170,21 +170,12 @@ def getattribute(self, space, attr): if attr == '__doc__': - return space.wrap(self.doc) + return self.w_doc # shortcuts return space.call_function(object_getattribute(space), space.wrap(self), space.wrap(attr)) getattribute.unwrap_spec = ['self', ObjSpace, str] - def fget(space, self): - return self.w_fget - - def fset(space, self): - return self.w_fset - - def fdel(space, self): - return self.w_fdel - def setattr(self, space, attr, w_value): raise OperationError(space.w_TypeError, space.wrap( "Trying to set readonly attribute %s on property" % (attr,))) @@ -208,8 +199,8 @@ __delete__ = interp2app(W_Property.delete), __getattribute__ = interp2app(W_Property.getattribute), __setattr__ = interp2app(W_Property.setattr), - fdel = GetSetProperty(W_Property.fdel), - fget = GetSetProperty(W_Property.fget), - fset = GetSetProperty(W_Property.fset), + fdel = interp_attrproperty_w('w_fdel', W_Property), + fget = interp_attrproperty_w('w_fget', W_Property), + fset = interp_attrproperty_w('w_fset', W_Property), ) From cfbolz at codespeak.net Sat Nov 24 15:12:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 15:12:15 +0100 (CET) Subject: [pypy-svn] r49042 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . test Message-ID: <20071124141215.8987A80D4@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 15:12:14 2007 New Revision: 49042 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py Log: fix error message that didn't work for tuples. fix test Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py Sat Nov 24 15:12:14 2007 @@ -896,7 +896,7 @@ or value is py.magic.AssertionError)): return 'PyExc_' + value.__name__ raise Exception("don't know how to simply render py object: %r" % - value) + (value, )) def forward_declaration(self): return [] Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_lltyped.py Sat Nov 24 15:12:14 2007 @@ -453,11 +453,13 @@ def f(n): result = () for cls in classes: - values = [getmin(cls), getmax(cls)] for OP in operators: - for x in values: - res1 = OP(x, n) - result += (res1,) + x = getmin(cls) + res1 = OP(x, n) + result = result + (res1,) + x = getmax(cls) + res1 = OP(x, n) + result = result + (res1,) return result def assert_eq(a, b): From pedronis at codespeak.net Sat Nov 24 17:42:00 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 24 Nov 2007 17:42:00 +0100 (CET) Subject: [pypy-svn] r49043 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: src test Message-ID: <20071124164200.87B6B81AF@code0.codespeak.net> Author: pedronis Date: Sat Nov 24 17:41:59 2007 New Revision: 49043 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/exception.h pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Log: more fixes Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/exception.h ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/exception.h (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/exception.h Sat Nov 24 17:41:59 2007 @@ -105,7 +105,7 @@ v = NULL; } else { - pycls = RPythonError; + pycls = PyExc_Exception; /* XXX RPythonError */ v = PyString_FromString(clsname); } Py_INCREF(pycls); Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_genc.py Sat Nov 24 17:41:59 2007 @@ -3,33 +3,18 @@ from pypy.annotation import model as annmodel from pypy.translator.translator import TranslationContext from pypy.translator.c.database import LowLevelDatabase +from pypy.translator.c import genc from pypy.translator.c.genc import gen_source from pypy.translator.c.gc import NoneGcPolicy from pypy.objspace.flow.model import Constant, Variable, SpaceOperation from pypy.objspace.flow.model import Block, Link, FunctionGraph from pypy.tool.udir import udir from pypy.translator.tool.cbuild import make_module_from_c -from pypy.translator.tool.cbuild import enable_fast_compilation from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation from pypy import conftest -# XXX this tries to make compiling faster for full-scale testing -# XXX tcc leaves some errors undetected! Bad! -#from pypy.translator.tool import cbuild -#cbuild.enable_fast_compilation() - -def compile_db(db): - enable_fast_compilation() # for testing - modulename = uniquemodulename('testing') - targetdir = udir.join(modulename).ensure(dir=1) - gen_source(db, modulename, str(targetdir), defines={'COUNT_OP_MALLOCS': 1}) - m = make_module_from_c(targetdir.join(modulename+'.c'), - include_dirs = [os.path.dirname(autopath.this_dir)], - libraries = db.gcpolicy.gc_libraries()) - return m - def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): t = Translation(fn, argtypes, gc=gcpolicy, backend="c", @@ -57,29 +42,24 @@ return res return checking_fn -def test_func_as_pyobject(): - py.test.skip("") +def test_simple(): def f(x): return x*2 t = TranslationContext() t.buildannotator().build_types(f, [int]) t.buildrtyper().specialize() - db = LowLevelDatabase(t) - entrypoint = db.get(pyobjectptr(f)) - db.complete() - module = compile_db(db) + t.config.translation.countmallocs = True + builder = genc.CExtModuleBuilder(t, f, config=t.config) + builder.generate_source() + builder.compile() + f1 = builder.get_entry_point() - f1 = getattr(module, entrypoint) assert f1(5) == 10 - assert f1(x=5) == 10 assert f1(-123) == -246 - assert module.malloc_counters() == (0, 0) + assert builder.get_malloc_counters()() == (0, 0) + py.test.raises(Exception, f1, "world") # check that it's really typed - py.test.raises(Exception, f1) - py.test.raises(Exception, f1, 2, 3) - py.test.raises(Exception, f1, 2, x=2) - #py.test.raises(Exception, f1, 2, y=2) XXX missing a check at the moment def test_rlist(): @@ -125,7 +105,6 @@ def test_runtime_type_info(): - py.test.skip("") S = GcStruct('s', ('is_actually_s1', Bool)) S1 = GcStruct('s1', ('sub', S)) attachRuntimeTypeInfo(S) @@ -143,7 +122,7 @@ p1 = malloc(S1) p1.sub.is_actually_s1 = True # and no crash when p and p1 are decref'ed - return sys + return None t = TranslationContext() t.buildannotator().build_types(does_stuff, []) rtyper = t.buildrtyper() @@ -152,15 +131,14 @@ rtyper.specialize() #t.view() - db = LowLevelDatabase(t) - entrypoint = db.get(pyobjectptr(does_stuff)) - db.complete() - - module = compile_db(db) - - f1 = getattr(module, entrypoint) + from pypy.translator.c import genc + t.config.translation.countmallocs = True + builder = genc.CExtModuleBuilder(t, does_stuff, config=t.config) + builder.generate_source() + builder.compile() + f1 = builder.get_entry_point() f1() - mallocs, frees = module.malloc_counters() + mallocs, frees = builder.get_malloc_counters()() assert mallocs == frees From cfbolz at codespeak.net Sat Nov 24 17:57:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 17:57:23 +0100 (CET) Subject: [pypy-svn] r49044 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/test Message-ID: <20071124165723.7A53581DA@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 17:57:23 2007 New Revision: 49044 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/test_extension.py Log: this is no longer supported From cfbolz at codespeak.net Sat Nov 24 18:01:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 18:01:28 +0100 (CET) Subject: [pypy-svn] r49045 - pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386 Message-ID: <20071124170128.541EB81F8@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 18:01:27 2007 New Revision: 49045 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py Log: missing import Modified: pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/jit/codegen/i386/rgenop.py Sat Nov 24 18:01:27 2007 @@ -1,4 +1,5 @@ import py +import ctypes from pypy.rlib.objectmodel import specialize, we_are_translated from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.codegen.model import AbstractRGenOp, GenLabel, GenBuilder From cfbolz at codespeak.net Sat Nov 24 18:07:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 18:07:25 +0100 (CET) Subject: [pypy-svn] r49046 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator/c: . src Message-ID: <20071124170725.0907581F8@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 18:07:25 2007 New Revision: 49046 Removed: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/pyobj.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/module.h Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/database.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/pyobj.h Log: kill pyobj.py and module.h. deal with the (few) consequences Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/database.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/database.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/database.py Sat Nov 24 18:07:25 2007 @@ -54,9 +54,6 @@ # constant self.late_initializations = [] self.namespace = CNameManager() - if not standalone: - from pypy.translator.c.pyobj import PyObjMaker - self.pyobjmaker = PyObjMaker(self.namespace, self, translator) if translator is None or translator.rtyper is None: self.exctransformer = None @@ -303,8 +300,6 @@ while True: while True: - if hasattr(self, 'pyobjmaker'): - self.pyobjmaker.collect_initcode() while self.pendingsetupnodes: lst = self.pendingsetupnodes self.pendingsetupnodes = [] Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Sat Nov 24 18:07:25 2007 @@ -40,7 +40,7 @@ self.libraries = libraries self.exports = {} - def build_database(self, pyobj_options=None): + def build_database(self): translator = self.translator gcpolicyclass = self.get_gcpolicyclass() @@ -63,10 +63,6 @@ thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) self.db = db - # pass extra options into pyobjmaker - if pyobj_options: - for key, value in pyobj_options.items(): - setattr(db.pyobjmaker, key, value) # we need a concrete gcpolicy to do this self.libraries += db.gcpolicy.gc_libraries() @@ -765,77 +761,6 @@ sg.set_strategy(targetdir) sg.gen_readable_parts_of_source(f) -## # -## # PyObject support (strange) code -## # -## pyobjmaker = database.pyobjmaker -## print >> f -## print >> f, '/***********************************************************/' -## print >> f, '/*** Table of global PyObjects ***/' -## print >> f -## print >> f, 'static globalobjectdef_t globalobjectdefs[] = {' -## for node in database.containerlist: -## if isinstance(node, PyObjectNode): -## for target in node.where_to_copy_me: -## print >> f, '\t{%s, "%s"},' % (target, node.exported_name) -## print >> f, '\t{ NULL, NULL }\t/* Sentinel */' -## print >> f, '};' -## print >> f -## print >> f, '/***********************************************************/' -## print >> f, '/*** Table of functions ***/' -## print >> f -## print >> f, 'static globalfunctiondef_t globalfunctiondefs[] = {' -## wrappers = pyobjmaker.wrappers.items() -## wrappers.sort() -## for globalobject_name, (base_name, wrapper_name, func_doc) in wrappers: -## print >> f, ('\t{&%s, "%s", {"%s", (PyCFunction)%s, ' -## 'METH_VARARGS|METH_KEYWORDS, %s}},' % ( -## globalobject_name, -## globalobject_name, -## base_name, -## wrapper_name, -## func_doc and c_string_constant(func_doc) or 'NULL')) -## print >> f, '\t{ NULL }\t/* Sentinel */' -## print >> f, '};' -## print >> f, 'static globalfunctiondef_t *globalfunctiondefsptr = &globalfunctiondefs[0];' -## print >> f -## print >> f, '/***********************************************************/' -## print >> f, '/*** Frozen Python bytecode: the initialization code ***/' -## print >> f -## print >> f, 'static char *frozen_initcode[] = {"\\' -## bytecode, originalsource = database.pyobjmaker.getfrozenbytecode() -## g = targetdir.join('frozen.py').open('w') -## g.write(originalsource) -## g.close() -## def char_repr(c): -## if c in '\\"': return '\\' + c -## if ' ' <= c < '\x7F': return c -## return '\\%03o' % ord(c) -## for i in range(0, len(bytecode), 32): -## print >> f, ''.join([char_repr(c) for c in bytecode[i:i+32]])+'\\' -## if (i+32) % 1024 == 0: -## print >> f, '", "\\' -## print >> f, '"};' -## print >> f, "#define FROZEN_INITCODE_SIZE %d" % len(bytecode) -## print >> f - -## # -## # Module initialization function -## # -## print >> f, '/***********************************************************/' -## print >> f, '/*** Module initialization function ***/' -## print >> f -## print >> f -## print >> f, 'MODULE_INITFUNC(%s)' % modulename -## print >> f, '{' -## print >> f, '\tSETUP_MODULE(%s);' % modulename -## for publicname, pyobjptr in exports.items(): -## # some fishing needed to find the name of the obj -## pyobjnode = database.containernodes[pyobjptr._obj] -## print >> f, '\tPyModule_AddObject(m, "%s", %s);' % (publicname, -## pyobjnode.name) -## print >> f, '\tcall_postsetup(m);' -## print >> f, '}' gen_startupcode(f, database) f.close() Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/pyobj.h ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/pyobj.h (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/pyobj.h Sat Nov 24 18:07:25 2007 @@ -218,17 +218,6 @@ #define OP_NEWDICT(args,r) if (!(r=PyDict_Pack args)) CFAIL() #define OP_NEWTUPLE(args,r) if (!(r=PyTuple_Pack args)) CFAIL() -/*** argument parsing ***/ - -#define OP_DECODE_ARG(fname, pos, name, vargs, vkwds, r) \ - if (!(r=decode_arg(fname, pos, name, vargs, vkwds, NULL))) CFAIL() -#define OP_DECODE_ARG_DEF(fname, pos, name, vargs, vkwds, def, r) \ - if (!(r=decode_arg(fname, pos, name, vargs, vkwds, def))) CFAIL() -#define OP_CHECK_NO_MORE_ARG(fname, n, vargs, r) \ - if ((r=check_no_more_arg(fname, n, vargs)) < 0) CFAIL() -#define OP_CHECK_SELF_NONZERO(fname, self, r) \ - if ((r=check_self_nonzero(fname, self)) < 0) CFAIL() - unsigned long long RPyLong_AsUnsignedLongLong(PyObject *v); long long RPyLong_AsLongLong(PyObject *v); From cfbolz at codespeak.net Sat Nov 24 18:08:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 18:08:55 +0100 (CET) Subject: [pypy-svn] r49047 - pypy/branch/remove-extcompiler-rctypes/pypy/translator/c Message-ID: <20071124170855.C1D9181F8@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 18:08:55 2007 New Revision: 49047 Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/support.py Log: delete old commented out code Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/support.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/support.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/support.py Sat Nov 24 18:08:55 2007 @@ -137,44 +137,6 @@ return '{%s}' % ', '.join(lines) -##def gen_assignments(assignments): -## # Generate a sequence of assignments that is possibly reordered -## # to avoid clashes -- i.e. do the equivalent of a tuple assignment, -## # reading all sources first, writing all targets next, but optimized - -## allsources = [] -## src2dest = {} -## types = {} -## for typename, dest, src in assignments: -## if src != dest: # ignore 'v=v;' -## allsources.append(src) -## src2dest.setdefault(src, []).append(dest) -## types[dest] = typename - -## for starting in allsources: -## # starting from some starting variable, follow a chain of assignments -## # 'vn=vn-1; ...; v3=v2; v2=v1; v1=starting;' -## v = starting -## srcchain = [] -## while src2dest.get(v): -## srcchain.append(v) -## v = src2dest[v].pop(0) -## if v == starting: -## break # loop -## if not srcchain: -## continue # already done in a previous chain -## srcchain.reverse() # ['vn-1', ..., 'v2', 'v1', 'starting'] -## code = [] -## for pair in zip([v] + srcchain[:-1], srcchain): -## code.append('%s = %s;' % pair) -## if v == starting: -## # assignment loop 'starting=vn-1; ...; v2=v1; v1=starting;' -## typename = types[starting] -## tmpdecl = cdecl(typename, 'tmp') -## code.insert(0, '{ %s = %s;' % (tmpdecl, starting)) -## code[-1] = '%s = tmp; }' % (srcchain[-2],) -## yield ' '.join(code) - def gen_assignments(assignments): # Generate a sequence of assignments that is possibly reordered # to avoid clashes -- i.e. do the equivalent of a tuple assignment, From cfbolz at codespeak.net Sat Nov 24 18:20:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 24 Nov 2007 18:20:07 +0100 (CET) Subject: [pypy-svn] r49048 - in pypy/branch/remove-extcompiler-rctypes/pypy/translator: c c/test llsupport llsupport/test Message-ID: <20071124172007.8D64081A2@code0.codespeak.net> Author: cfbolz Date: Sat Nov 24 18:20:05 2007 New Revision: 49048 Added: pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/ pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/__init__.py (contents, props changed) pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/test/ pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/test/__init__.py (contents, props changed) pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/test/test_wrapper.py - copied, changed from r49043, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/wrapper.py - copied, changed from r49043, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Removed: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/test_wrapper.py pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/wrapper.py Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Log: move the wrapper generator to a new llsupport directory Modified: pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py Sat Nov 24 18:20:05 2007 @@ -4,7 +4,7 @@ from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines -from pypy.translator.c.wrapper import new_wrapper +from pypy.translator.llsupport.wrapper import new_wrapper from pypy.translator.gensupp import uniquemodulename, NameManager from pypy.translator.tool.cbuild import so_ext from pypy.translator.tool.cbuild import compile_c_module Added: pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/__init__.py ============================================================================== Added: pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/test/__init__.py ============================================================================== From arigo at codespeak.net Sat Nov 24 18:49:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Nov 2007 18:49:58 +0100 (CET) Subject: [pypy-svn] r49049 - pypy/dist/pypy/config Message-ID: <20071124174958.977AF818F@code0.codespeak.net> Author: arigo Date: Sat Nov 24 18:49:57 2007 New Revision: 49049 Added: pypy/dist/pypy/config/pypyoption.py.merge.tmp - copied, changed from r49048, pypy/dist/pypy/config/pypyoption.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/remove-extcompiler-rctypes/pypy/config/pypyoption.py revisions 48921 to 49048: ------------------------------------------------------------------------ r48926 | cfbolz | 2007-11-22 15:56:01 +0100 (Thu, 22 Nov 2007) | 3 lines (pedronis, cfbolz): kill: the cpy object space, raymond (again), the extcompiler, support everywhere for the extcompiler ------------------------------------------------------------------------ r48923 | cfbolz | 2007-11-22 15:31:39 +0100 (Thu, 22 Nov 2007) | 4 lines (pedronis, cfbolz): a branch for removing the extcompiler, rctypes, raymond, and the ability to have the C backend generate dlls that happen to be CPython extension modules too. ------------------------------------------------------------------------ From arigo at codespeak.net Sat Nov 24 18:50:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Nov 2007 18:50:00 +0100 (CET) Subject: [pypy-svn] r49050 - pypy/dist/pypy/translator/c Message-ID: <20071124175000.6D90281A1@code0.codespeak.net> Author: arigo Date: Sat Nov 24 18:49:59 2007 New Revision: 49050 Added: pypy/dist/pypy/translator/c/genc.py.merge.tmp - copied, changed from r49048, pypy/dist/pypy/translator/c/genc.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/genc.py revisions 48921 to 49048: ------------------------------------------------------------------------ r49048 | cfbolz | 2007-11-24 18:20:05 +0100 (Sat, 24 Nov 2007) | 2 lines move the wrapper generator to a new llsupport directory ------------------------------------------------------------------------ r49046 | cfbolz | 2007-11-24 18:07:25 +0100 (Sat, 24 Nov 2007) | 2 lines kill pyobj.py and module.h. deal with the (few) consequences ------------------------------------------------------------------------ r49033 | cfbolz | 2007-11-24 14:14:57 +0100 (Sat, 24 Nov 2007) | 2 lines call the rpython startup code, which makes more tests pass ------------------------------------------------------------------------ r49025 | pedronis | 2007-11-24 13:02:16 +0100 (Sat, 24 Nov 2007) | 2 lines (cfbolz, pedronis) WIP: not making c extension anymore ------------------------------------------------------------------------ r48988 | cfbolz | 2007-11-23 15:48:10 +0100 (Fri, 23 Nov 2007) | 2 lines kill this function (again), it's not used ------------------------------------------------------------------------ r48987 | pedronis | 2007-11-23 13:56:19 +0100 (Fri, 23 Nov 2007) | 5 lines (cfbolz, pedronis) don't depend on the so generated to be proper c extensions ------------------------------------------------------------------------ r48983 | cfbolz | 2007-11-23 12:40:17 +0100 (Fri, 23 Nov 2007) | 2 lines remove the symboltable feature, it's disabled anyway ------------------------------------------------------------------------ r48926 | cfbolz | 2007-11-22 15:56:01 +0100 (Thu, 22 Nov 2007) | 3 lines (pedronis, cfbolz): kill: the cpy object space, raymond (again), the extcompiler, support everywhere for the extcompiler ------------------------------------------------------------------------ r48923 | cfbolz | 2007-11-22 15:31:39 +0100 (Thu, 22 Nov 2007) | 4 lines (pedronis, cfbolz): a branch for removing the extcompiler, rctypes, raymond, and the ability to have the C backend generate dlls that happen to be CPython extension modules too. ------------------------------------------------------------------------ From arigo at codespeak.net Sat Nov 24 18:50:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Nov 2007 18:50:05 +0100 (CET) Subject: [pypy-svn] r49051 - pypy/dist/pypy/rpython/test Message-ID: <20071124175005.6A39F81BE@code0.codespeak.net> Author: arigo Date: Sat Nov 24 18:50:02 2007 New Revision: 49051 Added: pypy/dist/pypy/rpython/test/test_rbuiltin.py.merge.tmp - copied, changed from r49048, pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/remove-extcompiler-rctypes/pypy/rpython/test/test_rbuiltin.py revisions 48921 to 49048: ------------------------------------------------------------------------ r48942 | cfbolz | 2007-11-22 18:46:18 +0100 (Thu, 22 Nov 2007) | 2 lines (pedronis, cfbolz): deal with the removal of rctypes ------------------------------------------------------------------------ r48923 | cfbolz | 2007-11-22 15:31:39 +0100 (Thu, 22 Nov 2007) | 4 lines (pedronis, cfbolz): a branch for removing the extcompiler, rctypes, raymond, and the ability to have the C backend generate dlls that happen to be CPython extension modules too. ------------------------------------------------------------------------ From arigo at codespeak.net Sat Nov 24 18:50:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Nov 2007 18:50:10 +0100 (CET) Subject: [pypy-svn] r49052 - pypy/dist/pypy/translator/c/src Message-ID: <20071124175010.62CC981B8@code0.codespeak.net> Author: arigo Date: Sat Nov 24 18:50:09 2007 New Revision: 49052 Added: pypy/dist/pypy/translator/c/src/g_include.h.merge.tmp - copied, changed from r49048, pypy/dist/pypy/translator/c/src/g_include.h Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/g_include.h revisions 48921 to 49048: ------------------------------------------------------------------------ r49025 | pedronis | 2007-11-24 13:02:16 +0100 (Sat, 24 Nov 2007) | 2 lines (cfbolz, pedronis) WIP: not making c extension anymore ------------------------------------------------------------------------ r48923 | cfbolz | 2007-11-22 15:31:39 +0100 (Thu, 22 Nov 2007) | 4 lines (pedronis, cfbolz): a branch for removing the extcompiler, rctypes, raymond, and the ability to have the C backend generate dlls that happen to be CPython extension modules too. ------------------------------------------------------------------------ From arigo at codespeak.net Sat Nov 24 18:53:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Nov 2007 18:53:42 +0100 (CET) Subject: [pypy-svn] r49053 - in pypy/dist/pypy: annotation bin config doc interpreter jit module/_file module/mmap module/readline module/signal objspace/cpy rlib rlib/rcairo rlib/rctypes rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/rctypes rpython/test translator translator/c translator/c/src translator/c/test translator/goal translator/llsupport translator/test translator/tool Message-ID: <20071124175342.9CE1581B3@code0.codespeak.net> Author: arigo Date: Sat Nov 24 18:53:42 2007 New Revision: 49053 Added: pypy/dist/pypy/annotation/builtin.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/annotation/builtin.py pypy/dist/pypy/bin/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/bin/ pypy/dist/pypy/config/pypyoption.py - copied unchanged from r49052, pypy/dist/pypy/config/pypyoption.py.merge.tmp pypy/dist/pypy/doc/cleanup-todo.txt - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/doc/cleanup-todo.txt pypy/dist/pypy/doc/faq.txt - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/doc/faq.txt pypy/dist/pypy/doc/index.txt - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/doc/index.txt pypy/dist/pypy/doc/project-ideas.txt - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/doc/project-ideas.txt pypy/dist/pypy/interpreter/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/interpreter/ pypy/dist/pypy/jit/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/jit/ pypy/dist/pypy/module/_file/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/module/_file/ pypy/dist/pypy/module/mmap/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/module/mmap/ pypy/dist/pypy/module/readline/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/module/readline/ pypy/dist/pypy/module/signal/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/module/signal/ pypy/dist/pypy/rlib/rcairo/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rlib/rcairo/ pypy/dist/pypy/rlib/streamio.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rlib/streamio.py pypy/dist/pypy/rpython/llinterp.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/lltype.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/rclass.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/lltypesystem/rvirtualizable.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/rvirtualizable.py pypy/dist/pypy/rpython/lltypesystem/test/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/lltypesystem/test/ pypy/dist/pypy/rpython/memory/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/memory/ pypy/dist/pypy/rpython/rbuiltin.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rmodel.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/rpython/rmodel.py pypy/dist/pypy/rpython/test/test_rbuiltin.py - copied unchanged from r49052, pypy/dist/pypy/rpython/test/test_rbuiltin.py.merge.tmp pypy/dist/pypy/translator/c/database.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/database.py pypy/dist/pypy/translator/c/funcgen.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/genc.py - copied unchanged from r49052, pypy/dist/pypy/translator/c/genc.py.merge.tmp pypy/dist/pypy/translator/c/node.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/node.py pypy/dist/pypy/translator/c/src/exception.h - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/exception.h pypy/dist/pypy/translator/c/src/g_include.h - copied unchanged from r49052, pypy/dist/pypy/translator/c/src/g_include.h.merge.tmp pypy/dist/pypy/translator/c/src/mem.h - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/src/pyobj.h - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/src/pyobj.h pypy/dist/pypy/translator/c/support.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/support.py pypy/dist/pypy/translator/c/test/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/c/test/ pypy/dist/pypy/translator/driver.py - copied unchanged from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/driver.py pypy/dist/pypy/translator/llsupport/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/llsupport/ pypy/dist/pypy/translator/test/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/test/ pypy/dist/pypy/translator/tool/ - copied from r49052, pypy/branch/remove-extcompiler-rctypes/pypy/translator/tool/ Removed: pypy/dist/pypy/config/pypyoption.py.merge.tmp pypy/dist/pypy/doc/extcompiler.txt pypy/dist/pypy/objspace/cpy/ pypy/dist/pypy/rlib/rctypes/ pypy/dist/pypy/rpython/rcpy.py pypy/dist/pypy/rpython/rctypes/ pypy/dist/pypy/rpython/test/test_rbuiltin.py.merge.tmp pypy/dist/pypy/translator/c/genc.py.merge.tmp pypy/dist/pypy/translator/c/pyobj.py pypy/dist/pypy/translator/c/src/g_include.h.merge.tmp pypy/dist/pypy/translator/c/src/module.h pypy/dist/pypy/translator/c/symboltable.py pypy/dist/pypy/translator/c/wrapper.py pypy/dist/pypy/translator/goal/targetdemomodule.py pypy/dist/pypy/translator/goal/targetrpystonex.py Log: (cfbolz, pedronis): Merge the remove-extcompiler-rctypes branch: kill rctypes, the cpy objspace, the extcompiler, the support for making cpython extension modules with genc. Make genc tests working again by writing a small ctypes wrapper. All thse "features" were broken in various degrees. From fijal at codespeak.net Sat Nov 24 19:00:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 19:00:45 +0100 (CET) Subject: [pypy-svn] r49054 - in pypy/branch/rewrite-compilation-logic: . pypy/rlib pypy/rpython/lltypesystem pypy/rpython/lltypesystem/module pypy/rpython/lltypesystem/test pypy/rpython/module pypy/rpython/tool pypy/rpython/tool/test pypy/translator/c pypy/translator/c/test pypy/translator/tool pypy/translator/tool/test Message-ID: <20071124180045.8DC0981B8@code0.codespeak.net> Author: fijal Date: Sat Nov 24 19:00:44 2007 New Revision: 49054 Added: pypy/branch/rewrite-compilation-logic/ - copied from r49053, pypy/dist/ Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/_rsocket_rffi.py pypy/branch/rewrite-compilation-logic/pypy/rlib/rposix.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/lltype.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/module/ll_math.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_termios.py pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rffi_platform.py pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rfficache.py pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/test/test_rffi_platform.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py Log: (fijal, arigo, xoraxax) In-progress checkin of rewriting compilation logic. Tries to sort out the need of includes include_dirs, libraries and friends. Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/_rsocket_rffi.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rlib/_rsocket_rffi.py Sat Nov 24 19:00:44 2007 @@ -4,6 +4,7 @@ from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rffi import CCHARP from pypy.rlib.rposix import get_errno as geterrno +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask, r_uint import os @@ -51,9 +52,13 @@ COND_HEADER = '' constants = {} +eci = ExternalCompilationInfo( + pre_include_lines = (HEADER + COND_HEADER).split("\n"), + includes = includes +) class CConfig: - _header_ = HEADER + COND_HEADER + _compilation_info_ = eci # constants linux = platform.Defined('linux') MS_WINDOWS = platform.Defined('_WIN32') @@ -376,10 +381,10 @@ for _name, _header in cond_includes: if getattr(cConfig, _name) is not None: includes.append(_header) + eci = ExternalCompilationInfo(includes=includes, libraries=libraries) def external(name, args, result): - return rffi.llexternal(name, args, result, - includes=includes, libraries=libraries, + return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv=calling_conv) if _POSIX: Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/rposix.py ============================================================================== --- pypy/dist/pypy/rlib/rposix.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rlib/rposix.py Sat Nov 24 19:00:44 2007 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem.rffi import CConstant, CExternVariable from pypy.rpython.lltypesystem import lltype, ll2ctypes +from pypy.translator.tool.cbuild import ExternalCompilationInfo class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -15,6 +16,10 @@ assert index == 0 ll2ctypes.TLS.errno = value -get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', CConstantErrno, - includes=['errno.h'], sandboxsafe=True) +errno_eci = ExternalCompilationInfo( + includes=['errno.h'] +) + +get_errno, set_errno = CExternVariable(lltype.Signed, 'errno', errno_eci, + CConstantErrno, sandboxsafe=True) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py Sat Nov 24 19:00:44 2007 @@ -513,15 +513,6 @@ # ____________________________________________ -def compile_c_snippet(name, source): - from pypy.tool.udir import udir - cname = udir.join(name + '.c') - f = cname.open('w') - f.write(source) - f.write('\n') - f.close() - return cache_c_module([cname], name) - def get_ctypes_callable(funcptr, calling_conv): if not ctypes: raise ImportError("ctypes is needed to use ll2ctypes") @@ -534,16 +525,12 @@ except AttributeError: pass - sources = getattr(funcptr._obj, 'sources', None) - if sources: - assert len(sources) == 1 - dllname = compile_c_snippet(funcptr._obj._name, sources[0]) - libraries = [dllname] - else: - libraries = getattr(funcptr._obj, 'libraries', None) + eci = funcptr._obj.compilation_info + libraries = list(eci.libraries) + funcname = funcptr._obj._name + eci = eci.make_shared_lib() FUNCTYPE = lltype.typeOf(funcptr).TO - funcname = funcptr._obj._name if not libraries: cfunc = get_on_lib(standard_c_lib, funcname) # XXX magic: on Windows try to load the function from 'kernel32' too Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/lltype.py Sat Nov 24 19:00:44 2007 @@ -413,9 +413,8 @@ def __init__(self, args, result): for arg in args: assert isinstance(arg, LowLevelType) - # -- disable the following check for the benefits of rffi -- - if isinstance(arg, ContainerType): - raise TypeError, "function arguments can only be primitives or pointers" + # There are external C functions eating raw structures, not + # pointers, don't check args not being container types self.ARGS = tuple(args) assert isinstance(result, LowLevelType) if isinstance(result, ContainerType): Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/module/ll_math.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/module/ll_math.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/module/ll_math.py Sat Nov 24 19:00:44 2007 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo math_frexp = rffi.llexternal('frexp', [rffi.DOUBLE, rffi.INTP], rffi.DOUBLE, sandboxsafe=True) @@ -55,9 +56,11 @@ else: raise ValueError("math domain error") +eci = ExternalCompilationInfo(libraries=['m']) + def new_unary_math_function(name): c_func = rffi.llexternal(name, [rffi.DOUBLE], rffi.DOUBLE, - sandboxsafe=True, libraries=['m']) + compilation_info=eci, sandboxsafe=True) def ll_math(x): _error_reset() @@ -69,7 +72,7 @@ def new_binary_math_function(name): c_func = rffi.llexternal(name, [rffi.DOUBLE, rffi.DOUBLE], rffi.DOUBLE, - sandboxsafe=True, libraries=['m']) + compilation_info=eci, sandboxsafe=True) def ll_math(x, y): _error_reset() Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py Sat Nov 24 19:00:44 2007 @@ -10,6 +10,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.tool.rfficache import platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo import os class CConstant(Symbolic): @@ -25,8 +26,8 @@ def lltype(self): return self.TP -def llexternal(name, args, result, _callable=None, sources=[], includes=[], - libraries=[], include_dirs=[], library_dirs=[], +def llexternal(name, args, result, _callable=None, + compilation_info=ExternalCompilationInfo(), sandboxsafe=False, threadsafe='auto', canraise=False, _nowrapper=False, calling_conv='c'): """Build an external function that will invoke the C function 'name' @@ -48,11 +49,7 @@ if _callable is None: _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv) funcptr = lltype.functionptr(ext_type, name, external='C', - sources=tuple(sources), - includes=tuple(includes), - libraries=tuple(libraries), - include_dirs=tuple(include_dirs), - library_dirs=tuple(library_dirs), + compilation_info=compilation_info, _callable=_callable, _safe_not_sandboxed=sandboxsafe, _debugexc=True, # on top of llinterp @@ -220,7 +217,9 @@ return lltype.Ptr(CArray(tp)) CArray._annspecialcase_ = 'specialize:memo' -def COpaque(name, hints=None, **kwds): +def COpaque(name, hints=None, compilation_info=None): + if compilation_info is None: + compilation_info = ExternalCompilationInfo() if hints is None: hints = {} else: @@ -230,11 +229,7 @@ def lazy_getsize(): from pypy.rpython.tool import rffi_platform k = {} - for _name, value in kwds.items(): - if _name in ['includes', 'include_dirs', 'libraries', - 'library_dirs']: - k['_%s_' % _name] = value - return rffi_platform.sizeof(name, '', **k) + return rffi_platform.sizeof(name, compilation_info) hints['getsize'] = lazy_getsize return lltype.OpaqueType(name, hints) @@ -242,12 +237,13 @@ def COpaquePtr(*args, **kwds): return lltype.Ptr(COpaque(*args, **kwds)) -def CExternVariable(TYPE, name, _CConstantClass=CConstant, includes=[], - include_dirs=[], sandboxsafe=False): +def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant, + sandboxsafe=False): """Return a pair of functions - a getter and a setter - to access the given global C variable. """ from pypy.translator.c.primitive import PrimitiveType + from pypy.translator.tool.cbuild import ExternalCompilationInfo # XXX we cannot really enumerate all C types here, do it on a case-by-case # basis if TYPE == CCHARPP: @@ -264,15 +260,18 @@ c_getter = "%(c_type)s %(getter_name)s () { return %(name)s; }" % locals() c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals() - lines = ["#include <%s>" % i for i in includes] + lines = ["#include <%s>" % i for i in eci.includes] lines.append(c_getter) lines.append(c_setter) sources = ('\n'.join(lines),) - - kwds = {'includes': includes, 'sources':sources, - 'include_dirs':include_dirs, 'sandboxsafe': sandboxsafe} - getter = llexternal(getter_name, [], TYPE, **kwds) - setter = llexternal(setter_name, [TYPE], lltype.Void, **kwds) + new_eci = eci.merge(ExternalCompilationInfo( + separate_module_sources = sources + )) + + getter = llexternal(getter_name, [], TYPE, compilation_info=new_eci, + sandboxsafe=sandboxsafe) + setter = llexternal(setter_name, [TYPE], lltype.Void, + compilation_info=new_eci, sandboxsafe=sandboxsafe) return getter, setter ## # XXX THIS IS ONLY A QUICK HACK TO MAKE IT WORK Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Nov 24 19:00:44 2007 @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.annlowlevel import llhelper from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo class TestLL2Ctypes(object): @@ -139,8 +140,9 @@ assert not ALLOCATED # detects memory leaks in the test def test_strlen(self): + eci = ExternalCompilationInfo(includes=['string.h']) strlen = rffi.llexternal('strlen', [rffi.CCHARP], rffi.SIZE_T, - includes=['string.h']) + compilation_info=eci) s = rffi.str2charp("xxx") res = strlen(s) rffi.free_charp(s) @@ -152,19 +154,22 @@ assert not ALLOCATED # detects memory leaks in the test def test_func_not_in_clib(self): + eci = ExternalCompilationInfo(libraries=['m']) foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed) py.test.raises(NotImplementedError, foobar) foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed, - libraries=['m']) # math library + compilation_info=eci) # math library py.test.raises(NotImplementedError, foobar) + eci = ExternalCompilationInfo(libraries=['m', 'z']) foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed, - libraries=['m', 'z']) # math and zlib + compilation_info=eci) # math and zlib py.test.raises(NotImplementedError, foobar) + eci = ExternalCompilationInfo(libraries=['I_really_dont_exist_either']) foobar = rffi.llexternal('I_really_dont_exist', [], lltype.Signed, - libraries=['I_really_dont_exist_either']) + compilation_info=eci) py.test.raises(NotImplementedError, foobar) assert not ALLOCATED # detects memory leaks in the test @@ -229,9 +234,9 @@ assert not ALLOCATED # detects memory leaks in the test def test_strchr(self): + eci = ExternalCompilationInfo(includes=['string.h']) strchr = rffi.llexternal('strchr', [rffi.CCHARP, rffi.INT], - rffi.CCHARP, - includes=['string.h']) + rffi.CCHARP, compilation_info=eci) s = rffi.str2charp("hello world") res = strchr(s, ord('r')) assert res[0] == 'r' @@ -243,11 +248,11 @@ assert not ALLOCATED # detects memory leaks in the test def test_frexp(self): + eci = ExternalCompilationInfo(includes=['math.h'], + libraries=['m']) A = lltype.FixedSizeArray(rffi.INT, 1) frexp = rffi.llexternal('frexp', [rffi.DOUBLE, lltype.Ptr(A)], - rffi.DOUBLE, - includes=['math.h'], - libraries=['m']) + rffi.DOUBLE, compilation_info=eci) p = lltype.malloc(A, flavor='raw') res = frexp(2.5, p) assert res == 0.625 @@ -256,10 +261,11 @@ assert not ALLOCATED # detects memory leaks in the test def test_rand(self): + eci = ExternalCompilationInfo(includes=['stdlib.h']) rand = rffi.llexternal('rand', [], rffi.INT, - includes=['stdlib.h']) + compilation_info=eci) srand = rffi.llexternal('srand', [rffi.UINT], lltype.Void, - includes=['stdlib.h']) + compilation_info=eci) srand(rffi.r_uint(123)) res1 = rand() res2 = rand() @@ -274,11 +280,13 @@ assert not ALLOCATED # detects memory leaks in the test def test_opaque_obj(self): - includes = ['sys/time.h', 'time.h'] - TIMEVALP = rffi.COpaquePtr('struct timeval', includes=includes) - TIMEZONEP = rffi.COpaquePtr('struct timezone', includes=includes) + eci = ExternalCompilationInfo( + includes = ['sys/time.h', 'time.h'] + ) + TIMEVALP = rffi.COpaquePtr('struct timeval', compilation_info=eci) + TIMEZONEP = rffi.COpaquePtr('struct timezone', compilation_info=eci) gettimeofday = rffi.llexternal('gettimeofday', [TIMEVALP, TIMEZONEP], - rffi.INT, includes=includes) + rffi.INT, compilation_info=eci) ll_timevalp = lltype.malloc(TIMEVALP.TO, flavor='raw') ll_timezonep = lltype.malloc(TIMEZONEP.TO, flavor='raw') res = gettimeofday(ll_timevalp, ll_timezonep) @@ -582,12 +590,13 @@ assert not ALLOCATED # detects memory leaks in the test def test_get_errno(self): + eci = ExternalCompilationInfo(includes=['string.h']) if sys.platform.startswith('win'): underscore_on_windows = '_' else: underscore_on_windows = '' strlen = rffi.llexternal('strlen', [rffi.CCHARP], rffi.SIZE_T, - includes=['string.h']) + compilation_info=eci) os_write = rffi.llexternal(underscore_on_windows+'write', [rffi.INT, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T) @@ -650,23 +659,6 @@ assert isinstance(b[2], rffi.r_singlefloat) assert abs(float(b[2]) - 2.2) < 1E-6 - def test_cfunc_returning_newly_allocated(self): - py.test.skip("complains about a double free") - from crypt import crypt as pycrypt - crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], - rffi.CCHARP, - libraries=['crypt']) - - s1 = rffi.str2charp("pass") - s2 = rffi.str2charp("ab") - r = crypt(s1, s2) - rffi.free_charp(s1) - rffi.free_charp(s2) - res = rffi.charp2str(r) - assert res == pycrypt("pass", "ab") - rffi.free_charp(r) - assert not ALLOCATED - def test_different_signatures(self): fcntl_int = rffi.llexternal('fcntl', [rffi.INT, rffi.INT, rffi.INT], rffi.INT) @@ -677,7 +669,36 @@ fcntl_int(12345, 1, 0) def test_llexternal_source(self): - fn = rffi.llexternal('fn', [], rffi.INT, sources = ["int fn() { return 42; }"]) + eci = ExternalCompilationInfo( + separate_module_sources = ["int fn() { return 42; }"] + ) + fn = rffi.llexternal('fn', [], rffi.INT, compilation_info=eci) res = fn() assert res == 42 + def test_prebuilt_constant(self): + source = py.code.Source(""" + int x = 3; + char** z = NULL; + """) + + eci = ExternalCompilationInfo(post_include_lines=source.lines) + + get_x, set_x = rffi.CExternVariable(lltype.Signed, 'x', eci) + get_z, set_z = rffi.CExternVariable(rffi.CCHARPP, 'z', eci) + + def f(): + one = get_x() + set_x(13) + return one + get_x() + + def g(): + l = rffi.liststr2charpp(["a", "b", "c"]) + try: + set_z(l) + return rffi.charp2str(get_z()[2]) + finally: + rffi.free_charpp(l) + + assert f() == 16 + assert g() == "c" Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py Sat Nov 24 19:00:44 2007 @@ -13,6 +13,7 @@ from pypy.translator.translator import graphof from pypy.conftest import option from pypy.objspace.flow.model import summary +from pypy.translator.tool.cbuild import ExternalCompilationInfo def test_basic(): c_source = py.code.Source(""" @@ -21,7 +22,9 @@ return (x + 3); } """) - z = llexternal('z', [Signed], Signed, sources=[c_source]) + + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('z', [Signed], Signed, eci) def f(): return z(8) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_termios.py Sat Nov 24 19:00:44 2007 @@ -15,11 +15,14 @@ from pypy.rpython import rclass from pypy.rlib import rtermios from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo -includes = ['termios.h', 'unistd.h'] +eci = ExternalCompilationInfo( + includes = ['termios.h', 'unistd.h'] +) class CConfig: - _includes_ = includes + _compilation_info_ = eci NCCS = rffi_platform.DefinedConstantInteger('NCCS') NCCS = rffi_platform.configure(CConfig)['NCCS'] @@ -39,7 +42,7 @@ ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) def c_external(name, args, result): - return rffi.llexternal(name, args, result, includes=includes) + return rffi.llexternal(name, args, result, compilation_info=eci) c_tcsetattr = c_external('tcsetattr', [INT, INT, TERMIOSP], INT) c_cfgetispeed = c_external('cfgetispeed', [TERMIOSP], SPEED_T) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rffi_platform.py Sat Nov 24 19:00:44 2007 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import llmemory from pypy.tool.gcc_cache import build_executable_cache, try_compile_cache +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.tool.udir import udir import distutils @@ -12,39 +13,44 @@ # # Helpers for simple cases +def eci_from_header(c_header_source): + return ExternalCompilationInfo( + pre_include_lines=c_header_source.split("\n") + ) + def getstruct(name, c_header_source, interesting_fields): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci_from_header(c_header_source) STRUCT = Struct(name, interesting_fields) return configure(CConfig)['STRUCT'] def getsimpletype(name, c_header_source, ctype_hint=rffi.INT): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci_from_header(c_header_source) TYPE = SimpleType(name, ctype_hint) return configure(CConfig)['TYPE'] def getconstantinteger(name, c_header_source): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci_from_header(c_header_source) CONST = ConstantInteger(name) return configure(CConfig)['CONST'] def getdefined(macro, c_header_source): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci_from_header(c_header_source) DEFINED = Defined(macro) return configure(CConfig)['DEFINED'] def has(name, c_header_source): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci_from_header(c_header_source) HAS = Has(name) return configure(CConfig)['HAS'] -def sizeof(name, c_header_source, **kwds): +def sizeof(name, eci, **kwds): class CConfig: - _header_ = c_header_source + _compilation_info_ = eci SIZE = SizeOf(name) for k, v in kwds.items(): setattr(CConfig, k, v) @@ -85,12 +91,7 @@ def write_header(self): f = self.f CConfig = self.config - # NB: the _header_ must be printed before everything else, - # because it might contain #defines that need to appear before - # any system #include. - print >> f, getattr(CConfig, '_header_', '') # optional - for path in getattr(CConfig, '_includes_', ()): # optional - print >> f, '#include <%s>' % (path,) + CConfig._compilation_info_.write_c_header(f) print >> f, C_HEADER print >> f @@ -122,12 +123,8 @@ self.start_main() self.f.write(question + "\n") self.close() - include_dirs = getattr(self.config, '_include_dirs_', []) - libraries = getattr(self.config, '_libraries_', []) - library_dirs = getattr(self.config, '_library_dirs_', []) - return try_compile_cache([self.path], include_dirs=include_dirs, - libraries=libraries, - library_dirs=library_dirs) + eci = self.config._compilation_info_ + return try_compile_cache([self.path], eci) def configure(CConfig): """Examine the local system by running the C compiler. @@ -135,6 +132,9 @@ what should be inspected; configure() returns a dict mapping names to the results. """ + for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_', + '_include_dirs_', '_header_']: + assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,) entries = [] for key in dir(CConfig): value = getattr(CConfig, key) @@ -153,11 +153,8 @@ writer.write_entry_main(key) writer.close() - include_dirs = getattr(CConfig, '_include_dirs_', []) - libraries = getattr(CConfig, '_libraries_', []) - library_dirs = getattr(CConfig, '_library_dirs_', []) - infolist = list(run_example_code(writer.path, include_dirs, - libraries, library_dirs)) + eci = CConfig._compilation_info_ + infolist = list(run_example_code(writer.path, eci)) assert len(infolist) == len(entries) resultinfo = {} @@ -503,10 +500,8 @@ } """ -def run_example_code(filepath, include_dirs=[], libraries=[], library_dirs=[]): - output = build_executable_cache([filepath], include_dirs=include_dirs, - libraries=libraries, - library_dirs=library_dirs) +def run_example_code(filepath, eci): + output = build_executable_cache([filepath], eci) section = None for line in output.splitlines(): line = line.strip() Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/rfficache.py Sat Nov 24 19:00:44 2007 @@ -1,12 +1,12 @@ -""" This file creates and maintains _cache/stdtypes.py, which -keeps information about C type sizes -""" +# XXX This is completely outdated file, kept here only for bootstrapping +# reasons. If you touch it, try removing it import py import os import distutils -from pypy.translator.tool.cbuild import build_executable +from pypy.translator.tool.cbuild import build_executable, \ + ExternalCompilationInfo from pypy.tool.udir import udir from pypy.tool.autopath import pypydir from pypy.rlib import rarithmetic @@ -31,7 +31,8 @@ ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") c_file.write(c_source) - return build_executable_cache([c_file], compiler_exe=compiler_exe) + eci = ExternalCompilationInfo() + return build_executable_cache([c_file], eci, compiler_exe=compiler_exe) def sizeof_c_type(c_typename, **kwds): question = 'printf("%%d", sizeof(%s));' % (c_typename,); Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/test/test_rffi_platform.py Sat Nov 24 19:00:44 2007 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi from pypy.tool.udir import udir +from pypy.translator.tool.cbuild import ExternalCompilationInfo def import_ctypes(): try: @@ -109,11 +110,12 @@ test_h.write('#define XYZZY 42\n') class CConfig: - _header_ = """ /* a C comment */ - #include - #include - """ - _include_dirs_ = [str(udir)] + _compilation_info_ = ExternalCompilationInfo( + pre_include_lines = ["/* a C comment */", + "#include ", + "#include "], + include_dirs = [str(udir)] + ) FILE = rffi_platform.Struct('FILE', []) ushort = rffi_platform.SimpleType('unsigned short') @@ -127,14 +129,14 @@ def test_ifdef(): class CConfig: - _header_ = """ /* a C comment */ -#define XYZZY 42 -typedef int foo; -struct s { - int i; - double f; -}; -""" + _compilation_info_ = ExternalCompilationInfo( + post_include_lines = ['/* a C comment */', + '#define XYZZY 42', + 'typedef int foo;', + 'struct s {', + 'int i;', + 'double f;' + '};']) s = rffi_platform.Struct('struct s', [('i', rffi.INT)], ifdef='XYZZY') @@ -152,16 +154,17 @@ def test_nested_structs(): class CConfig: - _header_ = """ -struct x { - int foo; - unsigned long bar; - }; -struct y { - char c; - struct x x; - }; -""" + _compilation_info_ = ExternalCompilationInfo( + post_include_lines=""" + struct x { + int foo; + unsigned long bar; + }; + struct y { + char c; + struct x x; + }; + """.split("\n")) x = rffi_platform.Struct("struct x", [("bar", rffi.SHORT)]) y = rffi_platform.Struct("struct y", [("x", x)]) @@ -193,4 +196,4 @@ assert not rffi_platform.has("x", "#include ") def test_sizeof(): - assert rffi_platform.sizeof("char", "") == 1 + assert rffi_platform.sizeof("char", ExternalCompilationInfo()) == 1 Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py Sat Nov 24 19:00:44 2007 @@ -6,7 +6,7 @@ from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper from pypy.translator.gensupp import uniquemodulename, NameManager -from pypy.translator.tool.cbuild import so_ext +from pypy.translator.tool.cbuild import so_ext, ExternalCompilationInfo from pypy.translator.tool.cbuild import compile_c_module from pypy.translator.tool.cbuild import build_executable, CCompiler, ProfOpt from pypy.translator.tool.cbuild import import_module_from_directory @@ -24,8 +24,8 @@ _compiled = False modulename = None - def __init__(self, translator, entrypoint, config, libraries=None, - gcpolicy=None): + def __init__(self, translator, entrypoint, config, + eci=ExternalCompilationInfo(), gcpolicy=None): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = self.entrypoint.func_name @@ -34,11 +34,8 @@ if gcpolicy is not None and gcpolicy.requires_stackless: config.translation.stackless = True self.config = config - - if libraries is None: - libraries = [] - self.libraries = libraries self.exports = {} + self.eci = eci def build_database(self): translator = self.translator @@ -65,7 +62,8 @@ self.db = db # we need a concrete gcpolicy to do this - self.libraries += db.gcpolicy.gc_libraries() + self.eci = self.eci.merge(ExternalCompilationInfo( + libraries=db.gcpolicy.gc_libraries())) # give the gc a chance to register interest in the start-up functions it # need (we call this for its side-effects of db.get()) @@ -77,19 +75,19 @@ self.exports[self.entrypoint_name] = pf self.c_entrypoint_name = pfname db.complete() - - # add library dependencies - seen = dict.fromkeys(self.libraries) - for node in db.globalcontainers(): - if hasattr(node, 'libraries'): - for library in node.libraries: - if library not in seen: - self.libraries.append(library) - seen[library] = True + + self.collect_compilation_info() return db have___thread = None + def collect_compilation_info(self): + all = [] + for node in self.db.globalcontainers(): + eci = getattr(node, 'compilation_info', None) + if eci: + all.append(eci) + self.eci = self.eci.merge(*all) def get_gcpolicyclass(self): if self.gcpolicy is None: @@ -112,7 +110,6 @@ db = self.build_database() pf = self.getentrypointptr() pfname = db.get(pf) - extra_info = extra_information(db) if self.modulename is None: self.modulename = uniquemodulename('testing') modulename = self.modulename @@ -128,27 +125,21 @@ CBuilder.have___thread = check_under_under_thread() if not self.standalone: assert not self.config.translation.instrument - cfile, extra, include_dirs, library_dirs = \ - gen_source(db, modulename, targetdir, - defines = defines, - exports = self.exports, - libraries = self.libraries, - extra_info = extra_info) + cfile, extra = gen_source(db, modulename, targetdir, self.eci, + defines = defines, + exports = self.exports) else: if self.config.translation.instrument: defines['INSTRUMENT'] = 1 if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 - cfile, extra, include_dirs, library_dirs = \ - gen_source_standalone(db, modulename, targetdir, - entrypointname = pfname, - defines = defines, - extra_info = extra_info) + cfile, extra = gen_source_standalone(db, modulename, targetdir, + self.eci, + entrypointname = pfname, + defines = defines) self.c_source_filename = py.path.local(cfile) self.extrafiles = extra - self.include_dirs = include_dirs.keys() - self.library_dirs = library_dirs.keys() if self.standalone: self.gen_makefile(targetdir) return cfile @@ -636,23 +627,8 @@ print >> f, '\treturn error;' print >> f, '}' -def extra_information(database): - includes = {} - sources = {} - include_dirs = {} - library_dirs = {} - for node in database.globalcontainers(): - for attrname in ['includes', 'sources', 'include_dirs', 'library_dirs']: - if hasattr(node, attrname): - for elem in getattr(node, attrname): - locals()[attrname][elem] = True - includes = includes.keys() - includes.sort() - return {'includes':includes, 'sources':sources, - 'include_dirs':include_dirs, 'library_dirs':library_dirs} - -def gen_source_standalone(database, modulename, targetdir, - entrypointname, defines={}, extra_info={}): +def gen_source_standalone(database, modulename, targetdir, eci, + entrypointname, defines={}): assert database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -710,12 +686,10 @@ print >>fi, "#define INSTRUMENT_NCOUNTER %d" % n fi.close() - return filename, sg.getextrafiles(), extra_info.get('include_dirs', None),\ - extra_info.get('library_dirs', None) + return filename, sg.getextrafiles() -def gen_source(database, modulename, targetdir, defines={}, exports={}, - libraries=[], extra_info={}): +def gen_source(database, modulename, targetdir, eci, defines={}, exports={}): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -741,11 +715,11 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - for include in extra_info.get('includes', []): - print >> fi, '#include <%s>' % (include,) + eci.write_c_header(fi) + fi.close() + for source in extra_info.get('sources', []): print >> f, source - fi.close() if database.translator is None or database.translator.rtyper is None: preimplementationlines = [] @@ -769,13 +743,10 @@ # pypy_include_dir = autopath.this_dir f = targetdir.join('setup.py').open('w') - include_dirs = extra_info.get('include_dirs', []) - library_dirs = extra_info.get('library_dirs', []) f.write(SETUP_PY % locals()) f.close() - return filename, sg.getextrafiles(), include_dirs,\ - library_dirs + return filename, sg.getextrafiles() SETUP_PY = ''' Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py Sat Nov 24 19:00:44 2007 @@ -13,7 +13,7 @@ from pypy.translator.c.primitive import PrimitiveType from pypy.rlib.rarithmetic import isinf, isnan from pypy.translator.c import extfunc - +from pypy.translator.tool.cbuild import ExternalCompilationInfo def needs_gcheader(T): if not isinstance(T, ContainerType): @@ -698,9 +698,8 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - for attrname in 'libraries', 'include_dirs', 'includes', 'sources', 'library_dirs': - if hasattr(obj, attrname): - setattr(self, attrname, getattr(obj, attrname)) + self.compilation_info = getattr(obj, 'compilation_info', + ExternalCompilationInfo()) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py Sat Nov 24 19:00:44 2007 @@ -10,6 +10,7 @@ from pypy.objspace.flow.model import Block, Link, FunctionGraph from pypy.tool.udir import udir from pypy.translator.tool.cbuild import make_module_from_c +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Sat Nov 24 19:00:44 2007 @@ -8,6 +8,7 @@ from pypy.tool.ansi_print import ansi_log log = py.log.Producer("cbuild") py.log.setconsumer("cbuild", ansi_log) +from pypy.tool.udir import udir debug = 0 @@ -16,6 +17,8 @@ _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs', 'post_include_lines', 'libraries', 'library_dirs', 'separate_module_sources', 'separate_module_files'] + _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes', + 'include_dirs', 'library_dirs'] def __init__(self, pre_include_lines = [], @@ -60,6 +63,54 @@ # XXX custom hash and eq functions, they should be compared # by contents + def merge(self, *others): + others = list(others) + attrs = {} + for name in self._ATTRIBUTES: + if name not in self._AVOID_DUPLICATES: + s = [] + for i in [self] + others: + s += getattr(i, name) + attrs[name] = s + else: + s = set() + attr = [] + for one in [self] + others: + for elem in getattr(one, name): + if elem not in s: + s.add(elem) + attr.append(elem) + attrs[name] = attr + return ExternalCompilationInfo(**attrs) + + def write_c_header(self, fileobj): + for line in self.pre_include_lines: + print >> fileobj, line + for path in self.includes: + print >> fileobj, '#include <%s>' % (path,) + for line in self.post_include_lines: + print >> fileobj, line + + def convert_sources_to_files(self, cache_dir=None): + if cache_dir is None: + cache_dir = udir.join('module_cache').ensure(dir=1) + num = 0 + files = [] + for source in self.separate_module_sources: + while 1: + filename = cache_dir.join('module_%d.c' % num) + num += 1 + if not filename.check(): + break + filename.write(source) + files.append(str(filename)) + d = {} + for attr in self._ATTRIBUTES: + d[attr] = getattr(self, attr) + d['separate_module_sources'] = () + d['separate_module_files'] += tuple(files) + return ExternalCompilationInfo(**d) + if sys.platform == 'win32': so_ext = '.dll' else: @@ -94,17 +145,24 @@ opt += '/Op' gcv['OPT'] = opt -def compile_c_module(cfiles, modname, include_dirs=[], libraries=[], - library_dirs=[]): +def compile_c_module(cfiles, modname, eci): #try: # from distutils.log import set_threshold # set_threshold(10000) #except ImportError: # print "ERROR IMPORTING" # pass - include_dirs = list(include_dirs) + cfiles = [py.path.local(f) for f in cfiles] + tmpdir = udir.join("modcache").ensure(dir=1) + num = 0 + for source in eci.separate_module_sources: + c_file = tmpdir.join('mod_%d.c' % num) + c_file.write(source) + cfiles.append(c_file) + cfiles += eci.separate_module_files + include_dirs = list(eci.include_dirs) include_dirs.append(py.path.local(pypydir).join('translator', 'c')) - library_dirs = list(library_dirs) + library_dirs = list(eci.library_dirs) if sys.platform == 'darwin': # support Fink & Darwinports for s in ('/sw/', '/opt/local/'): if s + 'include' not in include_dirs and \ @@ -114,8 +172,9 @@ os.path.exists(s + 'lib'): library_dirs.append(s + 'lib') - dirpath = cfiles[0].dirpath() + dirpath = py.path.local(modname).dirpath() lastdir = dirpath.chdir() + libraries = eci.libraries ensure_correct_math() try: if debug: print "modname", modname @@ -195,9 +254,9 @@ raise finally: lastdir.chdir() + return modname + so_ext -def cache_c_module(cfiles, modname, cache_dir=None, - include_dirs=[], libraries=[]): +def cache_c_module(cfiles, modname, eci, cache_dir=None): """ Same as build c module, but instead caches results. XXX currently there is no way to force a recompile, so this is pretty useless as soon as the sources (or headers they depend on) change :-/ @@ -210,14 +269,12 @@ cache_dir = py.path.local(cache_dir) assert cache_dir.check(dir=1) # XXX modname = str(cache_dir.join(modname)) - compile_c_module(cfiles, modname, include_dirs=include_dirs, - libraries=libraries) - return modname + so_ext + return compile_c_module(cfiles, modname, eci) -def make_module_from_c(cfile, include_dirs=None, libraries=[]): +def make_module_from_c(cfile, eci): cfile = py.path.local(cfile) modname = cfile.purebasename - compile_c_module([cfile], modname, include_dirs, libraries) + compile_c_module([cfile], modname, eci) return import_module_from_directory(cfile.dirpath(), modname) def import_module_from_directory(dir, modname): @@ -267,16 +324,15 @@ class CCompiler: - def __init__(self, cfilenames, outputfilename=None, include_dirs=[], - libraries=[], library_dirs=[], compiler_exe=None, - profbased=None): + def __init__(self, cfilenames, eci, outputfilename=None, + compiler_exe=None, profbased=None): self.cfilenames = cfilenames ext = '' self.compile_extra = [] self.link_extra = [] - self.libraries = list(libraries) - self.include_dirs = list(include_dirs) - self.library_dirs = list(library_dirs) + self.libraries = list(eci.libraries) + self.include_dirs = list(eci.include_dirs) + self.library_dirs = list(eci.library_dirs) self.compiler_exe = compiler_exe self.profbased = profbased if not sys.platform in ('win32', 'darwin'): # xxx Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py Sat Nov 24 19:00:44 2007 @@ -1,7 +1,9 @@ import py, sys from pypy.tool.udir import udir -from pypy.translator.tool.cbuild import build_executable, cache_c_module +from pypy.translator.tool.cbuild import build_executable, cache_c_module,\ + ExternalCompilationInfo +from subprocess import Popen, PIPE, STDOUT def test_simple_executable(): print udir @@ -14,7 +16,8 @@ return 0; } """) - testexec = build_executable([t]) + eci = ExternalCompilationInfo() + testexec = build_executable([t], eci) out = py.process.cmdexec(testexec) assert out.startswith('hello world') @@ -30,8 +33,81 @@ csourcedir = pypydir.join('translator', 'c', 'src') include_dirs = [str(csourcedir.dirpath())] files = [csourcedir.join('thread.c')] - cache_c_module(files, '_thread', cache_dir=udir, include_dirs=include_dirs, - libraries=['pthread']) + eci = ExternalCompilationInfo( + include_dirs=include_dirs, + libraries=['pthread'] + ) + cache_c_module(files, '_thread', eci, cache_dir=udir) cdll = ctypes.CDLL(str(udir.join('_thread.so'))) assert hasattr(cdll, 'RPyThreadLockInit') + +class TestEci: + def setup_class(cls): + tmpdir = udir.ensure('testeci', dir=1) + c_file = tmpdir.join('module.c') + c_file.write(py.code.Source(''' + int sum(int x, int y) + { + return x + y; + } + ''')) + cls.modfile = c_file + cls.tmpdir = tmpdir + + def test_standalone(self): + tmpdir = self.tmpdir + c_file = tmpdir.join('stand1.c') + c_file.write(''' + #include + #include + + int main() + { + printf("%f\\n", pow(2.0, 2.0)); + }''') + eci = ExternalCompilationInfo( + libraries = ['m'], + ) + output = build_executable([c_file], eci) + p = Popen(output, stdout=PIPE, stderr=STDOUT) + p.wait() + assert p.stdout.readline().startswith('4.0') + + def test_merge(self): + e1 = ExternalCompilationInfo( + pre_include_lines = ['1'], + includes = ['x.h'], + post_include_lines = ['p1'] + ) + e2 = ExternalCompilationInfo( + pre_include_lines = ['2'], + includes = ['x.h', 'y.h'], + post_include_lines = ['p2'], + ) + e3 = ExternalCompilationInfo( + pre_include_lines = ['3'], + includes = ['y.h', 'z.h'], + post_include_lines = ['p3'] + ) + e = e1.merge(e2, e3) + assert e.pre_include_lines == ('1', '2', '3') + assert e.includes == ('x.h', 'y.h', 'z.h') + assert e.post_include_lines == ('p1', 'p2', 'p3') + + def test_convert_sources_to_c_files(self): + eci = ExternalCompilationInfo( + separate_module_sources = ['xxx'], + separate_module_files = ['x.c'], + ) + cache_dir = udir.join('test_convert_sources').ensure(dir=1) + neweci = eci.convert_sources_to_files(cache_dir) + assert not neweci.separate_module_sources + res = neweci.separate_module_files + assert len(res) == 2 + assert res[0] == 'x.c' + assert str(res[1]).startswith(str(cache_dir)) + + def test_compile_c_files_to_ofiles(self): + pass + From rxe at codespeak.net Sat Nov 24 19:10:32 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sat, 24 Nov 2007 19:10:32 +0100 (CET) Subject: [pypy-svn] r49055 - pypy/dist/pypy/translator/llvm Message-ID: <20071124181032.2EE5481AF@code0.codespeak.net> Author: rxe Date: Sat Nov 24 19:10:31 2007 New Revision: 49055 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/opwriter.py Log: try to make fastcc calling convention work again (not there yet) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat Nov 24 19:10:31 2007 @@ -72,9 +72,7 @@ def create_constant_node(self, type_, value): node = None if isinstance(type_, lltype.FuncType): - if hasattr(value, '_external_name'): - node = ExternalFuncNode(self, value, value._external_name) - elif getattr(value, 'external', None) == 'C': + if getattr(value, 'external', None) == 'C': node = ExternalFuncNode(self, value) else: node = FuncImplNode(self, value) Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sat Nov 24 19:10:31 2007 @@ -6,18 +6,45 @@ from pypy.objspace.flow.model import FunctionGraph from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype -from pypy.translator.llvm.buildllvm import llvm_gcc_version from pypy.tool.udir import udir -support_functions = [ - "@LLVM_RPython_StartupCode", - ] +def get_c_cpath(): + from pypy.translator.c import genc + return os.path.dirname(genc.__file__) + +def get_llvm_cpath(): + return os.path.join(os.path.dirname(__file__), "module") def get_module_file(name): return os.path.join(get_llvm_cpath(), name) -def get_ll(ccode, function_names, default_cconv, c_include_dirs): - function_names += support_functions +def get_incdirs(c_include_dirs): + import distutils.sysconfig + + includes = tuple(c_include_dirs) + ("/sw/include", + distutils.sysconfig.EXEC_PREFIX + "/include", + distutils.sysconfig.EXEC_PREFIX + "/include/gc", + distutils.sysconfig.get_python_inc(), + get_c_cpath(), + get_llvm_cpath()) + + includestr = "" + for ii in includes: + includestr += "-I %s " % ii + return includestr + +# call entrypoint needs to be fastcc +# call boehm finalizers need to be fastcc + +def generate_ll(ccode, default_cconv, c_include_dirs, call_funcnames=[]): + + call_funcnames += ['@LLVM_RPython_StartupCode'] + define_funcnames = ['@pypy_malloc', + '@pypy_malloc_atomic', + '@pypy_gc__collect', + '@pypy_register_finalizer'] + declare_funcnames = ['@LLVM_RPython_StartupCode'] + filename = str(udir.join("ccode.c")) f = open(filename, "w") f.write(ccode) @@ -25,13 +52,8 @@ plain = filename[:-2] includes = get_incdirs(c_include_dirs) - if llvm_gcc_version() < 4.0: - emit_llvm = '' - else: - emit_llvm = '-emit-llvm -O0' - - cmd = "llvm-gcc %s %s -S %s.c -o %s.ll 2>&1" % ( - includes, emit_llvm, plain, plain) + cmd = "llvm-gcc -emit-llvm -O0 -S %s %s.c -o %s.ll 2>&1" % ( + includes, plain, plain) if os.system(cmd) != 0: raise Exception("Failed to run '%s'" % cmd) @@ -39,10 +61,10 @@ llcode = open(plain + '.ll').read() # strip lines - ll_lines = [] - funcnames = dict([(k, True) for k in function_names]) + lines = [] - # strip declares that are in funcnames + calltag, declaretag, definetag = 'call ', 'declare ', 'define ' + for line in llcode.split('\n'): # get rid of any of the structs that llvm-gcc introduces to struct types @@ -54,102 +76,62 @@ line = line[:comment] line = line.rstrip() - # find function names, declare them with the default calling convertion - #if '(' in line and line[-1:] == '{': - # returntype, s = line.split(' ', 1) - # funcname , s = s.split('(', 1) - # funcnames[funcname] = True - # if line.find("internal") == -1: - # if funcname not in ["@main", "@ctypes_RPython_StartupCode"]: - # internal = 'internal ' - # line = '%s%s %s' % (internal, default_cconv, line,) - ll_lines.append(line) - - # patch calls to function that we just declared with different cconv - ll_lines2, calltag, declaretag, definetag = [], 'call ', 'declare ', 'define ' - for line in ll_lines: + # patch calls (upgrade to default_cconv) i = line.find(calltag) if i >= 0: - cconv = 'ccc' - for funcname in funcnames.iterkeys(): + for funcname in call_funcnames: if line.find(funcname) >= 0: - cconv = default_cconv + line = "%scall %s %s" % (line[:i], default_cconv, line[i+len(calltag):]) break - line = "%scall %s %s" % (line[:i], cconv, line[i+len(calltag):]) + if line[:len(declaretag)] == declaretag: - cconv = 'ccc' - for funcname in funcnames.keys(): - if line.find(funcname) >= 0: - cconv = default_cconv + xline = line[len(declaretag):] + for funcname in declare_funcnames: + if xline.find(funcname) != -1: + line = "declare %s %s %s" % (internal, default_cconv, xline) break - line = "declare %s %s" % (cconv, line[len(declaretag):]) + if line[:len(definetag)] == definetag: - line = line.replace("internal ", "") - cconv = 'ccc' - for funcname in funcnames.keys(): - if line.find(funcname) >= 0: - cconv = default_cconv + xline = line[len(definetag):] + internal = '' + if xline.startswith('internal '): + internal = 'internal ' + xline = xline.replace('internal ', '') + + for funcname in define_funcnames: + if xline.find(funcname) != -1: + line = "define %s %s %s" % (internal, default_cconv, xline) break - line = "define %s %s" % (cconv, line[len(definetag):]) - ll_lines2.append(line) - ll_lines2.append("declare ccc void @abort()") - return'\n'.join(ll_lines2) + lines.append(line) -def get_c_cpath(): - from pypy.translator.c import genc - return os.path.dirname(genc.__file__) - -def get_llvm_cpath(): - return os.path.join(os.path.dirname(__file__), "module") + lines.append("declare ccc void @abort()") + return'\n'.join(lines) -def get_incdirs(c_include_dirs): - - import distutils.sysconfig - - includes = tuple(c_include_dirs) + ("/sw/include", - distutils.sysconfig.EXEC_PREFIX + "/include", - distutils.sysconfig.EXEC_PREFIX + "/include/gc", - distutils.sysconfig.get_python_inc(), - get_c_cpath(), - get_llvm_cpath(),) - - includestr = "" - for ii in includes: - includestr += "-I %s " % ii - return includestr - -def generate_llfile(db, entrynode, c_include_dirs, c_includes, c_sources, standalone, default_cconv): +def generate_c(db, entrynode, c_includes, c_sources, standalone): ccode = [] - function_names = [] - def predeclarefn(c_name, llname): - function_names.append(llname) - assert llname[0] == "@" - llname = llname[1:] - assert '\n' not in llname - ccode.append('#define\t%s\t%s\n' % (c_name, llname)) - if standalone: - predeclarefn("__ENTRY_POINT__", entrynode.get_ref()) - ccode.append('#define ENTRY_POINT_DEFINED 1\n\n') + ccode.append('#define __ENTRY_POINT__ %s' % entrynode.get_ref()[1:]) + ccode.append('#define ENTRY_POINT_DEFINED 1') # include python.h early - ccode.append('#include \n') + ccode.append('#include ') # ask gcpolicy for any code needed - ccode.append('%s\n' % db.gcpolicy.genextern_code()) + ccode.append('%s' % db.gcpolicy.genextern_code()) # ask rffi for includes/source for c_include in c_includes: - ccode.append('#include <%s>\n' % c_include) + ccode.append('#include <%s>' % c_include) + ccode.append('') + for c_source in c_sources: - ccode.append('\n') - ccode.append(c_source + '\n') - ccode.append('\n') + ccode.append(c_source) + + ccode.append('') # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) - llcode = get_ll("".join(ccode), function_names, default_cconv, c_include_dirs) - return llcode + return "\n".join(ccode) Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Sat Nov 24 19:10:31 2007 @@ -39,5 +39,5 @@ return "%s %s(%s)" % (rettype, self.ref, ", ".join(argtypes)) def writedecl(self, codewriter): - codewriter.declare(self.getdecl()) + codewriter.declare(self.getdecl(), cconv="ccc") Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sat Nov 24 19:10:31 2007 @@ -12,7 +12,7 @@ from pypy.translator.llvm import extfuncnode from pypy.translator.llvm.module.support import extfunctions from pypy.translator.llvm.node import Node -from pypy.translator.llvm.externs2ll import generate_llfile +from pypy.translator.llvm.externs2ll import generate_c, generate_ll from pypy.translator.llvm.gc import GcPolicy from pypy.translator.llvm.log import log from pypy.rlib.nonconst import NonConstant @@ -198,14 +198,9 @@ for source in sources: c_sources[source] = True - self.llcode = generate_llfile(self.db, - self.entrynode, - c_include_dirs, - c_includes, - c_sources, - self.standalone, - codewriter.cconv) - + ccode = generate_c(self.db, self.entrynode, c_includes, c_sources, self.standalone) + self.llcode = generate_ll(ccode, codewriter.cconv, c_include_dirs) + def create_codewriter(self): # prevent running the same function twice in a test filename = udir.join(self.entry_name).new(ext='.ll') Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sat Nov 24 19:10:31 2007 @@ -295,8 +295,23 @@ opr.argrefs[0], "null") def direct_call(self, opr): + cconv = None + + # XXX aargh - more illegal fishing + # XXX sort this out later... + arg = opr.op.args[0] + from pypy.objspace.flow.model import Constant + if isinstance(arg, Constant): + T = arg.concretetype.TO + assert isinstance(T, lltype.FuncType) + value = opr.op.args[0].value._obj + if getattr(value, 'external', None) == 'C': + cconv = 'ccc' + + # if we are external node - should use standard calling conventions self.codewriter.call(opr.retref, opr.rettype, opr.argrefs[0], - opr.argtypes[1:], opr.argrefs[1:]) + opr.argtypes[1:], opr.argrefs[1:], cconv=cconv) + # the following works since the extra arguments that indirect_call has # is of type Void, which is removed by direct_call indirect_call = direct_call From pedronis at codespeak.net Sat Nov 24 19:42:04 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 24 Nov 2007 19:42:04 +0100 (CET) Subject: [pypy-svn] r49056 - pypy/dist/pypy/annotation Message-ID: <20071124184204.02B8081E0@code0.codespeak.net> Author: pedronis Date: Sat Nov 24 19:42:03 2007 New Revision: 49056 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/unaryop.py Log: remove rctypes annotations Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Sat Nov 24 19:42:03 2007 @@ -13,7 +13,7 @@ from pypy.annotation.model import SomePBC, SomeSlice, SomeFloat, s_None from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeCTypesObject, SomeSingleFloat +from pypy.annotation.model import SomeSingleFloat from pypy.annotation.model import unionof, UnionError, set, missing_operation, TLS from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata @@ -1011,53 +1011,3 @@ class __extend__(pairtype(SomeObject, SomeAddress)): def union((s_obj, s_addr)): raise UnionError, "union of address and anything else makes no sense" - - - -class __extend__(pairtype(SomeCTypesObject, SomeInteger)): - def setitem((s_cto, s_index), s_value): - pass - - def getitem((s_cto, s_index)): - # Note: The following works for index either pointers and arrays, - # because both have a _type_ attribute that contains the type of the - # object pointed to or in the case of an array the element type. - result_ctype = s_cto.knowntype._type_ - s_result = SomeCTypesObject(result_ctype, ownsmemory=False) - return s_result.return_annotation() - -class __extend__(pairtype(SomeCTypesObject, SomeSlice)): - def setitem((s_cto, s_slice), s_iterable): - raise NotImplementedError("ctypes array slice assignment") - - def getitem((s_cto, s_slice)): - result_ctype = s_cto.knowntype._type_ - s_result = SomeCTypesObject(result_ctype, ownsmemory=False) - list_item = s_result.return_annotation() - if isinstance(list_item, SomeChar): - return SomeString() - raise NotImplementedError("ctypes array slicing: " - "only for arrays of char") - -class __extend__(pairtype(SomeCTypesObject, SomeCTypesObject)): - def union((s_cto1, s_cto2)): - if s_cto1.knowntype == s_cto2.knowntype: - return SomeCTypesObject(s_cto1.knowntype, - ownsmemory = (s_cto1.ownsmemory and - s_cto2.ownsmemory)) - else: - return SomeObject() - -class __extend__(pairtype(SomeCTypesObject, SomePBC)): - def union((obj, pbc)): - if pbc.isNone() and obj.can_be_none(): - return obj - else: - return SomeObject() - -class __extend__(pairtype(SomePBC, SomeCTypesObject)): - def union((pbc, obj)): - if pbc.isNone() and obj.can_be_none(): - return obj - else: - return SomeObject() Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Sat Nov 24 19:42:03 2007 @@ -473,34 +473,6 @@ attributes as well as methods """ -class SomeCTypesObject(SomeExternalObject): - """Stands for an object of the ctypes module.""" - - def __init__(self, knowntype, ownsmemory): - self.knowntype = knowntype - self.ownsmemory = ownsmemory - # 'ownsmemory' specifies if the object is *statically known* to own - # its C memory. If it is False, it will be rtyped as an alias object. - # Alias objects are allowed, at run-time, to have keepalives, so - # that they can indirectly own their memory too (it's just less - # efficient). - - def can_be_none(self): - # only 'py_object' can also be None - import ctypes - return issubclass(self.knowntype, ctypes.py_object) - - def return_annotation(self): - """Returns either 'self' or the annotation of the unwrapped version - of this ctype, following the logic used when ctypes operations - return a value. - """ - from pypy.rpython import extregistry - assert extregistry.is_registered_type(self.knowntype) - entry = extregistry.lookup_type(self.knowntype) - # special case for returning primitives or c_char_p - return getattr(entry, 's_return_trick', self) - class SomeImpossibleValue(SomeObject): """The empty set. Instances are placeholders for objects that will never show up at run-time, e.g. elements of an empty list.""" Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Sat Nov 24 19:42:03 2007 @@ -7,7 +7,7 @@ SomeDict, SomeUnicodeCodePoint, SomeTuple, SomeImpossibleValue, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ - SomeCTypesObject, s_ImpossibleValue, s_Bool, s_None, \ + s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ SomeGenericCallable, SomeWeakRef, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper @@ -735,27 +735,6 @@ v = smeth(*llargs) return ll_to_annotation(v) -class __extend__(SomeCTypesObject): - def setattr(cto, s_attr, s_value): - pass - - def getattr(cto, s_attr): - if s_attr.is_constant() and isinstance(s_attr.const, str): - attr = s_attr.const - entry = extregistry.lookup_type(cto.knowntype) - s_value = entry.get_field_annotation(cto, attr) - return s_value - else: - return SomeObject() - - def is_true(cto): - return s_Bool - - def simple_call(cto, *args_s): - # for variables containing ctypes function pointers - entry = extregistry.lookup_type(cto.knowntype) - return entry.compute_result_annotation(*args_s) - #_________________________________________ # weakrefs From pedronis at codespeak.net Sat Nov 24 19:49:35 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 24 Nov 2007 19:49:35 +0100 (CET) Subject: [pypy-svn] r49057 - pypy/dist/pypy/rpython/numpy Message-ID: <20071124184935.0074D81E0@code0.codespeak.net> Author: pedronis Date: Sat Nov 24 19:49:35 2007 New Revision: 49057 Modified: pypy/dist/pypy/rpython/numpy/implementation.py Log: oops Modified: pypy/dist/pypy/rpython/numpy/implementation.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/implementation.py (original) +++ pypy/dist/pypy/rpython/numpy/implementation.py Sat Nov 24 19:49:35 2007 @@ -1,6 +1,5 @@ # Base classes describing annotation and rtyping -from pypy.annotation.model import SomeCTypesObject from pypy.rpython import extregistry from pypy.rpython.extregistry import ExtRegistryEntry From fijal at codespeak.net Sat Nov 24 23:14:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Nov 2007 23:14:19 +0100 (CET) Subject: [pypy-svn] r49060 - in pypy/branch/rewrite-compilation-logic/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/tool translator/tool/test Message-ID: <20071124221419.8FB8181AF@code0.codespeak.net> Author: fijal Date: Sat Nov 24 23:14:18 2007 New Revision: 49060 Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py Log: * A bit of cleanups * Extremely smart hack to make tests work. I fear this hack would be too smart or become dependant on caching on different level or sth. * Add extern keyword next to external variables Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py Sat Nov 24 23:14:18 2007 @@ -14,14 +14,13 @@ from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.annotation import model as annmodel -from pypy.translator.tool.cbuild import cache_c_module - def uaddressof(obj): return fixid(ctypes.addressof(obj)) _ctypes_cache = {} +_eci_cache = {} def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -525,10 +524,15 @@ except AttributeError: pass - eci = funcptr._obj.compilation_info - libraries = list(eci.libraries) + old_eci = funcptr._obj.compilation_info funcname = funcptr._obj._name - eci = eci.make_shared_lib() + try: + eci = _eci_cache[old_eci] + except KeyError: + eci = old_eci.compile_shared_lib() + _eci_cache[old_eci] = eci + + libraries = list(eci.libraries) FUNCTYPE = lltype.typeOf(funcptr).TO if not libraries: Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py Sat Nov 24 23:14:18 2007 @@ -250,6 +250,8 @@ c_type = 'char **' elif TYPE == CCHARP: c_type = 'char *' + elif TYPE == INT: + c_type = 'int' else: c_type = PrimitiveType[TYPE] assert c_type.endswith(' @') @@ -261,6 +263,7 @@ c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals() lines = ["#include <%s>" % i for i in eci.includes] + lines.append('extern %s %s;' % (c_type, name)) lines.append(c_getter) lines.append(c_setter) sources = ('\n'.join(lines),) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Nov 24 23:14:18 2007 @@ -10,6 +10,7 @@ from pypy.rpython.annlowlevel import llhelper from pypy.rlib import rposix from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -677,14 +678,19 @@ assert res == 42 def test_prebuilt_constant(self): - source = py.code.Source(""" - int x = 3; - char** z = NULL; + header = py.code.Source(""" + #include + + static int x = 3; + char **z = NULL; """) - - eci = ExternalCompilationInfo(post_include_lines=source.lines) + h_file = udir.join("some_h.h") + h_file.write(header) + + eci = ExternalCompilationInfo(includes=['stdio.h', str(h_file.basename)], + include_dirs=[str(udir)]) - get_x, set_x = rffi.CExternVariable(lltype.Signed, 'x', eci) + get_x, set_x = rffi.CExternVariable(rffi.LONG, 'x', eci) get_z, set_z = rffi.CExternVariable(rffi.CCHARPP, 'z', eci) def f(): @@ -700,5 +706,6 @@ finally: rffi.free_charpp(l) - assert f() == 16 + res = f() + assert res == 16 assert g() == "c" Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Sat Nov 24 23:14:18 2007 @@ -60,8 +60,25 @@ assert isinstance(value, (list, tuple)) setattr(self, name, tuple(value)) - # XXX custom hash and eq functions, they should be compared - # by contents + def _value(self): + return tuple([getattr(self, x) for x in self._ATTRIBUTES]) + + def __hash__(self): + return hash(self._value()) + + def __eq__(self, other): + return self.__class__ is other.__class__ and \ + self._value() == other._value() + + def __ne__(self, other): + return not self == other + + def __repr__(self): + info = [] + for attr in self._ATTRIBUTES: + val = getattr(self, attr) + info.append("%s=%s" % (attr, repr(val))) + return "" % ", ".join(info) def merge(self, *others): others = list(others) @@ -91,7 +108,15 @@ for line in self.post_include_lines: print >> fileobj, line + def _copy_attributes(self): + d = {} + for attr in self._ATTRIBUTES: + d[attr] = getattr(self, attr) + return d + def convert_sources_to_files(self, cache_dir=None): + if not self.separate_module_sources: + return self if cache_dir is None: cache_dir = udir.join('module_cache').ensure(dir=1) num = 0 @@ -104,13 +129,22 @@ break filename.write(source) files.append(str(filename)) - d = {} - for attr in self._ATTRIBUTES: - d[attr] = getattr(self, attr) + d = self._copy_attributes() d['separate_module_sources'] = () d['separate_module_files'] += tuple(files) return ExternalCompilationInfo(**d) + def compile_shared_lib(self): + self = self.convert_sources_to_files() + if not self.separate_module_files: + return self + lib = compile_c_module([], 'externmod', self) + d = self._copy_attributes() + d['libraries'] += (lib,) + d['separate_module_files'] = () + d['separate_module_sources'] = () + return ExternalCompilationInfo(**d) + if sys.platform == 'win32': so_ext = '.dll' else: @@ -145,7 +179,7 @@ opt += '/Op' gcv['OPT'] = opt -def compile_c_module(cfiles, modname, eci): +def compile_c_module(cfiles, modbasename, eci): #try: # from distutils.log import set_threshold # set_threshold(10000) @@ -153,12 +187,8 @@ # print "ERROR IMPORTING" # pass cfiles = [py.path.local(f) for f in cfiles] - tmpdir = udir.join("modcache").ensure(dir=1) + tmpdir = udir.join("module_cache").ensure(dir=1) num = 0 - for source in eci.separate_module_sources: - c_file = tmpdir.join('mod_%d.c' % num) - c_file.write(source) - cfiles.append(c_file) cfiles += eci.separate_module_files include_dirs = list(eci.include_dirs) include_dirs.append(py.path.local(pypydir).join('translator', 'c')) @@ -172,8 +202,15 @@ os.path.exists(s + 'lib'): library_dirs.append(s + 'lib') - dirpath = py.path.local(modname).dirpath() - lastdir = dirpath.chdir() + num = 0 + modname = modbasename + while 1: + if not tmpdir.join(modname + so_ext).check(): + break + num += 1 + modname = '%s_%d' % (modbasename, num) + + lastdir = tmpdir.chdir() libraries = eci.libraries ensure_correct_math() try: @@ -186,7 +223,7 @@ from distutils import sysconfig gcv = sysconfig.get_config_vars() cmd = compiler_command().replace('%s', - str(dirpath.join(modname))) + str(tmpdir.join(modname))) for dir in [gcv['INCLUDEPY']] + list(include_dirs): cmd += ' -I%s' % dir for dir in library_dirs: @@ -254,22 +291,7 @@ raise finally: lastdir.chdir() - return modname + so_ext - -def cache_c_module(cfiles, modname, eci, cache_dir=None): - """ Same as build c module, but instead caches results. - XXX currently there is no way to force a recompile, so this is pretty - useless as soon as the sources (or headers they depend on) change :-/ - XXX for now I'm forcing a recompilation all the time. Better than not... - """ - from pypy.tool.autopath import pypydir - if cache_dir is None: - cache_dir = py.path.local(pypydir).join('_cache') - else: - cache_dir = py.path.local(cache_dir) - assert cache_dir.check(dir=1) # XXX - modname = str(cache_dir.join(modname)) - return compile_c_module(cfiles, modname, eci) + return str(tmpdir.join(modname) + so_ext) def make_module_from_c(cfile, eci): cfile = py.path.local(cfile) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/test/test_cbuild.py Sat Nov 24 23:14:18 2007 @@ -1,8 +1,8 @@ import py, sys from pypy.tool.udir import udir -from pypy.translator.tool.cbuild import build_executable, cache_c_module,\ - ExternalCompilationInfo +from pypy.translator.tool.cbuild import build_executable, \ + ExternalCompilationInfo, compile_c_module from subprocess import Popen, PIPE, STDOUT def test_simple_executable(): @@ -37,11 +37,10 @@ include_dirs=include_dirs, libraries=['pthread'] ) - cache_c_module(files, '_thread', eci, cache_dir=udir) - cdll = ctypes.CDLL(str(udir.join('_thread.so'))) + mod = compile_c_module(files, '_thread', eci) + cdll = ctypes.CDLL(mod) assert hasattr(cdll, 'RPyThreadLockInit') - class TestEci: def setup_class(cls): tmpdir = udir.ensure('testeci', dir=1) @@ -107,7 +106,23 @@ assert len(res) == 2 assert res[0] == 'x.c' assert str(res[1]).startswith(str(cache_dir)) + e = ExternalCompilationInfo() + assert e.convert_sources_to_files() is e - def test_compile_c_files_to_ofiles(self): - pass - + def test_make_shared_lib(self): + eci = ExternalCompilationInfo( + separate_module_sources = [''' + int get() + { + return 42; + }'''] + ) + neweci = eci.compile_shared_lib() + assert len(neweci.libraries) == 1 + try: + import ctypes + except ImportError: + py.test.skip("Need ctypes for that test") + assert ctypes.CDLL(neweci.libraries[0]).get() == 42 + assert not neweci.separate_module_sources + assert not neweci.separate_module_files From fijal at codespeak.net Sun Nov 25 01:19:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 01:19:36 +0100 (CET) Subject: [pypy-svn] r49061 - in pypy/branch/rewrite-compilation-logic/pypy: rlib rpython/lltypesystem rpython/lltypesystem/test rpython/module translator/c translator/tool Message-ID: <20071125001936.C2C5F81A5@code0.codespeak.net> Author: fijal Date: Sun Nov 25 01:19:35 2007 New Revision: 49061 Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/rstack.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Log: * Move few modules to use new interface * Whack c backend until it can cope with ExternalCompilationInfo * Whack a bit test_rffi, still failures Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/rstack.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rlib/rstack.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rlib/rstack.py Sun Nov 25 01:19:35 2007 @@ -9,7 +9,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.controllerentry import Controller, SomeControlledInstance - +from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): @@ -33,7 +33,11 @@ else: return len(inspect.stack()) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, includes=['src/stack.h'], _callable=lambda: 0) +compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) + +stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, + compilation_info=compilation_info, + _callable=lambda: 0) def stack_check(): if stack_too_big(): Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py Sun Nov 25 01:19:35 2007 @@ -45,6 +45,8 @@ don't bother releasing the GIL. An explicit True or False overrides this logic. """ + if _callable is not None: + assert callable(_callable) ext_type = lltype.FuncType(args, result) if _callable is None: _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py Sun Nov 25 01:19:35 2007 @@ -17,14 +17,15 @@ def test_basic(): c_source = py.code.Source(""" - int z(int x) + int someexternalfunction(int x) { return (x + 3); } """) eci = ExternalCompilationInfo(separate_module_sources=[c_source]) - z = llexternal('z', [Signed], Signed, eci) + z = llexternal('someexternalfunction', [Signed], Signed, + compilation_info=eci) def f(): return z(8) @@ -33,14 +34,16 @@ assert xf() == 8+3 def test_hashdefine(): - c_source = """ + h_source = """ #define X(i) (i+3) """ - c_file = udir.join("stuff.c") - c_file.write(c_source) + h_file = udir.join("stuff.h") + h_file.write(h_source) - z = llexternal('X', [Signed], Signed, includes=[str(c_file)]) + eci = ExternalCompilationInfo(includes=['stuff.h'], + include_dirs=[udir]) + z = llexternal('X', [Signed], Signed, compilation_info=eci) def f(): return z(8) @@ -49,7 +52,8 @@ assert xf() == 8+3 def test_string(): - z = llexternal('strlen', [CCHARP], Signed, includes=['string.h']) + eci = ExternalCompilationInfo(includes=['string.h']) + z = llexternal('strlen', [CCHARP], Signed, compilation_info=eci) def f(): s = str2charp("xxx") @@ -72,7 +76,8 @@ return ret; } """) - z = llexternal('f', [CCHARP], CCHARP, sources=[c_source]) + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) def f(): s = str2charp("xxx") @@ -99,7 +104,8 @@ return (l); } """ - z = llexternal('f', [CCHARPP], Signed, sources=[c_source]) + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('f', [CCHARPP], Signed, compilation_info=eci) def f(): l = ["xxx", "x", "xxxx"] @@ -113,16 +119,21 @@ def test_struct(): h_source = """ + #ifndef _MY_SOURCE_H + #define _MY_SOURCE_H struct xx { int one; char two; int three; }; + #endif """ h_file = udir.join("structxx.h") h_file.write(h_source) c_source = """ + #include + int f(struct xx* z) { return (z->one + z->three); @@ -130,8 +141,12 @@ """ TP = CStructPtr('xx', ('one', INT), ('two', Char), ('three', INT)) - z = llexternal('f', [TP], INT, sources=[c_source], - includes=[str(h_file)], include_dirs=[udir]) + eci = ExternalCompilationInfo( + includes=['structxx.h'], + include_dirs=[udir], + separate_module_sources=[c_source] + ) + z = llexternal('f', [TP], INT, compilation_info=eci) def f(): struct = lltype.malloc(TP.TO, flavor='raw') Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py Sun Nov 25 01:19:35 2007 @@ -3,7 +3,6 @@ from pypy.rpython.extfunc import BaseLazyRegistering, extdef, registering from pypy.rlib import rarithmetic from pypy.rpython.lltypesystem import lltype, rffi -from pypy.translator.tool.cbuild import cache_c_module from pypy.tool.autopath import pypydir from pypy.rpython.ootypesystem import ootype from pypy.rlib import rposix Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py Sun Nov 25 01:19:35 2007 @@ -171,12 +171,9 @@ def compile(self): assert self.c_source_filename assert not self._compiled - extra_includes = self.include_dirs compile_c_module([self.c_source_filename] + self.extrafiles, - self.c_source_filename.purebasename, - include_dirs = [autopath.this_dir] + extra_includes, - library_dirs = self.library_dirs, - libraries=self.libraries) + self.c_source_filename.purebasename, self.eci, + tmpdir=self.c_source_filename.dirpath()) self._compiled = True def _make_wrapper_module(self): @@ -716,11 +713,9 @@ print >> fi, line eci.write_c_header(fi) + eci = eci.convert_sources_to_files() fi.close() - for source in extra_info.get('sources', []): - print >> f, source - if database.translator is None or database.translator.rtyper is None: preimplementationlines = [] else: @@ -743,10 +738,13 @@ # pypy_include_dir = autopath.this_dir f = targetdir.join('setup.py').open('w') + include_dirs = eci.include_dirs + library_dirs = eci.library_dirs + libraries = eci.libraries f.write(SETUP_PY % locals()) f.close() - return filename, sg.getextrafiles() + return filename, sg.getextrafiles() + list(eci.separate_module_files) SETUP_PY = ''' Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py Sun Nov 25 01:19:35 2007 @@ -692,8 +692,9 @@ self.db = db self.T = T self.obj = obj - if hasattr(obj, 'includes') and not db.need_sandboxing(obj): - self.includes = obj.includes + # XXX # what this obscure code is doing??? + self.eci = getattr(obj, 'eci', ExternalCompilationInfo()) + if self.eci.includes and not db.need_sandboxing(obj): self.name = forcename or self.basename() else: self.name = (forcename or @@ -706,8 +707,6 @@ self.ptrname = self.name def make_funcgens(self): - if hasattr(self.obj, 'sources'): - self.sources = self.obj.sources self.funcgens = select_function_code_generators(self.obj, self.db, self.name) if self.funcgens: argnames = self.funcgens[0].argnames() #Assume identical for all funcgens @@ -827,6 +826,7 @@ elif getattr(fnobj, 'external', None) == 'C': if sandbox: return sandbox_stub(fnobj, db) + # XXX broken if hasattr(fnobj, 'includes'): return [] # assume no wrapper needed else: Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Sun Nov 25 01:19:35 2007 @@ -179,7 +179,7 @@ opt += '/Op' gcv['OPT'] = opt -def compile_c_module(cfiles, modbasename, eci): +def compile_c_module(cfiles, modbasename, eci, tmpdir=None): #try: # from distutils.log import set_threshold # set_threshold(10000) @@ -187,7 +187,8 @@ # print "ERROR IMPORTING" # pass cfiles = [py.path.local(f) for f in cfiles] - tmpdir = udir.join("module_cache").ensure(dir=1) + if tmpdir is None: + tmpdir = udir.join("module_cache").ensure(dir=1) num = 0 cfiles += eci.separate_module_files include_dirs = list(eci.include_dirs) From rxe at codespeak.net Sun Nov 25 09:58:49 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 25 Nov 2007 09:58:49 +0100 (CET) Subject: [pypy-svn] r49062 - pypy/dist/pypy/translator/llvm Message-ID: <20071125085849.C9CC58193@code0.codespeak.net> Author: rxe Date: Sun Nov 25 09:58:48 2007 New Revision: 49062 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/codewriter.py Log: fix debug print Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Sun Nov 25 09:58:48 2007 @@ -17,11 +17,10 @@ return '%s c"%s\\00"' % (self.get_typerepr(), self.value) def get_childref(self, index): - x = "getelementptr(%s* %s, i32 0, i32 %s)" % ( + return "getelementptr(%s* %s, i32 0, i32 %s)" % ( self.get_typerepr(), self.name, index) - return 'bitcast(i8* %s to [0 x i8]*)' % x def writeglobalconstants(self, codewriter): codewriter.globalinstance(self.ref, self.constantvalue()) Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Sun Nov 25 09:58:48 2007 @@ -210,5 +210,5 @@ var = self.db.repr_tmpvar() node = self.db.create_debug_string(s) self.call(var, "i32", "@write", - ['i32', '[0 x i8]*', 'i32'], - ['2', node.get_childref(0), '%d' % node.get_length()]) + ['i32', 'i8*', 'i32'], + ['2', node.get_childref(0), '%d' % node.get_length()], cconv='ccc') From rxe at codespeak.net Sun Nov 25 10:00:11 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Sun, 25 Nov 2007 10:00:11 +0100 (CET) Subject: [pypy-svn] r49063 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071125090011.2F0018195@code0.codespeak.net> Author: rxe Date: Sun Nov 25 10:00:10 2007 New Revision: 49063 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/test_standalone.py Log: more fixing debug print. use database to store externs that call back into rpython code (needed for fastcc -> ccc conversions) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sun Nov 25 10:00:10 2007 @@ -39,6 +39,9 @@ self.debugstringnodes = [] + # call back into rpython code from c code + self.extern_to_funcnodes = [] + #_______debuggging______________________________________ @@ -317,6 +320,9 @@ self._tmpcount += 1 return "%tmp_" + str(count) + def add_extern_to_funcnode(self, name): + self.extern_to_funcnodes.append(name) + # __________________________________________________________ # Other helpers Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Sun Nov 25 10:00:10 2007 @@ -33,17 +33,18 @@ includestr += "-I %s " % ii return includestr -# call entrypoint needs to be fastcc # call boehm finalizers need to be fastcc def generate_ll(ccode, default_cconv, c_include_dirs, call_funcnames=[]): - - call_funcnames += ['@LLVM_RPython_StartupCode'] + call_funcnames = ['@LLVM_RPython_StartupCode'] + call_funcnames define_funcnames = ['@pypy_malloc', '@pypy_malloc_atomic', '@pypy_gc__collect', - '@pypy_register_finalizer'] - declare_funcnames = ['@LLVM_RPython_StartupCode'] + '@pypy_register_finalizer', + '@raw_malloc', + '@raw_free', + ] + declare_funcnames = call_funcnames filename = str(udir.join("ccode.c")) f = open(filename, "w") Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Sun Nov 25 10:00:10 2007 @@ -75,7 +75,6 @@ # malloc_size is unsigned right now codewriter.malloc(targetvar, "i8", size) - # XXX uses own cconv codewriter.call(None, 'void', '@llvm.memset' + postfix(), ['i8*', 'i8', word, word], [targetvar, 0, size, boundary_size], Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Sun Nov 25 10:00:10 2007 @@ -45,8 +45,6 @@ return new_entrypoint class GenLLVM(object): - debug = False - # see create_codewriter() below function_count = {} @@ -96,7 +94,7 @@ if hasattr(node, 'writeimpl'): node.writeimpl(codewriter) - self._debug() + self._debug(codewriter) codewriter.comment("End of file") codewriter.close() @@ -125,7 +123,7 @@ # get entry point entry_point = self.get_entry_point(func) self._checkpoint('get_entry_point') - + # set up all nodes self.db.setup_all() @@ -181,6 +179,7 @@ else: self.function_count[name] = 1 + self.db.add_extern_to_funcnode(name) self.entry_name = name[6:] return c.value._obj @@ -199,7 +198,7 @@ c_sources[source] = True ccode = generate_c(self.db, self.entrynode, c_includes, c_sources, self.standalone) - self.llcode = generate_ll(ccode, codewriter.cconv, c_include_dirs) + self.llcode = generate_ll(ccode, codewriter.cconv, c_include_dirs, self.db.extern_to_funcnodes) def create_codewriter(self): # prevent running the same function twice in a test @@ -267,12 +266,11 @@ for s in stats: log('STATS %s' % str(s)) - def _debug(self): - if self.debug: - if self.db.debugstringnodes: - codewriter.header_comment("Debug string") - for node in self.db.debugstringnodes: - node.writeglobalconstants(codewriter) + def _debug(self, codewriter): + if self.db.debugstringnodes: + codewriter.header_comment("Debug string") + for node in self.db.debugstringnodes: + node.writeglobalconstants(codewriter) #print "Start" #print self.db.dump_pbcs() Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Sun Nov 25 10:00:10 2007 @@ -126,6 +126,7 @@ if self.db.genllvm.config.translation.llvm.debug: self.codewriter.comment(str(op)) + #self.codewriter.debug_print(str(op) + "\n") if op.opname in ("direct_call", 'indirect_call'): opr = OpReprCall(op, self.db) @@ -307,6 +308,9 @@ value = opr.op.args[0].value._obj if getattr(value, 'external', None) == 'C': cconv = 'ccc' + + #self.codewriter.debug_print(str(opr.op) + "\n") + #self.codewriter.debug_print(str(cconv) + "\n") # if we are external node - should use standard calling conventions self.codewriter.call(opr.retref, opr.rettype, opr.argrefs[0], @@ -323,6 +327,8 @@ self.db.gcpolicy._zeromalloc(self.codewriter, opr.retref, opr.argrefs[0], atomic=True) def boehm_register_finalizer(self, opr): + # XXX point in note - the registeree here have fastcc.... not sure if llvm is dealing with this + # because it is a pointer... - presumably tmpvar = self._tmp() self.codewriter.cast(tmpvar, opr.argtypes[1], opr.argrefs[1], 'i8 *') self.codewriter.call(None, 'void', '@pypy_register_finalizer', ['i8 *', 'i8 *'], [opr.argrefs[0], tmpvar]) Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_standalone.py Sun Nov 25 10:00:10 2007 @@ -37,3 +37,44 @@ data = cmdexec(exe_name, 'hi', 'there') assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''') +def test__del__(): + from pypy.rpython.lltypesystem import lltype + from pypy.rpython.lltypesystem.lloperation import llop + class State: + pass + s = State() + class A(object): + def __del__(self): + s.a_dels += 1 + class B(A): + def __del__(self): + s.b_dels += 1 + class C(A): + pass + def f(): + s.a_dels = 0 + s.b_dels = 0 + A() + B() + C() + A() + B() + C() + llop.gc__collect(lltype.Void) + return s.a_dels * 10 + s.b_dels + + def entry_point(args): + res = 0 + res += f() + res += f() + print 'count %d' % res + return 0 + + exe_name = 'test__del__' + compile_standalone(entry_point, exe_name=exe_name) + data = cmdexec(exe_name, 'abc', 'def') + print data + #assert data.startswith('argument count: 3') + + #assert 0 < res <= 84 + From cfbolz at codespeak.net Sun Nov 25 10:51:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 25 Nov 2007 10:51:43 +0100 (CET) Subject: [pypy-svn] r49064 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071125095143.8916E8193@code0.codespeak.net> Author: cfbolz Date: Sun Nov 25 10:51:42 2007 New Revision: 49064 Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py Log: remove an assertion since it is not true for some of the things rffi did. Maciek did this on his branch, do it on the trunk too so that the tests don't fail anymore. Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Sun Nov 25 10:51:42 2007 @@ -413,9 +413,8 @@ def __init__(self, args, result): for arg in args: assert isinstance(arg, LowLevelType) - # -- disable the following check for the benefits of rffi -- - if isinstance(arg, ContainerType): - raise TypeError, "function arguments can only be primitives or pointers" + # There are external C functions eating raw structures, not + # pointers, don't check args not being container types self.ARGS = tuple(args) assert isinstance(result, LowLevelType) if isinstance(result, ContainerType): From cfbolz at codespeak.net Sun Nov 25 11:58:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 25 Nov 2007 11:58:48 +0100 (CET) Subject: [pypy-svn] r49065 - pypy/extradoc/sprintinfo/gothenburg-2007 Message-ID: <20071125105848.36E8E817E@code0.codespeak.net> Author: cfbolz Date: Sun Nov 25 11:58:46 2007 New Revision: 49065 Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt (original) +++ pypy/extradoc/sprintinfo/gothenburg-2007/planning.txt Sun Nov 25 11:58:46 2007 @@ -10,21 +10,23 @@ Discussion topics (dynamically scheduled): - - discuss callback mechanism for rffi/lltype/ll2ctypes - - discuss approaches for extension modules + - discuss callback mechanism for rffi/lltype/ll2ctypes DONE, minutes missing + (maciek) + - discuss approaches for extension modules DONE (probably Armin, Maciek, Alexander) - - discuss Q/A (Holger, Samuele, Carl Friedrich) + - discuss Q/A (Holger, Samuele, Carl Friedrich) DONE - discuss how to deal with the windows situation + - write blog posts (carl friedrich; maciek, samuele to review) Translation toolchain +++++++++++++++++++++ - - refactor rffi header, sources, includes management IN-PROGRESS (maciek, + - refactor rffi header, sources, includes management MORE PROGRESS (maciek, alexander, armin) - - py-lib 0.9.1 release? (holger, maciek) + - py-lib 0.9.1 release? - tried to translate PyPy with LLVM DONE @@ -36,18 +38,18 @@ - find out why our memcpy behaves so erratically GCC behaves very erratically, so we won't work on this for now - - finish rctypes removal DONE on a branch + - finish rctypes removal DONE - raw_malloc should be split, rffi one should not use obmalloc (it's not thread-safe) - delegate finding type stuff like vtables etc to GC, cleaner interface - for rtti, simplify translator/c/gc.py - - kill opaque hacks in the C backend + for rtti, simplify translator/c/gc.py (carl friedrich, samuele to discuss) + - kill opaque hacks in the C backend - cleanup pyobj.py and rclass stuff and remove the ability to make cpython - extensions, just dll - update gencapicall impl (cfbolz, samuele) MORE-PROGRESS + extensions, just dll DONE + - update gencapicall impl - clean up the tangle of including headers in the C backend @@ -60,6 +62,10 @@ - llvm profiling (richard) + - fix generational GC bug? (armin, maciek) + + - general wizardry (samuele) + Interpreter +++++++++++ @@ -71,14 +77,13 @@ - review the things implemented at applevel whether they are performance-critical IN-PROGRESS: moved super, classmethod, statimethod to - interplevel, some dict methods - to be done: property, exceptions? - - - remove prebuilt costate vars (christian, armin?) + interplevel, some dict methods, property + to be done: exceptions? - - look at the new failing test in test_descr.py + - remove prebuilt costate vars (christian, armin?) STARTED - - fix __future__ imports IN-PROGRESS (arre) + - trying to port only the fixes of __future__ imports to dist (arre, + alexander) Modules +++++++ From fijal at codespeak.net Sun Nov 25 12:37:43 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 12:37:43 +0100 (CET) Subject: [pypy-svn] r49070 - in pypy/branch/rewrite-compilation-logic/pypy: rpython rpython/lltypesystem rpython/lltypesystem/test rpython/module translator/tool Message-ID: <20071125113743.A0842817F@code0.codespeak.net> Author: fijal Date: Sun Nov 25 12:37:43 2007 New Revision: 49070 Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Log: rffi tests are passing by now. Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py Sun Nov 25 12:37:43 2007 @@ -92,19 +92,13 @@ class BaseLazyRegistering(object): __metaclass__ = LazyRegisteringMeta - __ATTRIBUTES = ['includes', 'include_dirs', 'libraries', 'library_dirs', - 'sources'] - def configure(self, CConfig): classes_seen = self.__dict__.setdefault('__classes_seen', {}) if CConfig in classes_seen: return from pypy.rpython.tool import rffi_platform as platform # copy some stuff - for item in self.__ATTRIBUTES: - value = getattr(CConfig, '_%s_' % item, None) - if value: - setattr(self, '_%s_' % item, value) + self.compilation_info = CConfig._compilation_info_ self.__dict__.update(platform.configure(CConfig)) classes_seen[CConfig] = True @@ -112,11 +106,11 @@ kwds = kwds.copy() from pypy.rpython.lltypesystem import rffi - for item in self.__ATTRIBUTES: - if item not in kwds: - kwds[item] = getattr(self, '_%s_' % item, []) - else: - kwds[item] += getattr(self, '_%s_' % item, []) + if 'compilation_info' in kwds: + kwds['compilation_info'] = self._compilation_info_.merge( + kwds['compilation_info']) + else: + kwds['compilation_info'] = self.compilation_info return rffi.llexternal(*args, **kwds) def _freeze_(self): Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py Sun Nov 25 12:37:43 2007 @@ -261,6 +261,8 @@ getter_name = 'get_' + name setter_name = 'set_' + name + getter_prototype = "%(c_type)s %(getter_name)s ();" % locals() + setter_prototype = "void %(setter_name)s (%(c_type)s v);" % locals() c_getter = "%(c_type)s %(getter_name)s () { return %(name)s; }" % locals() c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals() @@ -270,7 +272,8 @@ lines.append(c_setter) sources = ('\n'.join(lines),) new_eci = eci.merge(ExternalCompilationInfo( - separate_module_sources = sources + separate_module_sources = sources, + post_include_lines = [getter_prototype, setter_prototype], )) getter = llexternal(getter_name, [], TYPE, compilation_info=new_eci, Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/test_rffi.py Sun Nov 25 12:37:43 2007 @@ -201,8 +201,12 @@ return (C); } """) - z = llexternal('fun', [], Signed, sources=[c_source], include_dirs= - [str(udir.join("incl"))]) + eci = ExternalCompilationInfo( + includes=['incl.h'], + include_dirs=[str(udir.join('incl'))], + separate_module_sources=[c_source] + ) + z = llexternal('fun', [], Signed, compilation_info=eci) def f(): return z() @@ -243,11 +247,13 @@ h_file.write(h_source) from pypy.rpython.tool import rffi_platform - STUFFP = COpaquePtr('struct stuff', includes=['opaque.h'], - include_dirs=[str(udir)]) + eci = ExternalCompilationInfo( + includes=['opaque.h'], + include_dirs=[str(udir)] + ) + STUFFP = COpaquePtr('struct stuff', compilation_info=eci) - ll_get = llexternal('get', [STUFFP], lltype.Char, includes=['opaque.h'], - include_dirs=[str(udir)]) + ll_get = llexternal('get', [STUFFP], lltype.Char, compilation_info=eci) def f(): ll_stuff = lltype.malloc(STUFFP.TO, flavor='raw') @@ -307,9 +313,8 @@ assert offsetof(S, "c_c") == struct.calcsize("hii") - struct.calcsize("i") def test_prebuilt_constant(): + py.test.skip("Think how to do it sane") h_source = py.code.Source(""" - #ifndef _CONSTANTS - #define _CONSTANTS int x = 3; char** z = NULL; #endif @@ -317,9 +322,12 @@ h_include = udir.join('constants.h') h_include.write(h_source) - kwds = {'includes':['constants.h'], 'include_dirs':[str(udir)]} - get_x, set_x = CExternVariable(lltype.Signed, 'x', **kwds) - get_z, set_z = CExternVariable(CCHARPP, 'z', **kwds) + eci = ExternalCompilationInfo(includes=['stdio.h', + str(h_include.basename)], + include_dirs=[str(udir)]) + + get_x, set_x = CExternVariable(lltype.Signed, 'x', eci) + get_z, set_z = CExternVariable(CCHARPP, 'z', eci) def f(): one = get_x() @@ -395,21 +403,15 @@ def test_stringpolicy1(): - strlen = llexternal('strlen', [CCHARP], SIZE_T, includes=['string.h']) + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, compilation_info=eci) def f(): return cast(LONG, strlen("Xxx")) assert interpret(f, [], backendopt=True) == 3 -def test_stringpolicy2(): - py.test.skip("stringpolicy='autocast' no longer implemented") - def f(): - return strlen("Xxx") - strlen = llexternal('strlen', [CCHARP], INT, - includes=['string.h'], stringpolicy='autocast') - py.test.raises(MallocMismatch, interpret, f, [], backendopt=True) - def test_stringpolicy3(): - strlen = llexternal('strlen', [CCHARP], INT, includes=['string.h']) + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], INT, compilation_info=eci) def f(): ll_str = str2charp("Xxx") res = strlen(ll_str) @@ -419,8 +421,9 @@ assert interpret(f, [], backendopt=True) == 3 def test_stringpolicy_mixed(): + eci = ExternalCompilationInfo(includes=['string.h']) strlen = llexternal('strlen', [CCHARP], SIZE_T, - includes=['string.h']) + compilation_info=eci) def f(): res1 = strlen("abcd") ll_str = str2charp("Xxx") Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py Sun Nov 25 12:37:43 2007 @@ -18,6 +18,7 @@ from pypy.rpython.tool import rffi_platform as platform from pypy.rlib import rposix from pypy.tool.udir import udir +from pypy.translator.tool.cbuild import ExternalCompilationInfo posix = __import__(os.name) @@ -26,6 +27,15 @@ else: underscore_on_windows = '' +includes = [] +if not sys.platform.startswith('win'): + # XXX many of these includes are not portable at all + includes += ['dirent.h', 'sys/stat.h', + 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', + 'signal.h', 'sys/wait.h'] +else: + includes += ['sys/utime.h'] + class CConfig: """ @@ -38,13 +48,11 @@ register(function, [CLOCK_T], ...) """ - _includes_ = [] - if not sys.platform.startswith('win'): - # XXX many of these includes are not portable at all - _includes_ += ['dirent.h', 'sys/stat.h', - 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', - 'signal.h', 'sys/wait.h'] + _compilation_info_ = ExternalCompilationInfo( + includes=includes + ) + if not sys.platform.startswith('win'): CLOCK_T = platform.SimpleType('clock_t', rffi.INT) TMS = platform.Struct( @@ -52,9 +60,6 @@ ('tms_stime', rffi.INT), ('tms_cutime', rffi.INT), ('tms_cstime', rffi.INT)]) - else: - _includes_ += ['sys/utime.h'] - _include_dirs_ = [str(udir)] SEEK_SET = platform.DefinedConstantInteger('SEEK_SET') SEEK_CUR = platform.DefinedConstantInteger('SEEK_CUR') Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Sun Nov 25 12:37:43 2007 @@ -18,7 +18,7 @@ 'post_include_lines', 'libraries', 'library_dirs', 'separate_module_sources', 'separate_module_files'] _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes', - 'include_dirs', 'library_dirs'] + 'include_dirs', 'library_dirs', 'separate_module_sources'] def __init__(self, pre_include_lines = [], From tismer at codespeak.net Sun Nov 25 13:38:13 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 25 Nov 2007 13:38:13 +0100 (CET) Subject: [pypy-svn] r49073 - pypy/branch/windows48396/pypy/translator/goal Message-ID: <20071125123813.E199B817F@code0.codespeak.net> Author: tismer Date: Sun Nov 25 13:38:12 2007 New Revision: 49073 Modified: pypy/branch/windows48396/pypy/translator/goal/nanos.py Log: there are side effects with geninterp'ing os.path when compiling on top of pypy-c. Whatever the reason is, this side-effects is not necessary. Modified: pypy/branch/windows48396/pypy/translator/goal/nanos.py ============================================================================== --- pypy/branch/windows48396/pypy/translator/goal/nanos.py (original) +++ pypy/branch/windows48396/pypy/translator/goal/nanos.py Sun Nov 25 13:38:12 2007 @@ -29,10 +29,12 @@ import os app_os_path = applevel(r''' + # NOT_RPYTHON from os.path import dirname, join, abspath, isfile, islink ''', filename=__file__) app_os = applevel(r''' + # NOT_RPYTHON from os import sep, pathsep try: from os import readlink From tismer at codespeak.net Sun Nov 25 13:44:47 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sun, 25 Nov 2007 13:44:47 +0100 (CET) Subject: [pypy-svn] r49074 - pypy/dist/pypy/translator/goal Message-ID: <20071125124447.5477E817F@code0.codespeak.net> Author: tismer Date: Sun Nov 25 13:44:46 2007 New Revision: 49074 Modified: pypy/dist/pypy/translator/goal/nanos.py Log: geninterping os is a bit too much Modified: pypy/dist/pypy/translator/goal/nanos.py ============================================================================== --- pypy/dist/pypy/translator/goal/nanos.py (original) +++ pypy/dist/pypy/translator/goal/nanos.py Sun Nov 25 13:44:46 2007 @@ -29,10 +29,12 @@ import os app_os_path = applevel(r''' + # NOT_RPYTHON from os.path import dirname, join, abspath, isfile, islink ''', filename=__file__) app_os = applevel(r''' + # NOT_RPYTHON from os import sep, pathsep try: from os import readlink From xoraxax at codespeak.net Sun Nov 25 14:01:40 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 25 Nov 2007 14:01:40 +0100 (CET) Subject: [pypy-svn] r49075 - pypy/dist/pypy/tool Message-ID: <20071125130140.51A2D816E@code0.codespeak.net> Author: xoraxax Date: Sun Nov 25 14:01:33 2007 New Revision: 49075 Modified: pypy/dist/pypy/tool/ansi_mandelbrot.py Log: Small workaround for a pylib bug that defaults the terminal width in ansi_mandelbrot to make pypy-c translate.py work. Modified: pypy/dist/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/dist/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/dist/pypy/tool/ansi_mandelbrot.py Sun Nov 25 14:01:33 2007 @@ -110,7 +110,7 @@ self.init() def init(self): - self.width = get_terminal_width() + self.width = get_terminal_width() or 80 # in some envs, the py lib doesnt default the width correctly self.mandelbrot = Mandelbrot(width=(self.width or 1), **self.kwargs) self.mandelbrot.init() self.gen = self.mandelbrot.generate() From fijal at codespeak.net Sun Nov 25 14:06:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 14:06:54 +0100 (CET) Subject: [pypy-svn] r49076 - in pypy/branch/rewrite-compilation-logic/pypy: rpython/module translator/c translator/llsupport translator/llsupport/test translator/tool Message-ID: <20071125130654.A2C59816E@code0.codespeak.net> Author: fijal Date: Sun Nov 25 14:06:53 2007 New Revision: 49076 Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py pypy/branch/rewrite-compilation-logic/pypy/translator/llsupport/ (props changed) pypy/branch/rewrite-compilation-logic/pypy/translator/llsupport/test/ (props changed) pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Log: * fixeol * make a hack which allows us to pass ll_strtod tests * comment out part of logic, test_standalone passes Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_strtod.py Sun Nov 25 14:06:53 2007 @@ -6,10 +6,13 @@ from pypy.tool.autopath import pypydir from pypy.rpython.ootypesystem import ootype from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo class CConfig: - _includes_ = ['src/ll_strtod.h'] - _sources_ = ['#include '] + _compilation_info_ = ExternalCompilationInfo( + includes = ['src/ll_strtod.h'], + separate_module_sources = ['#include '], + ) class RegisterStrtod(BaseLazyRegistering): def __init__(self): Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py Sun Nov 25 14:06:53 2007 @@ -134,6 +134,14 @@ if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 + # explicitely include python.h and exceptions.h + # XXX for now, we always include Python.h + from distutils import sysconfig + python_inc = sysconfig.get_python_inc() + pypy_include_dir = autopath.this_dir + self.eci = self.eci.merge(ExternalCompilationInfo( + include_dirs=[python_inc, pypy_include_dir], + )) cfile, extra = gen_source_standalone(db, modulename, targetdir, self.eci, entrypointname = pfname, @@ -252,10 +260,7 @@ bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) - def getccompiler(self, extra_includes): - # XXX for now, we always include Python.h - from distutils import sysconfig - python_inc = sysconfig.get_python_inc() + def getccompiler(self): cc = self.config.translation.cc profbased = None if self.config.translation.instrumentctl is not None: @@ -267,16 +272,14 @@ return CCompiler( [self.c_source_filename] + self.extrafiles, - include_dirs = [autopath.this_dir, python_inc] + extra_includes, - libraries = self.libraries, - library_dirs = self.library_dirs, - compiler_exe = cc, profbased = profbased) + self.eci, compiler_exe = cc, profbased = profbased) def compile(self): assert self.c_source_filename assert not self._compiled - compiler = self.getccompiler(extra_includes=[str(self.targetdir)] + - self.include_dirs) + eci = self.eci.merge(ExternalCompilationInfo(includes= + [str(self.targetdir)])) + compiler = self.getccompiler() if sys.platform == 'darwin': compiler.compile_extra.append('-mdynamic-no-pic') if self.config.translation.compilerflags: @@ -302,7 +305,9 @@ print >> f prefix = ' ' * len(prefix) - compiler = self.getccompiler(extra_includes=['.', str(self.targetdir)] + self.include_dirs) + self.eci = self.eci.merge(ExternalCompilationInfo( + includes=['.', str(self.targetdir)])) + compiler = self.getccompiler() if sys.platform == 'darwin': compiler.compile_extra.append('-mdynamic-no-pic') if self.config.translation.compilerflags: @@ -335,11 +340,11 @@ print >> f write_list(ofiles, 'OBJECTS =') print >> f - args = ['-l'+libname for libname in compiler.libraries] + args = ['-l'+libname for libname in self.eci.libraries] print >> f, 'LIBS =', ' '.join(args) - args = ['-L'+path for path in compiler.library_dirs] + args = ['-L'+path for path in self.eci.library_dirs] print >> f, 'LIBDIRS =', ' '.join(args) - args = ['-I'+path for path in compiler.include_dirs] + args = ['-I'+path for path in self.eci.include_dirs] write_list(args, 'INCLUDEDIRS =') print >> f print >> f, 'CFLAGS =', ' '.join(compiler.compile_extra) @@ -648,15 +653,13 @@ for line in database.gcpolicy.pre_pre_gc_code(): print >> fi, line + eci.write_c_header(fi) + print >> fi, '#include "src/g_prerequisite.h"' for line in database.gcpolicy.pre_gc_code(): print >> fi, line - for include in extra_info.get('includes', []): - print >> fi, '#include <%s>' % (include,) - for source in extra_info.get('sources', []): - print >> f, source fi.close() preimplementationlines = list( @@ -683,7 +686,8 @@ print >>fi, "#define INSTRUMENT_NCOUNTER %d" % n fi.close() - return filename, sg.getextrafiles() + eci = eci.convert_sources_to_files(being_main=True) + return filename, sg.getextrafiles() + list(eci.separate_module_files) def gen_source(database, modulename, targetdir, eci, defines={}, exports={}): @@ -713,7 +717,6 @@ print >> fi, line eci.write_c_header(fi) - eci = eci.convert_sources_to_files() fi.close() if database.translator is None or database.translator.rtyper is None: @@ -743,6 +746,7 @@ libraries = eci.libraries f.write(SETUP_PY % locals()) f.close() + eci = eci.convert_sources_to_files() return filename, sg.getextrafiles() + list(eci.separate_module_files) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py Sun Nov 25 14:06:53 2007 @@ -692,9 +692,7 @@ self.db = db self.T = T self.obj = obj - # XXX # what this obscure code is doing??? - self.eci = getattr(obj, 'eci', ExternalCompilationInfo()) - if self.eci.includes and not db.need_sandboxing(obj): + if getattr(obj, 'external', None) == 'C' and not db.need_sandboxing(obj): self.name = forcename or self.basename() else: self.name = (forcename or @@ -827,11 +825,12 @@ if sandbox: return sandbox_stub(fnobj, db) # XXX broken - if hasattr(fnobj, 'includes'): - return [] # assume no wrapper needed - else: - # deprecated case - return [CExternalFunctionCodeGenerator(fnobj, db)] + return [] + #if hasattr(fnobj, 'includes'): + # return [] # assume no wrapper needed + #else: + # # deprecated case + # return [CExternalFunctionCodeGenerator(fnobj, db)] else: raise ValueError, "don't know how to generate code for %r" % (fnobj,) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/tool/cbuild.py Sun Nov 25 14:06:53 2007 @@ -114,7 +114,7 @@ d[attr] = getattr(self, attr) return d - def convert_sources_to_files(self, cache_dir=None): + def convert_sources_to_files(self, cache_dir=None, being_main=False): if not self.separate_module_sources: return self if cache_dir is None: @@ -127,7 +127,11 @@ num += 1 if not filename.check(): break - filename.write(source) + f = filename.open("w") + if being_main: + f.write("#define PYPY_NOT_MAIN_FILE\n") + f.write(source) + f.close() files.append(str(filename)) d = self._copy_attributes() d['separate_module_sources'] = () @@ -381,7 +385,8 @@ if outputfilename is None: self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext) else: - self.outputfilename = py.path.local(outputfilename) + self.outputfilename = py.path.local(outputfilename) + self.eci = eci def build(self, noerr=False): basename = self.outputfilename.new(ext='') @@ -433,7 +438,7 @@ old = cfile.dirpath().chdir() try: res = compiler.compile([cfile.basename], - include_dirs=self.include_dirs, + include_dirs=self.eci.include_dirs, extra_preargs=self.compile_extra) assert len(res) == 1 cobjfile = py.path.local(res[0]) @@ -442,9 +447,9 @@ finally: old.chdir() compiler.link_executable(objects, str(self.outputfilename), - libraries=self.libraries, + libraries=self.eci.libraries, extra_preargs=self.link_extra, - library_dirs=self.library_dirs) + library_dirs=self.eci.library_dirs) def build_executable(*args, **kwds): noerr = kwds.pop('noerr', False) From pedronis at codespeak.net Sun Nov 25 14:40:40 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 25 Nov 2007 14:40:40 +0100 (CET) Subject: [pypy-svn] r49077 - pypy/branch/rewrite-compilation-logic/pypy/translator/c/test Message-ID: <20071125134040.C24C98170@code0.codespeak.net> Author: pedronis Date: Sun Nov 25 14:40:38 2007 New Revision: 49077 Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py Log: test about the checking of py exception around python capi calls Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py Sun Nov 25 14:40:38 2007 @@ -10,7 +10,6 @@ from pypy.objspace.flow.model import Block, Link, FunctionGraph from pypy.tool.udir import udir from pypy.translator.tool.cbuild import make_module_from_c -from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation @@ -62,6 +61,25 @@ py.test.raises(Exception, f1, "world") # check that it's really typed +def test_py_capi_exc(): + def f(x): + if x: + l = None + else: + l = [2] + x = x*2 + return l[0] + t = TranslationContext() + t.buildannotator().build_types(f, [int]) + t.buildrtyper().specialize() + + builder = genc.CExtModuleBuilder(t, f, config=t.config) + builder.generate_source() + builder.compile() + f1 = builder.get_entry_point(isolated=True) + + x = py.test.raises(Exception, f1, "world") + assert not isinstance(x.value, EOFError) # EOFError === segfault def test_rlist(): def f(x): From pedronis at codespeak.net Sun Nov 25 14:51:52 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 25 Nov 2007 14:51:52 +0100 (CET) Subject: [pypy-svn] r49078 - in pypy/dist/pypy/translator/c: . test Message-ID: <20071125135152.A4D6A813A@code0.codespeak.net> Author: pedronis Date: Sun Nov 25 14:51:52 2007 New Revision: 49078 Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/test/test_genc.py Log: mmh, s funky names Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sun Nov 25 14:51:52 2007 @@ -204,7 +204,7 @@ _entry_point.restype = ctypes.py_object _entry_point.argtypes = %(nargs)d*(ctypes.py_object,) -def %(entrypoint_name)s(*args): +def entrypoint(*args): return _entry_point(*args) try: @@ -221,7 +221,6 @@ _rpython_startup = _lib.RPython_StartupCode _rpython_startup() """ % {'so_name': self.c_source_filename.new(ext=so_ext), - 'entrypoint_name': self.entrypoint_name, 'c_entrypoint_name': wrapped_entrypoint_c_name, 'nargs': len(lltype.typeOf(entrypoint_ptr).TO.ARGS)} modfile.write(CODE) @@ -243,7 +242,7 @@ def get_entry_point(self, isolated=False): self._import_module(isolated=isolated) - return getattr(self._module, self.entrypoint_name) + return getattr(self._module, "entrypoint") def get_malloc_counters(self, isolated=False): self._import_module(isolated=isolated) Modified: pypy/dist/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_genc.py (original) +++ pypy/dist/pypy/translator/c/test/test_genc.py Sun Nov 25 14:51:52 2007 @@ -61,6 +61,19 @@ py.test.raises(Exception, f1, "world") # check that it's really typed +def test_simple_lambda(): + f = lambda x: x*2 + t = TranslationContext() + t.buildannotator().build_types(f, [int]) + t.buildrtyper().specialize() + + t.config.translation.countmallocs = True + builder = genc.CExtModuleBuilder(t, f, config=t.config) + builder.generate_source() + builder.compile() + f1 = builder.get_entry_point() + + assert f1(5) == 10 def test_rlist(): def f(x): From pedronis at codespeak.net Sun Nov 25 15:01:54 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 25 Nov 2007 15:01:54 +0100 (CET) Subject: [pypy-svn] r49079 - in pypy/branch/rewrite-compilation-logic/pypy/translator/c: . test Message-ID: <20071125140154.E9383816C@code0.codespeak.net> Author: pedronis Date: Sun Nov 25 15:01:53 2007 New Revision: 49079 Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py Log: port 49078 from trunk Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py Sun Nov 25 15:01:53 2007 @@ -200,7 +200,7 @@ _entry_point.restype = ctypes.py_object _entry_point.argtypes = %(nargs)d*(ctypes.py_object,) -def %(entrypoint_name)s(*args): +def entrypoint(*args): return _entry_point(*args) try: @@ -217,7 +217,6 @@ _rpython_startup = _lib.RPython_StartupCode _rpython_startup() """ % {'so_name': self.c_source_filename.new(ext=so_ext), - 'entrypoint_name': self.entrypoint_name, 'c_entrypoint_name': wrapped_entrypoint_c_name, 'nargs': len(lltype.typeOf(entrypoint_ptr).TO.ARGS)} modfile.write(CODE) @@ -239,7 +238,7 @@ def get_entry_point(self, isolated=False): self._import_module(isolated=isolated) - return getattr(self._module, self.entrypoint_name) + return getattr(self._module, "entrypoint") def get_malloc_counters(self, isolated=False): self._import_module(isolated=isolated) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py Sun Nov 25 15:01:53 2007 @@ -81,6 +81,20 @@ x = py.test.raises(Exception, f1, "world") assert not isinstance(x.value, EOFError) # EOFError === segfault +def test_simple_lambda(): + f = lambda x: x*2 + t = TranslationContext() + t.buildannotator().build_types(f, [int]) + t.buildrtyper().specialize() + + t.config.translation.countmallocs = True + builder = genc.CExtModuleBuilder(t, f, config=t.config) + builder.generate_source() + builder.compile() + f1 = builder.get_entry_point() + + assert f1(5) == 10 + def test_rlist(): def f(x): l = [x] From fijal at codespeak.net Sun Nov 25 15:20:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 15:20:58 +0100 (CET) Subject: [pypy-svn] r49080 - in pypy/branch/rewrite-compilation-logic/pypy: rpython rpython/module rpython/module/test translator/c Message-ID: <20071125142058.5DFAD8170@code0.codespeak.net> Author: fijal Date: Sun Nov 25 15:20:55 2007 New Revision: 49080 Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_environ.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_stat.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_time.py pypy/branch/rewrite-compilation-logic/pypy/rpython/module/test/test_posix.py pypy/branch/rewrite-compilation-logic/pypy/rpython/rtyper.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py Log: * Make most test run (besides llvm) * Fix genc to really wrap with exceptions cpython api calls * make gencapicall use extern='CPython' instead of extern='C' Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py Sun Nov 25 15:20:55 2007 @@ -107,7 +107,7 @@ from pypy.rpython.lltypesystem import rffi if 'compilation_info' in kwds: - kwds['compilation_info'] = self._compilation_info_.merge( + kwds['compilation_info'] = self.compilation_info.merge( kwds['compilation_info']) else: kwds['compilation_info'] = self.compilation_info Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os.py Sun Nov 25 15:20:55 2007 @@ -90,9 +90,13 @@ data = {'ret_type': 'int', 'name': name} decls.append(decl_snippet % data) defs.append(def_snippet % data) - h_source = ['#include "sys/wait.h"'] + decls + ["#ifndef PYPY_NOT_MAIN_FILE"] + defs + ["#endif", ""] - h_file = udir.join("pypy_os_macros.h") - h_file.write("\n".join(h_source)) + h_source = ['#include "sys/wait.h"'] + decls + defs + + self.compilation_info = self.compilation_info.merge( + ExternalCompilationInfo( + post_include_lines = decls, + separate_module_sources = ["\n".join(h_source)] + )) # a simple, yet usefull factory def extdef_for_os_function_returning_int(self, name, **kwds): @@ -184,21 +188,24 @@ os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) class CConfig: - _includes_ = ['sys/time.h'] + _compilation_info_ = ExternalCompilationInfo( + includes=['sys/time.h'] + ) HAVE_UTIMES = platform.Has('utimes') config = platform.configure(CConfig) if config['HAVE_UTIMES']: class CConfig: - _includes_ = ['sys/time.h'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['sys/time.h'] + ) TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.LONG), ('tv_usec', rffi.LONG)]) config = platform.configure(CConfig) TIMEVAL = config['TIMEVAL'] TIMEVAL2P = rffi.CArrayPtr(TIMEVAL) os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], - rffi.INT, - includes=['sys/time.h']) + rffi.INT, compilation_info=CConfig._compilation_info_) def os_utime_platform(path, actime, modtime): import math @@ -344,7 +351,9 @@ def register_os_uname(self): CHARARRAY = lltype.FixedSizeArray(lltype.Char, 1) class CConfig: - _includes_ = ['sys/utsname.h'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['sys/utsname.h'] + ) UTSNAME = platform.Struct('struct utsname', [ ('sysname', CHARARRAY), ('nodename', CHARARRAY), @@ -355,7 +364,7 @@ UTSNAMEP = lltype.Ptr(config['UTSNAME']) os_uname = self.llexternal('uname', [UTSNAMEP], rffi.INT, - includes=CConfig._includes_) + compilation_info=CConfig._compilation_info_) def uname_llimpl(): l_utsbuf = lltype.malloc(UTSNAMEP.TO, flavor='raw') @@ -572,7 +581,9 @@ # to get a correct implementation of os.abspath # XXX why do we ignore WINAPI conventions everywhere? class CConfig: - _includes_ = ['Windows.h'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['Windows.h'] + ) MAX_PATH = platform.ConstantInteger('MAX_PATH') DWORD = platform.SimpleType("DWORD", rffi.ULONG) LPCTSTR = platform.SimpleType("LPCTSTR", rffi.CCHARP) @@ -654,7 +665,9 @@ # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): class CConfig: - _includes_ = ['windows.h'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['windows.h'] + ) WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) INVALID_HANDLE_VALUE = platform.ConstantInteger( @@ -719,8 +732,11 @@ lltype.free(filedata, flavor='raw') else: + compilation_info = ExternalCompilationInfo( + includes = ['sys/types.h', 'dirent.h'] + ) class CConfig: - _includes_ = ['sys/types.h', 'dirent.h'] + _compilation_info_ = compilation_info DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]) @@ -729,11 +745,11 @@ DIRENT = config['DIRENT'] DIRENTP = lltype.Ptr(DIRENT) os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP, - includes=CConfig._includes_) + compilation_info=compilation_info) os_readdir = self.llexternal('readdir', [DIRP], DIRENTP, - includes=CConfig._includes_) + compilation_info=compilation_info) os_closedir = self.llexternal('closedir', [DIRP], rffi.INT, - includes=CConfig._includes_) + compilation_info=compilation_info) def os_listdir_llimpl(path): dirp = os_opendir(path) @@ -1107,8 +1123,8 @@ return int(getattr(os, name)(status)) fake.func_name = 'fake_' + name - os_c_func = self.llexternal("pypy_macro_wrapper_" + name, [lltype.Signed], - lltype.Signed, includes=['pypy_os_macros.h'], + os_c_func = self.llexternal("pypy_macro_wrapper_" + name, + [lltype.Signed], lltype.Signed, _callable=fake) if name in self.w_star_returning_int: Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_environ.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_environ.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_environ.py Sun Nov 25 15:20:55 2007 @@ -132,6 +132,8 @@ # ____________________________________________________________ # Access to the 'environ' external variable +from pypy.translator.tool.cbuild import ExternalCompilationInfo + if sys.platform.startswith('darwin'): CCHARPPP = rffi.CArrayPtr(rffi.CCHARPP) _os_NSGetEnviron = rffi.llexternal('_NSGetEnviron', [], CCHARPPP, @@ -140,7 +142,8 @@ return _os_NSGetEnviron()[0] else: os_get_environ, _os_set_environ = rffi.CExternVariable(rffi.CCHARPP, - 'environ') + 'environ', + ExternalCompilationInfo()) # ____________________________________________________________ Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_os_stat.py Sun Nov 25 15:20:55 2007 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo # XXX on Windows, stat() is flawed; see CPython's posixmodule.c for # an implementation based on the Win32 API @@ -144,11 +145,15 @@ _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] +compilation_info = ExternalCompilationInfo( + pre_include_lines = ['#define _FILE_OFFSET_BITS 64'], + includes = INCLUDES +) + from pypy.rpython.tool import rffi_platform as platform class CConfig: # This must be set to 64 on some systems to enable large file support. - _header_ = '#define _FILE_OFFSET_BITS 64' - _includes_ = INCLUDES + _compilation_info_ = compilation_info STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) config = platform.configure(CConfig) @@ -198,7 +203,7 @@ else: ARG1 = rffi.INT os_mystat = rffi.llexternal(c_func_name, [ARG1, STAT_STRUCT], rffi.INT, - includes=INCLUDES) + compilation_info=compilation_info) def os_mystat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ll_time.py Sun Nov 25 15:20:55 2007 @@ -9,14 +9,19 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.extfunc import BaseLazyRegistering, registering, extdef from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo + +if sys.platform.startswith('win'): + includes = ['time.h', 'windows.h'] +else: + includes = ['sys/time.h', 'time.h', 'errno.h', 'sys/select.h', + 'sys/types.h', 'unistd.h', 'sys/timeb.h'] + class CConfig: - if sys.platform.startswith('win'): - _includes_ = ['time.h', 'windows.h'] - else: - _includes_ = ['sys/time.h', 'time.h', 'errno.h', 'sys/select.h', - 'sys/types.h', 'unistd.h', 'sys/timeb.h'] - + _compilation_info_ = ExternalCompilationInfo( + includes=includes + ) CLOCK_T = platform.SimpleType('clock_t', rffi.INT) TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.INT), ('tv_usec', rffi.INT)]) @@ -25,7 +30,7 @@ HAVE_FTIME = platform.Has('ftime') class CConfigForFTime: - _includes_ = ['sys/timeb.h'] + _compilation_info_ = ExternalCompilationInfo(includes=['sys/timeb.h']) TIMEB = platform.Struct('struct timeb', [('time', rffi.INT), ('millitm', rffi.INT)]) Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/module/test/test_posix.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/module/test/test_posix.py Sun Nov 25 15:20:55 2007 @@ -154,5 +154,5 @@ assert res == fun(value) def test_os_wstar(): - yield os_wstar_tester, compile_llvm yield os_wstar_tester, compile + yield os_wstar_tester, compile_llvm Modified: pypy/branch/rewrite-compilation-logic/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rpython/rtyper.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rpython/rtyper.py Sun Nov 25 15:20:55 2007 @@ -925,7 +925,7 @@ return self.genop('direct_call', [cf]+list(args_v), resulttype) def gencapicall(self, cfnname, args_v, resulttype=None, **flags): - return self.genexternalcall(cfnname, args_v, resulttype=resulttype, external="C", **flags) + return self.genexternalcall(cfnname, args_v, resulttype=resulttype, external="CPython", **flags) def genconst(self, ll_value): return inputconst(typeOf(ll_value), ll_value) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py Sun Nov 25 15:20:55 2007 @@ -745,7 +745,7 @@ libraries = eci.libraries f.write(SETUP_PY % locals()) f.close() - eci = eci.convert_sources_to_files() + eci = eci.convert_sources_to_files(being_main=True) return filename, sg.getextrafiles() + list(eci.separate_module_files) Modified: pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py Sun Nov 25 15:20:55 2007 @@ -821,16 +821,14 @@ exception_policy = getattr(fnobj, 'exception_policy', None) return [FunctionCodeGenerator(fnobj.graph, db, exception_policy, functionname)] - elif getattr(fnobj, 'external', None) == 'C': + elif getattr(fnobj, 'external', None) is not None: if sandbox: return sandbox_stub(fnobj, db) - # XXX broken - return [] - #if hasattr(fnobj, 'includes'): - # return [] # assume no wrapper needed - #else: - # # deprecated case - # return [CExternalFunctionCodeGenerator(fnobj, db)] + elif fnobj.external == 'C': + return [] + else: + assert fnobj.external == 'CPython' + return [CExternalFunctionCodeGenerator(fnobj, db)] else: raise ValueError, "don't know how to generate code for %r" % (fnobj,) From ac at codespeak.net Sun Nov 25 15:45:03 2007 From: ac at codespeak.net (ac at codespeak.net) Date: Sun, 25 Nov 2007 15:45:03 +0100 (CET) Subject: [pypy-svn] r49081 - in pypy/dist/pypy/interpreter: . pyparser pyparser/data pyparser/test Message-ID: <20071125144503.212BB8170@code0.codespeak.net> Author: ac Date: Sun Nov 25 15:45:01 2007 New Revision: 49081 Removed: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder_future.py Modified: pypy/dist/pypy/interpreter/pycompiler.py pypy/dist/pypy/interpreter/pyparser/astbuilder.py pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a pypy/dist/pypy/interpreter/pyparser/ebnfparse.py pypy/dist/pypy/interpreter/pyparser/future.py pypy/dist/pypy/interpreter/pyparser/grammar.py pypy/dist/pypy/interpreter/pyparser/pythonlexer.py pypy/dist/pypy/interpreter/pyparser/pythonparse.py pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py pypy/dist/pypy/interpreter/pyparser/test/test_parser.py pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Log: Merge the __future__ mending from the dist-future-fixing branch, but skip the parser refactorings. Modified: pypy/dist/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/dist/pypy/interpreter/pycompiler.py (original) +++ pypy/dist/pypy/interpreter/pycompiler.py Sun Nov 25 15:45:01 2007 @@ -2,10 +2,12 @@ General classes for bytecode compilers. Compiler instances are stored into 'space.getexecutioncontext().compiler'. """ + +import sys from codeop import PyCF_DONT_IMPLY_DEDENT from pypy.interpreter.error import OperationError -class AbstractCompiler: +class AbstractCompiler(object): """Abstract base class for a bytecode compiler.""" # The idea is to grow more methods here over the time, @@ -78,42 +80,53 @@ # faked compiler import warnings -from pypy.tool import stdlib___future__ -compiler_flags = 0 -compiler_features = {} -for fname in stdlib___future__.all_feature_names: - flag = getattr(stdlib___future__, fname).compiler_flag - compiler_flags |= flag - compiler_features[fname] = flag -allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT - -def get_flag_names(space, flags): - if flags & ~allowed_flags: - raise OperationError(space.w_ValueError, - space.wrap("compile(): unrecognized flags")) - flag_names = [] - for name, value in compiler_features.items(): - if flags & value: - flag_names.append( name ) - return flag_names + +## from pypy.tool import stdlib___future__ +## compiler_flags = 0 +## compiler_features = {} +## for fname in stdlib___future__.all_feature_names: +## flag = getattr(stdlib___future__, fname).compiler_flag +## compiler_flags |= flag +## compiler_features[fname] = flag +## allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT + +## def get_flag_names(space, flags): +## if flags & ~allowed_flags: +## raise OperationError(space.w_ValueError, +## space.wrap("compile(): unrecognized flags")) +## flag_names = [] +## for name, value in compiler_features.items(): +## if flags & value: +## flag_names.append( name ) +## return flag_names class PyCodeCompiler(AbstractCompiler): """Base class for compilers producing PyCode objects.""" def getcodeflags(self, code): + """Return the __future__ compiler flags that were used to compile + the given code object.""" from pypy.interpreter.pycode import PyCode if isinstance(code, PyCode): - return code.co_flags & compiler_flags + return code.co_flags & self.compiler_flags else: return 0 +from pypy.interpreter.pyparser import future class CPythonCompiler(PyCodeCompiler): """Faked implementation of a compiler, using the underlying compile().""" + def __init__(self, space): + self.space = space + self.w_compile_hook = space.w_None + if sys.version_info >= (2.5): + self.compiler_flags = future.futureFlags_2_5.allowed_flags + else: + self.compiler_flags = future.futureFlags_2_4.allowed_flags + def compile(self, source, filename, mode, flags): - flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) space = self.space try: old = self.setup_warn_explicit(warnings) @@ -191,6 +204,7 @@ ######## + class PythonAstCompiler(PyCodeCompiler): """Uses the stdlib's python implementation of compiler @@ -206,6 +220,11 @@ self.grammar_version = override_version or space.config.objspace.pyversion self.parser = make_pyparser(self.grammar_version) self.additional_rules = {} + if self.grammar_version >= '2.5': + self.futureFlags = future.futureFlags_2_5 + else: + self.futureFlags = future.futureFlags_2_4 + self.compiler_flags = self.futureFlags.allowed_flags def compile(self, source, filename, mode, flags): from pyparser.error import SyntaxError @@ -218,13 +237,16 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.function import Function - flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) + from pypy.interpreter.pyparser.future import getFutures + +## flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) space = self.space try: builder = AstBuilder(self.parser, self.grammar_version, space=space) for rulename, buildfunc in self.additional_rules.iteritems(): assert isinstance(buildfunc, Function) builder.user_build_rules[rulename] = buildfunc + flags |= getFutures(self.futureFlags, source) self.parser.parse_source(source, mode, builder, flags) ast_tree = builder.rule_stack[-1] encoding = builder.source_encoding @@ -244,7 +266,7 @@ raise try: astcompiler.misc.set_filename(filename, ast_tree) - flag_names = get_flag_names(space, flags) + flag_names = self.futureFlags.get_flag_names(space, flags) if mode == 'exec': codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names) elif mode == 'single': @@ -257,7 +279,7 @@ e.wrap_info(space, filename)) except (ValueError, TypeError), e: raise OperationError(space.w_SystemError, space.wrap(str(e))) - assert isinstance(c,PyCode) + assert isinstance(c, PyCode) return c Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Sun Nov 25 15:45:01 2007 @@ -872,52 +872,9 @@ names.append((name, as_name)) if index < l: # case ',' index += 1 -## if from_name == '__future__': -## for name, asname in names: -## if name == 'with_statement': -## # found from __future__ import with_statement -## if not builder.with_enabled: -## builder.enable_with() -## #raise pythonparse.AlternateGrammarException() builder.push(ast.From(from_name, names, atoms[0].lineno)) -def build_future_import_feature(builder, nb): - """ - future_import_feature: NAME [('as'|NAME) NAME] - - Enables python language future imports. Called once per feature imported, - no matter how you got to this one particular feature. - """ - - atoms = peek_atoms(builder, nb) - - feature_name = atoms[0].get_value() - assert type(feature_name) is str - space = builder.space - feature_code = space.unwrap(space.appexec([space.wrap(feature_name)], - """(feature): - import __future__ as f - feature = getattr(f, feature, None) - return feature and feature.compiler_flag or 0 - """)) - - # We will call a method on the parser (the method exists only in unit - # tests). - if feature_code == consts.CO_FUTURE_WITH_STATEMENT: - rules = """ - compound_stmt: (if_stmt | while_stmt | for_stmt | try_stmt | - funcdef | classdef | with_stmt) - with_stmt: 'with' test [ 'as' expr ] ':' suite - """ - builder.insert_grammar_rule(rules, { - 'with_stmt': build_with_stmt}) - - # We need to keep the rule on the stack so we can share atoms - # with a later rule - return True - - def build_yield_stmt(builder, nb): atoms = get_atoms(builder, nb) builder.push(ast.Yield(atoms[1], atoms[0].lineno)) @@ -1089,6 +1046,7 @@ 'exprlist' : build_exprlist, 'decorator' : build_decorator, 'eval_input' : build_eval_input, + 'with_stmt' : build_with_stmt, } @@ -1099,28 +1057,15 @@ self.d = len(rule_stack) class AstBuilder(BaseGrammarBuilder): - """A builder that directly produce the AST""" + """A builder that directly produces the AST""" def __init__(self, parser, grammar_version, debug=0, space=None): BaseGrammarBuilder.__init__(self, parser, debug) self.rule_stack = [] self.space = space self.source_encoding = None -## self.with_enabled = False self.build_rules = ASTRULES_Template self.user_build_rules = {} - if grammar_version >= "2.5": - self.build_rules.update({ - 'future_import_feature': build_future_import_feature, - 'import_from_future': build_import_from, - }) - -## def enable_with(self): -## if self.with_enabled: -## return -## self.with_enabled = True -## # XXX -## # self.keywords.update({'with':None, 'as': None}) def context(self): return AstBuilderContext(self.rule_stack) @@ -1156,7 +1101,9 @@ self.push(astnode) else: builder_func = self.build_rules.get(rulename, None) - if not builder_func or builder_func(self, 1): + if builder_func: + builder_func(self, 1) + else: self.push_rule(rule.codename, 1, source) else: self.push_rule(rule.codename, 1, source) @@ -1176,7 +1123,9 @@ self.push(astnode) else: builder_func = self.build_rules.get(rulename, None) - if not builder_func or builder_func(self, elts_number): + if builder_func: + builder_func(self, elts_number) + else: self.push_rule(rule.codename, elts_number, source) else: self.push_rule(rule.codename, elts_number, source) @@ -1227,14 +1176,6 @@ else: return None - def insert_grammar_rule(self, rule, buildfuncs): - """Inserts new grammar rules for the builder - This allows to change the rules during the parsing - """ - self.parser.insert_rule(rule) - self.build_rules.update(buildfuncs) - - def show_stack(before, after): """debugging helper function""" size1 = len(before) Modified: pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a (original) +++ pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a Sun Nov 25 15:45:01 2007 @@ -24,16 +24,10 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! - -single_input: NEWLINE | future_import_list [';'] | [future_import_list ';'] simple_stmt | compound_stmt NEWLINE -file_input: (future_import_list [';'] NEWLINE)* (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER -future_import_list: import_from_future (';' import_from_future)* -import_from_future: 'from' '__future__' 'import' ('(' future_import_as_names [','] ')' | future_import_as_names) -future_import_feature: NAME [('as'|NAME) NAME] -future_import_as_names: future_import_feature (',' future_import_feature)* - decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ funcdef: [decorators] 'def' NAME parameters ':' suite @@ -69,12 +63,13 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite) +with_stmt: 'with' test [ 'as' expr ] ':' suite # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test [',' test]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT Modified: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/ebnfparse.py (original) +++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py Sun Nov 25 15:45:01 2007 @@ -40,9 +40,8 @@ class NameToken(Token): """A token that is not a keyword""" - def __init__(self, parser, keywords=None): + def __init__(self, parser): Token.__init__(self, parser, parser.tokens['NAME']) - self.keywords = keywords def match(self, source, builder, level=0): """Matches a token. @@ -60,7 +59,7 @@ if tk.codename == self.codename: # XXX (adim): this is trunk's keyword management # if tk.value not in builder.keywords: - if tk.value not in self.keywords: + if not tk.isKeyword: ret = builder.token( tk.codename, tk.value, source ) return ret source.restore( ctx ) @@ -78,7 +77,7 @@ return False # XXX (adim): this is trunk's keyword management # if other.value in builder.keywords: - if other.value in self.keywords: + if other.isKeyword: return False return True @@ -107,7 +106,7 @@ self.keywords = [] NAME = dest_parser.add_token('NAME') # NAME = dest_parser.tokens['NAME'] - self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords) + self.tokens[NAME] = NameToken(dest_parser) def context(self): return EBNFBuilderContext(len(self.rule_stack), self.seqcounts, self.altcounts) Modified: pypy/dist/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/future.py (original) +++ pypy/dist/pypy/interpreter/pyparser/future.py Sun Nov 25 15:45:01 2007 @@ -27,8 +27,8 @@ from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \ CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT -def getFutures(source): - futures = FutureAutomaton(source) +def getFutures(futureFlags, source): + futures = FutureAutomaton(futureFlags, source) try: futures.start() except (IndexError, DoneException), e: @@ -62,9 +62,9 @@ precede a future statement. """ - def __init__(self, string): + def __init__(self, futureFlags, string): + self.futureFlags = futureFlags self.s = string - self.end = len(string) self.pos = 0 self.docstringConsumed = False self.flags = 0 @@ -237,10 +237,40 @@ self.getMore(parenList=parenList) def setFlag(self, feature): - if feature == "division": - self.flags |= CO_FUTURE_DIVISION - elif feature == "generators": - self.flags |= CO_GENERATOR_ALLOWED - elif feature == "with_statement": - self.flags |= CO_FUTURE_WITH_STATEMENT + try: + self.flags |= self.futureFlags.compiler_features[feature] + except KeyError: + pass + +from codeop import PyCF_DONT_IMPLY_DEDENT +from pypy.interpreter.error import OperationError + +from pypy.tool import stdlib___future__ as future + +class FutureFlags(object): + def __init__(self, version): + compiler_flags = 0 + self.compiler_features = {} + self.mandatory_flags = 0 + for fname in future.all_feature_names: + feature = getattr(future, fname) + if version >= feature.getOptionalRelease(): + flag = feature.compiler_flag + compiler_flags |= flag + self.compiler_features[fname] = flag + if version >= feature.getMandatoryRelease(): + self.mandatory_flags |= feature.compiler_flag + self.allowed_flags = compiler_flags | PyCF_DONT_IMPLY_DEDENT + + def get_flag_names(self, space, flags): + if flags & ~self.allowed_flags: + raise OperationError(space.w_ValueError, + space.wrap("compile(): unrecognized flags")) + flag_names = [] + for name, value in self.compiler_features.items(): + if flags & value: + flag_names.append(name) + return flag_names +futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0)) +futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0)) Modified: pypy/dist/pypy/interpreter/pyparser/grammar.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/grammar.py (original) +++ pypy/dist/pypy/interpreter/pyparser/grammar.py Sun Nov 25 15:45:01 2007 @@ -656,6 +656,7 @@ class Token(GrammarElement): """Represents a Token in a grammar rule (a lexer token)""" + isKeyword = True def __init__(self, parser, codename, value=None): GrammarElement.__init__(self, parser, codename) self.value = value @@ -680,7 +681,7 @@ ctx = source.context() tk = source.next() - if tk.codename == self.codename: + if tk.codename == self.codename and tk.isKeyword: if self.value is None: ret = builder.token( tk.codename, tk.value, source ) return ret @@ -711,12 +712,7 @@ raise RuntimeError("Unexpected token type") if other is self.parser.EmptyToken: return False - # XXX (adim): this is trunk's keyword management - # if (self.value is not None and builder.keywords is not None - # and self.value not in builder.keywords): - # return False - res = other.codename == self.codename and self.value in [None, other.value] - #print "matching", self, other, res + res = other.isKeyword and other.codename == self.codename and self.value in [None, other.value] return res def __eq__(self, other): Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/pythonlexer.py (original) +++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py Sun Nov 25 15:45:01 2007 @@ -62,7 +62,7 @@ SyntaxError.__init__(self, msg, lineno, offset, line) self.token_stack = token_stack -def generate_tokens( parser, lines, flags): +def generate_tokens( parser, lines, flags, keywords): """ This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since the original function is not RPYTHON (uses yield) @@ -246,6 +246,8 @@ last_comment = '' elif initial in namechars: # ordinary name tok = Token(parser, parser.tokens['NAME'], token) + if token not in keywords: + tok.isKeyword = False token_list.append((tok, line, lnum, pos)) last_comment = '' elif initial == '\\': # continued stmt @@ -302,12 +304,12 @@ class PythonSource(TokenSource): """This source uses Jonathan's tokenizer""" - def __init__(self, parser, strings, flags=0): + def __init__(self, parser, strings, keywords, flags=0): # TokenSource.__init__(self) #self.parser = parser self.input = strings - tokens = generate_tokens( parser, strings, flags) + tokens = generate_tokens( parser, strings, flags, keywords) self.token_stack = tokens self._current_line = '' # the current line (as a string) self._lineno = -1 @@ -361,13 +363,6 @@ def get_pos(self): return self.stack_pos -# Old code. Can't work, as self.stack_pos returns an int and the else branch -# returns a tuple. -## if self.stack_pos >= len(self.token_stack): -## return self.stack_pos -## else: -## token, line, lnum, pos = self.token_stack[self.stack_pos] -## return lnum, pos def get_source_text(self, p0, p1): "We get passed two token stack positions." @@ -376,21 +371,6 @@ def debug(self): """return context for debug information""" return (self._current_line, self._lineno) - # return 'line %s : %s' % ('XXX', self._current_line) - -#NONE_LIST = [pytoken.ENDMARKER, pytoken.INDENT, pytoken.DEDENT] -#NAMED_LIST = [pytoken.OP] Source = PythonSource -def tokenize_file(filename): - f = file(filename).read() - src = Source(f) - token = src.next() - while token != ("ENDMARKER", None) and token != (None, None): - print token - token = src.next() - -if __name__ == '__main__': - import sys - tokenize_file(sys.argv[1]) Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/pythonparse.py (original) +++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py Sun Nov 25 15:45:01 2007 @@ -4,26 +4,17 @@ helper functions are provided that use the grammar to parse using file_input, single_input and eval_input targets """ -import sys -import os -from pypy.interpreter.error import OperationError, debug_print from pypy.interpreter import gateway from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT -import pypy.interpreter.pyparser.pysymbol as pysymbol import pypy.interpreter.pyparser.pytoken as pytoken import pypy.interpreter.pyparser.ebnfparse as ebnfparse from pypy.interpreter.pyparser.ebnflexer import GrammarSource from pypy.interpreter.pyparser.ebnfgrammar import GRAMMAR_GRAMMAR import pypy.interpreter.pyparser.grammar as grammar -from pypy.interpreter.pyparser.pythonutil import build_parser_for_version, build_parser - -# try: +from pypy.interpreter.pyparser.pythonutil import build_parser_for_version from pypy.interpreter.pyparser import symbol -# except ImportError: -# # for standalone testing -# import symbol from codeop import PyCF_DONT_IMPLY_DEDENT @@ -123,12 +114,16 @@ def parse_lines(self, lines, goal, builder, flags=0): - # builder.keywords = self.keywords.copy() - # if flags & CO_FUTURE_WITH_STATEMENT: - # builder.enable_with() goalnumber = self.symbols[goal] target = self.root_rules[goalnumber] - src = Source(self, lines, flags) + keywords = {} # dict.fromkeys(self.keywords) + disable_with = not (flags & CO_FUTURE_WITH_STATEMENT) + for keyword in self.keywords: + if disable_with and keyword in ('with', 'as'): + continue + keywords[keyword] = None + src = Source(self, lines, keywords, flags) + if not target.match(src, builder): line, lineno = src.debug() # XXX needs better error messages @@ -190,6 +185,7 @@ ## grammar.build_first_sets(ebnfbuilder.all_rules) ## return space.wrap( ebnfbuilder.root_rules ) +# XXX Unused? def grammar_rules( space ): w_rules = space.newdict() parser = make_pyparser(space.config.objspace.pyversion) Modified: pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py Sun Nov 25 15:45:01 2007 @@ -1,10 +1,9 @@ import py import pypy.interpreter.pyparser.future as future -from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \ - CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT +from pypy.tool import stdlib___future__ as fut def run(s): - f = future.FutureAutomaton(s) + f = future.FutureAutomaton(future.futureFlags_2_5, s) try: f.start() except IndexError, future.DoneException: @@ -15,7 +14,7 @@ s = '"Docstring\\" "\nfrom __future__ import division\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_comment(): s = '# A comment about nothing ;\n' @@ -49,88 +48,97 @@ s = 'from __future__ import division\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms(): s = 'from __future__ import division, generators, with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_from_as(): s = 'from __future__ import division as b\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms_as(): s = 'from __future__ import division as b, generators as c\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_from_paren(): s = 'from __future__ import (division)\n' f = run(s) assert f.pos == len(s) - assert f.flags == CO_FUTURE_DIVISION + assert f.flags == fut.CO_FUTURE_DIVISION def test_froms_paren(): s = 'from __future__ import (division, generators)\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_froms_paren_as(): s = 'from __future__ import (division as b, generators,)\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED) def test_multiline(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_windows_style_lineendings(): s = '"abc" #def\r\n #ghi\r\nfrom __future__ import (division as b, generators,)\r\nfrom __future__ import with_statement\r\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_mac_style_lineendings(): s = '"abc" #def\r #ghi\rfrom __future__ import (division as b, generators,)\rfrom __future__ import with_statement\r' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_semicolon(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' f = run(s) assert f.pos == len(s) - assert f.flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + assert f.flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - flags = future.getFutures(s) - assert flags == (CO_FUTURE_DIVISION | - CO_GENERATOR_ALLOWED | - CO_FUTURE_WITH_STATEMENT) + flags = future.getFutures(future.futureFlags_2_5, s) + assert flags == (fut.CO_FUTURE_DIVISION | + fut.CO_GENERATOR_ALLOWED | + fut.CO_FUTURE_WITH_STATEMENT) def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' - flags = future.getFutures(s) - assert flags & CO_FUTURE_WITH_STATEMENT == 0 + flags = future.getFutures(future.futureFlags_2_5, s) + assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0 + + +def test_nonexisting(): + s = 'from __future__ import non_existing_feature\n' + f = run(s) + assert f.pos == len(s) + assert f.flags == 0 + + Modified: pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py Sun Nov 25 15:45:01 2007 @@ -1,5 +1,5 @@ -from pypy.interpreter.pyparser.grammar import Alternative, Sequence, KleeneStar, \ - Token, Parser +from pypy.interpreter.pyparser.grammar import Alternative, \ + Sequence, KleeneStar, Token, Parser class TestLookAheadBasics: Modified: pypy/dist/pypy/interpreter/pyparser/test/test_parser.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_parser.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_parser.py Sun Nov 25 15:45:01 2007 @@ -46,123 +46,3 @@ assert v == 9 -class RuleTracer(dict): - - def __init__(self, *args, **kw): - self.trace = [] - self.exclude_rules = [ - 'dotted_name', 'dotted_as_name', 'dotted_as_names', - 'import_stmt', 'small_stmt', 'simple_stmt', 'stmt', - 'single_input', 'file_input', 'future_import_list', - 'import_from_future', 'future_import_as_names'] - - def __getitem__(self, attr): - if attr in self.exclude_rules: - return None - - def record_trace(builder, number): - result = [t.value for t in get_atoms(builder, number)] - self.trace.append((attr, result)) - return record_trace - - def get(self, attr, default): - return self.__getitem__(attr) - -from pypy.interpreter.pyparser.astbuilder import AstBuilder -class MockBuilder(AstBuilder): - - def __init__(self, *args, **kw): - AstBuilder.__init__(self, *args, **kw) - self.build_rules = RuleTracer() - - -class TestFuture(object): - - _grammar_ver = '2.5a' - - def setup_class(self): - from pypy.interpreter.pyparser.pythonparse import make_pyparser - self.parser = make_pyparser(self._grammar_ver) - - def setup_method(self, method): - self.builder = MockBuilder(self.parser, self._grammar_ver, - space=FakeSpace()) - - def check_parse_mode(self, tst, expected, mode): - self.parser.parse_source(tst, mode, self.builder) - assert self.builder.build_rules.trace == expected - - def check_parse(self, tst, expected): - self.check_parse_mode(tst, expected, 'exec') - self.builder.build_rules.trace = [] - self.check_parse_mode(tst, expected, 'single') - - - def test_single_future_import(self): - tst = 'from __future__ import na\n' - expected = [('future_import_feature', ['na'])] - self.check_parse(tst, expected) - - def test_double_future_import(self): - tst = 'from __future__ import na, xx\n' - expected = [('future_import_feature', ['na']), - ('future_import_feature', ['xx'])] - self.check_parse(tst, expected) - - def test_two_future_imports(self): - tst = 'from __future__ import na;from __future__ import xx\n' - expected = [('future_import_feature', ['na']), - ('future_import_feature', ['xx'])] - self.check_parse(tst, expected) - - def test_future_imports_nl(self): - tst = ''' -from __future__ import na -from __future__ import xx; -from __future__ import yy -''' - expected = [('future_import_feature', ['na']), - ('future_import_feature', ['xx']), - ('future_import_feature', ['yy'])] - self.check_parse_mode(tst, expected,'exec') - - def test_single_future_as(self): - tst = 'from __future__ import na as x\n' - expected = [('future_import_feature', ['na', 'as', 'x'])] - self.check_parse(tst, expected) - - def test_single_future_as(self): - tst = 'import sys;from __future__ import na as x\n' - expected = [] - try: - self.check_parse_mode(tst, expected,'exec') - assert False == 'An import before a future import should throw an error.' - except error.SyntaxError: - pass - - def test_regular_import(self): - tst = 'import sys' - expected = [('import_name', ['import', 'sys'])] - self.check_parse(tst, expected) - - def test_future_import(self): - tst = 'import __future__' - expected = [('import_name', ['import', '__future__'])] - self.check_parse(tst, expected) - - def test_future_import_atoms(self): - self.builder.build_rules.exclude_rules.remove('import_from_future') - self.builder.build_rules.exclude_rules.append('future_import_feature') - tst = 'from __future__ import na as x\n' - expected = [('import_from_future', - ['from', '__future__', 'import', 'na', 'as', 'x'])] - self.check_parse(tst, expected) - - def test_regular_from(self): - self.builder.build_rules.exclude_rules.extend([ - 'import_as_name', 'import_as_names']) - - tst = 'from stuff import na as x\n' - expected = [('import_from', - ['from', 'stuff', 'import', 'na', 'as', 'x'])] - self.check_parse(tst, expected) Modified: pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Sun Nov 25 15:45:01 2007 @@ -18,7 +18,7 @@ def parse_source(source): """returns list of parsed tokens""" - lexer = Source( P, source.splitlines(True)) + lexer = Source( P, source.splitlines(True), {}) tokens = [] last_token = Token( P, NULLTOKEN, None) while last_token.codename != ENDMARKER: From fijal at codespeak.net Sun Nov 25 15:45:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 15:45:42 +0100 (CET) Subject: [pypy-svn] r49082 - pypy/branch/rewrite-compilation-logic/pypy/module/bz2 Message-ID: <20071125144542.A08868170@code0.codespeak.net> Author: fijal Date: Sun Nov 25 15:45:41 2007 New Revision: 49082 Modified: pypy/branch/rewrite-compilation-logic/pypy/module/bz2/interp_bz2.py Log: Fix bz2 module Modified: pypy/branch/rewrite-compilation-logic/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/module/bz2/interp_bz2.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/module/bz2/interp_bz2.py Sun Nov 25 15:45:41 2007 @@ -7,11 +7,14 @@ from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app from pypy.rlib.streamio import Stream +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys class CConfig: - _includes_ = ['stdio.h', 'sys/types.h', 'bzlib.h'] - _libraries_ = ['bz2'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['stdio.h', 'sys/types.h', 'bzlib.h'], + libraries = ['bz2'], + ) calling_conv = 'c' off_t = platform.SimpleType("off_t", rffi.LONGLONG) @@ -101,8 +104,8 @@ return bzs.total_out def external(name, args, result): - return rffi.llexternal(name, args, result, includes=CConfig._includes_, - libraries=['bz2']) + return rffi.llexternal(name, args, result, compilation_info= + CConfig._compilation_info_) # the least but one parameter should be rffi.VOIDP but it's not used # so I trick the compiler to not complain about constanst pointer passed From fijal at codespeak.net Sun Nov 25 15:45:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 15:45:54 +0100 (CET) Subject: [pypy-svn] r49083 - in pypy/branch/rewrite-compilation-logic/pypy: module/_ffi/test module/struct rlib Message-ID: <20071125144554.6B460817A@code0.codespeak.net> Author: fijal Date: Sun Nov 25 15:45:53 2007 New Revision: 49083 Modified: pypy/branch/rewrite-compilation-logic/pypy/module/_ffi/test/test__ffi.py pypy/branch/rewrite-compilation-logic/pypy/module/struct/nativefmttable.py pypy/branch/rewrite-compilation-logic/pypy/rlib/libffi.py Log: Fix few modules Modified: pypy/branch/rewrite-compilation-logic/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/module/_ffi/test/test__ffi.py Sun Nov 25 15:45:53 2007 @@ -1,7 +1,7 @@ from pypy.conftest import gettestobjspace -from pypy.translator.tool.cbuild import compile_c_module +from pypy.translator.tool.cbuild import compile_c_module, ExternalCompilationInfo import os, sys, py @@ -90,7 +90,7 @@ } ''')) - compile_c_module([c_file], 'x') + compile_c_module([c_file], 'x', ExternalCompilationInfo()) return str(udir.join('x.so')) prepare_c_example = staticmethod(prepare_c_example) Modified: pypy/branch/rewrite-compilation-logic/pypy/module/struct/nativefmttable.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/module/struct/nativefmttable.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/module/struct/nativefmttable.py Sun Nov 25 15:45:53 2007 @@ -3,6 +3,7 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import r_singlefloat +from pypy.translator.tool.cbuild import ExternalCompilationInfo native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1) @@ -83,16 +84,21 @@ 'd': 'double', } - class CConfig: - _header_ = "" - + pre_include_lines = [] for fmtchar, ctype in INSPECT.items(): - CConfig._header_ += """ + pre_include_lines += (""" struct about_%s { char pad; %s field; }; - """ % (fmtchar, ctype) + """ % (fmtchar, ctype)).split("\n") + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + pre_include_lines = pre_include_lines + ) + + for fmtchar, ctype in INSPECT.items(): setattr(CConfig, fmtchar, rffi_platform.Struct( "struct about_%s" % (fmtchar,), [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) Modified: pypy/branch/rewrite-compilation-logic/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/rlib/libffi.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/rlib/libffi.py Sun Nov 25 15:45:53 2007 @@ -8,6 +8,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import we_are_translated from pypy.tool.autopath import pypydir +from pypy.translator.tool.cbuild import ExternalCompilationInfo import py includes = ['dlfcn.h', 'ffi.h'] @@ -24,10 +25,12 @@ FFI_TYPE_PP = rffi.CArrayPtr(FFI_TYPE_P) class CConfig: - _includes_ = includes - _libraries_ = ['ffi'] - _include_dirs_ = include_dirs - _library_dirs_ = lib_dirs + _compilation_info_ = ExternalCompilationInfo( + includes = includes, + libraries = ['ffi', 'dl'], + include_dirs = include_dirs, + library_dirs = lib_dirs, + ) RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') @@ -95,9 +98,8 @@ } def external(name, args, result, **kwds): - return rffi.llexternal(name, args, result, includes=includes, - include_dirs=include_dirs, library_dirs=lib_dirs, - libraries=['dl', 'ffi'], **kwds) + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, **kwds) c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP, _nowrapper=True) @@ -111,9 +113,7 @@ FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) -FFI_CIFP = rffi.COpaquePtr('ffi_cif', includes=includes, - include_dirs=include_dirs, - library_dirs=lib_dirs) +FFI_CIFP = rffi.COpaquePtr('ffi_cif', compilation_info=CConfig._compilation_info_) VOIDPP = rffi.CArrayPtr(rffi.VOIDP) From fijal at codespeak.net Sun Nov 25 15:50:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 15:50:30 +0100 (CET) Subject: [pypy-svn] r49084 - in pypy/branch/rewrite-compilation-logic/pypy/module: fcntl rctime Message-ID: <20071125145030.A65108170@code0.codespeak.net> Author: fijal Date: Sun Nov 25 15:50:29 2007 New Revision: 49084 Modified: pypy/branch/rewrite-compilation-logic/pypy/module/fcntl/interp_fcntl.py pypy/branch/rewrite-compilation-logic/pypy/module/rctime/interp_time.py Log: Wack at modules until they work. Modified: pypy/branch/rewrite-compilation-logic/pypy/module/fcntl/interp_fcntl.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/module/fcntl/interp_fcntl.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/module/fcntl/interp_fcntl.py Sun Nov 25 15:50:29 2007 @@ -3,10 +3,13 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys class CConfig: - _includes_ = ['fcntl.h', 'sys/file.h', 'sys/ioctl.h'] + _compilation_info_ = ExternalCompilationInfo( + includes = ['fcntl.h', 'sys/file.h', 'sys/ioctl.h'] + ) flock = platform.Struct("struct flock", [('l_start', rffi.LONGLONG), ('l_len', rffi.LONGLONG), ('l_pid', rffi.LONG), ('l_type', rffi.SHORT), @@ -53,7 +56,7 @@ locals().update(constants) def external(name, args, result): - return rffi.llexternal(name, args, result, includes=CConfig._includes_) + return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_) _flock = lltype.Ptr(cConfig.flock) strerror = external('strerror', [rffi.INT], rffi.CCHARP) Modified: pypy/branch/rewrite-compilation-logic/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/branch/rewrite-compilation-logic/pypy/module/rctime/interp_time.py (original) +++ pypy/branch/rewrite-compilation-logic/pypy/module/rctime/interp_time.py Sun Nov 25 15:50:29 2007 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rlib.rarithmetic import ovfcheck_float_to_int from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo import math import os import sys @@ -18,7 +19,9 @@ _includes.append('sys/time.h') class CConfig: - _includes_ = _includes + _compilation_info_ = ExternalCompilationInfo( + includes = _includes + ) CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC") clock_t = platform.SimpleType("clock_t", rffi.ULONG) time_t = platform.SimpleType("time_t", rffi.LONG) @@ -51,7 +54,8 @@ def external(name, args, result): return rffi.llexternal(name, args, result, - includes=_includes, calling_conv=calling_conv, + compilation_info=CConfig._compilation_info_, + calling_conv=calling_conv, threadsafe=False) if _POSIX: From pedronis at codespeak.net Sun Nov 25 16:14:37 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 25 Nov 2007 16:14:37 +0100 (CET) Subject: [pypy-svn] r49085 - pypy/dist/pypy/translator/goal Message-ID: <20071125151437.E2811817E@code0.codespeak.net> Author: pedronis Date: Sun Nov 25 16:14:36 2007 New Revision: 49085 Added: pypy/dist/pypy/translator/goal/targetrpystonex.py - copied, changed from r49050, pypy/dist/pypy/translator/goal/targetrpystonex.py Log: resurrect this, it is still needed by various tests and by targerpystone itself From arigo at codespeak.net Sun Nov 25 16:24:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Nov 2007 16:24:50 +0100 (CET) Subject: [pypy-svn] r49086 - pypy/dist/pypy/translator/c Message-ID: <20071125152450.E12D781B6@code0.codespeak.net> Author: arigo Date: Sun Nov 25 16:24:49 2007 New Revision: 49086 Added: pypy/dist/pypy/translator/c/genc.py.merge.tmp - copied, changed from r49085, pypy/dist/pypy/translator/c/genc.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/rewrite-compilation-logic/pypy/translator/c/genc.py revisions 49053 to 49085: ------------------------------------------------------------------------ r49080 | fijal | 2007-11-25 15:20:55 +0100 (Sun, 25 Nov 2007) | 4 lines * Make most test run (besides llvm) * Fix genc to really wrap with exceptions cpython api calls * make gencapicall use extern='CPython' instead of extern='C' ------------------------------------------------------------------------ r49079 | pedronis | 2007-11-25 15:01:53 +0100 (Sun, 25 Nov 2007) | 3 lines port 49078 from trunk ------------------------------------------------------------------------ r49076 | fijal | 2007-11-25 14:06:53 +0100 (Sun, 25 Nov 2007) | 4 lines * fixeol * make a hack which allows us to pass ll_strtod tests * comment out part of logic, test_standalone passes ------------------------------------------------------------------------ r49061 | fijal | 2007-11-25 01:19:35 +0100 (Sun, 25 Nov 2007) | 4 lines * Move few modules to use new interface * Whack c backend until it can cope with ExternalCompilationInfo * Whack a bit test_rffi, still failures ------------------------------------------------------------------------ r49054 | fijal | 2007-11-24 19:00:44 +0100 (Sat, 24 Nov 2007) | 4 lines (fijal, arigo, xoraxax) In-progress checkin of rewriting compilation logic. Tries to sort out the need of includes include_dirs, libraries and friends. ------------------------------------------------------------------------ From arigo at codespeak.net Sun Nov 25 16:24:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Nov 2007 16:24:58 +0100 (CET) Subject: [pypy-svn] r49087 - pypy/dist/pypy/translator/c/test Message-ID: <20071125152458.8DCB681BE@code0.codespeak.net> Author: arigo Date: Sun Nov 25 16:24:56 2007 New Revision: 49087 Added: pypy/dist/pypy/translator/c/test/test_genc.py.merge.tmp - copied, changed from r49085, pypy/dist/pypy/translator/c/test/test_genc.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/rewrite-compilation-logic/pypy/translator/c/test/test_genc.py revisions 49053 to 49085: ------------------------------------------------------------------------ r49079 | pedronis | 2007-11-25 15:01:53 +0100 (Sun, 25 Nov 2007) | 3 lines port 49078 from trunk ------------------------------------------------------------------------ r49077 | pedronis | 2007-11-25 14:40:38 +0100 (Sun, 25 Nov 2007) | 3 lines test about the checking of py exception around python capi calls ------------------------------------------------------------------------ r49054 | fijal | 2007-11-24 19:00:44 +0100 (Sat, 24 Nov 2007) | 4 lines (fijal, arigo, xoraxax) In-progress checkin of rewriting compilation logic. Tries to sort out the need of includes include_dirs, libraries and friends. ------------------------------------------------------------------------ From arigo at codespeak.net Sun Nov 25 16:30:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Nov 2007 16:30:38 +0100 (CET) Subject: [pypy-svn] r49088 - in pypy/dist/pypy: module rlib rpython rpython/lltypesystem rpython/lltypesystem/module rpython/lltypesystem/test rpython/module rpython/tool translator/c translator/c/test translator/llsupport translator/tool Message-ID: <20071125153038.D238781B7@code0.codespeak.net> Author: arigo Date: Sun Nov 25 16:30:38 2007 New Revision: 49088 Added: pypy/dist/pypy/module/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/module/ pypy/dist/pypy/rlib/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/rlib/ pypy/dist/pypy/rpython/extfunc.py - copied unchanged from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/lltypesystem/ll2ctypes.py - copied unchanged from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/module/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/module/ pypy/dist/pypy/rpython/lltypesystem/rffi.py - copied unchanged from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/lltypesystem/test/ pypy/dist/pypy/rpython/module/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/module/ pypy/dist/pypy/rpython/rtyper.py - copied unchanged from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/rtyper.py pypy/dist/pypy/rpython/tool/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/rpython/tool/ pypy/dist/pypy/translator/c/genc.py - copied unchanged from r49087, pypy/dist/pypy/translator/c/genc.py.merge.tmp pypy/dist/pypy/translator/c/node.py - copied unchanged from r49087, pypy/branch/rewrite-compilation-logic/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_genc.py - copied unchanged from r49087, pypy/dist/pypy/translator/c/test/test_genc.py.merge.tmp pypy/dist/pypy/translator/llsupport/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/translator/llsupport/ pypy/dist/pypy/translator/tool/ - copied from r49087, pypy/branch/rewrite-compilation-logic/pypy/translator/tool/ Removed: pypy/dist/pypy/translator/c/genc.py.merge.tmp pypy/dist/pypy/translator/c/test/test_genc.py.merge.tmp Log: (fijal, xoraxax, arigo) Merge of the branch: rewriting the C compiling logic, by packaging all information and basic logic in a new class, pypy.translator.tool.cbuild.ExternalCompilationInfo. This solves a number of issues, e.g. ll2ctypes can now call C functions defined in snippets of C code. The capi calls are a bit saner in genc, but not completely yet - there is still a wrapper inserted by genc for that case. There are still some more clean-ups that we can do in genc now, too. From fijal at codespeak.net Sun Nov 25 16:35:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 16:35:52 +0100 (CET) Subject: [pypy-svn] r49089 - pypy/branch/remove-extcompiler-rctypes Message-ID: <20071125153552.B7C2081BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 16:35:52 2007 New Revision: 49089 Removed: pypy/branch/remove-extcompiler-rctypes/ Log: Remove dead branch. From fijal at codespeak.net Sun Nov 25 16:36:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 16:36:09 +0100 (CET) Subject: [pypy-svn] r49090 - pypy/branch/rewrite-compilation-logic Message-ID: <20071125153609.5575381BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 16:36:08 2007 New Revision: 49090 Removed: pypy/branch/rewrite-compilation-logic/ Log: Remove another dead branch as well. From fijal at codespeak.net Sun Nov 25 16:40:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 16:40:36 +0100 (CET) Subject: [pypy-svn] r49091 - pypy/dist/pypy/rlib Message-ID: <20071125154036.6633B81BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 16:40:35 2007 New Revision: 49091 Modified: pypy/dist/pypy/rlib/rzlib.py Log: Fix zlib Modified: pypy/dist/pypy/rlib/rzlib.py ============================================================================== --- pypy/dist/pypy/rlib/rzlib.py (original) +++ pypy/dist/pypy/rlib/rzlib.py Sun Nov 25 16:40:35 2007 @@ -1,6 +1,7 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo includes = ['zlib.h'] libraries = ['z'] @@ -18,7 +19,9 @@ """ Definitions for basic types defined by zlib. """ - _includes_ = includes + _compilation_info_ = ExternalCompilationInfo( + includes = includes + ) # XXX If Z_PREFIX was defined for the libz build, then these types are # named z_uInt, z_uLong, and z_Bytef instead. @@ -60,7 +63,9 @@ Definitions of structure types defined by zlib and based on SimpleCConfig definitions. """ - _includes_ = includes + _compilation_info_ = ExternalCompilationInfo( + includes = includes + ) z_stream = rffi_platform.Struct( 'z_stream', @@ -91,8 +96,10 @@ z_stream_p = lltype.Ptr(z_stream) def zlib_external(*a, **kw): - kw['includes'] = includes - kw['libraries'] = libraries + kw['compilation_info'] = ExternalCompilationInfo( + libraries=libraries, + includes=includes + ) return rffi.llexternal(*a, **kw) _crc32 = zlib_external('crc32', [uLong, Bytefp, uInt], uLong) From fijal at codespeak.net Sun Nov 25 16:43:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 16:43:24 +0100 (CET) Subject: [pypy-svn] r49092 - pypy/dist/pypy/rlib/test Message-ID: <20071125154324.34D4881BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 16:43:23 2007 New Revision: 49092 Modified: pypy/dist/pypy/rlib/test/test_libffi.py Log: Skip segfaulting test. Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Sun Nov 25 16:43:23 2007 @@ -104,6 +104,8 @@ assert len(ALLOCATED) == 1 def test_compile(self): + import py + py.test.skip("Segfaulting test, skip") # XXX cannot run it on top of llinterp, some problems # with pointer casts From fijal at codespeak.net Sun Nov 25 16:47:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 16:47:24 +0100 (CET) Subject: [pypy-svn] r49093 - pypy/dist/pypy/rlib Message-ID: <20071125154724.1268F81BC@code0.codespeak.net> Author: fijal Date: Sun Nov 25 16:47:23 2007 New Revision: 49093 Modified: pypy/dist/pypy/rlib/rmmap.py Log: Fix rmmap Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Sun Nov 25 16:47:23 2007 @@ -2,6 +2,7 @@ from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.rlib import rposix +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys import os @@ -21,11 +22,17 @@ def __init__(self, message): self.message = message +includes = ["sys/types.h"] +if _POSIX: + includes += ['unistd.h', 'sys/mman.h'] +elif _MS_WINDOWS: + includes.append("windows.h") + class CConfig: - _includes_ = ["sys/types.h"] - if _POSIX: - _includes_.append('unistd.h') - _header_ = '#define _GNU_SOURCE\n' + _compilation_info_ = ExternalCompilationInfo( + includes=includes, + pre_include_lines=['#define _GNU_SOURCE'] + ) size_t = rffi_platform.SimpleType("size_t", rffi.LONG) off_t = rffi_platform.SimpleType("off_t", rffi.LONG) if _MS_WINDOWS: @@ -47,7 +54,6 @@ constants = {} if _POSIX: - CConfig._includes_ += ("sys/mman.h",) # constants, look in sys/mman.h and platform docs for the meaning # some constants are linux only so they will be correctly exposed outside # depending on the OS @@ -68,7 +74,6 @@ # a dirty hack, this is probably a macro elif _MS_WINDOWS: - CConfig._includes_ += ("windows.h",) constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS'] @@ -91,10 +96,11 @@ _ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) def external(name, args, result): - return rffi.llexternal(name, args, result, includes=CConfig._includes_) + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_) def winexternal(name, args, result): - return rffi.llexternal(name, args, result, includes=CConfig._includes_, calling_conv='win') + return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, calling_conv='win') PTR = rffi.CCHARP From fijal at codespeak.net Sun Nov 25 17:27:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 17:27:59 +0100 (CET) Subject: [pypy-svn] r49095 - pypy/dist/pypy/jit/codegen/i386 Message-ID: <20071125162759.764F781BC@code0.codespeak.net> Author: fijal Date: Sun Nov 25 17:27:58 2007 New Revision: 49095 Modified: pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py Log: Fix the jit (part 1) Modified: pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py (original) +++ pypy/dist/pypy/jit/codegen/i386/codebuf_posix.py Sun Nov 25 17:27:58 2007 @@ -1,8 +1,13 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.tool import rffi_platform as platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo + +compilation_info = ExternalCompilationInfo( + includes = ("sys/types.h", "sys/mman.h") +) class CConfig: - _includes_ = ("sys/types.h", "sys/mman.h") + _compilation_info_ = compilation_info size_t = platform.SimpleType("size_t", rffi.ULONG) off_t = platform.SimpleType("off_t", rffi.LONG) @@ -26,11 +31,11 @@ mmap_ = rffi.llexternal('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, - includes = ["sys/mman.h"]) + compilation_info=compilation_info) munmap_ = rffi.llexternal('munmap', [PTR, size_t], rffi.INT, - includes = ["sys/mman.h"]) + compilation_info=compilation_info) class Hint: pos = -0x4fff0000 # for reproducible results From fijal at codespeak.net Sun Nov 25 18:42:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 18:42:06 +0100 (CET) Subject: [pypy-svn] r49096 - in pypy/dist/pypy/module: _curses crypt readline Message-ID: <20071125174206.57D1280B7@code0.codespeak.net> Author: fijal Date: Sun Nov 25 18:42:04 2007 New Revision: 49096 Modified: pypy/dist/pypy/module/_curses/fficurses.py pypy/dist/pypy/module/crypt/interp_crypt.py pypy/dist/pypy/module/readline/c_readline.py Log: Fix few modules Modified: pypy/dist/pypy/module/_curses/fficurses.py ============================================================================== --- pypy/dist/pypy/module/_curses/fficurses.py (original) +++ pypy/dist/pypy/module/_curses/fficurses.py Sun Nov 25 18:42:04 2007 @@ -9,20 +9,24 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.module._curses import interp_curses from pypy.rpython.lltypesystem import llmemory +from pypy.translator.tool.cbuild import ExternalCompilationInfo + +eci = ExternalCompilationInfo( + includes = ['curses.h', 'term.h'], + libraries = ['curses'], +) + -# waaa... -includes = ['curses.h', 'term.h'] -libs = ['curses'] INT = rffi.INT INTP = lltype.Ptr(lltype.Array(INT, hints={'nolength':True})) c_setupterm = rffi.llexternal('setupterm', [rffi.CCHARP, INT, INTP], INT, - includes=includes, libraries=libs) + compilation_info=eci) c_tigetstr = rffi.llexternal('tigetstr', [rffi.CCHARP], rffi.CCHARP, - includes=includes, libraries=libs) + compilation_info=eci) c_tparm = rffi.llexternal('tparm', [rffi.CCHARP, INT, INT, INT, INT, INT, INT, INT, INT, INT, INT], rffi.CCHARP, - includes=includes, libraries=libs) + compilation_info=eci) ERR = rffi.CConstant('ERR', INT) OK = rffi.CConstant('OK', INT) Modified: pypy/dist/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/dist/pypy/module/crypt/interp_crypt.py (original) +++ pypy/dist/pypy/module/crypt/interp_crypt.py Sun Nov 25 18:42:04 2007 @@ -1,10 +1,11 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rpython.lltypesystem import rffi, lltype +from pypy.translator.tool.cbuild import ExternalCompilationInfo - +eci = ExternalCompilationInfo(libraries=['crypt']) c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, - libraries=["crypt"], threadsafe=False) + compilation_info=eci, threadsafe=False) def crypt(space, word, salt): """word will usually be a user's password. salt is a 2-character string Modified: pypy/dist/pypy/module/readline/c_readline.py ============================================================================== --- pypy/dist/pypy/module/readline/c_readline.py (original) +++ pypy/dist/pypy/module/readline/c_readline.py Sun Nov 25 18:42:04 2007 @@ -2,13 +2,15 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, interp2app +from pypy.translator.tool.cbuild import ExternalCompilationInfo -includes = ["readline/readline.h", "readline/history.h"] -libs = ['readline'] +eci = ExternalCompilationInfo( + includes = ["readline/readline.h", "readline/history.h"], + libraries = ['readline'] +) def external(name, args, result): - return rffi.llexternal(name, args, result, includes=includes, - libraries=libs) + return rffi.llexternal(name, args, result, compilation_info=eci) # get a binding to c library functions and define their args and return types # char *readline(char *) From fijal at codespeak.net Sun Nov 25 18:42:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 18:42:46 +0100 (CET) Subject: [pypy-svn] r49097 - pypy/dist/pypy/translator/tool Message-ID: <20071125174246.AA1C88111@code0.codespeak.net> Author: fijal Date: Sun Nov 25 18:42:46 2007 New Revision: 49097 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: Wuaaaa Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Sun Nov 25 18:42:46 2007 @@ -130,7 +130,7 @@ f = filename.open("w") if being_main: f.write("#define PYPY_NOT_MAIN_FILE\n") - f.write(source) + f.write(str(source)) f.close() files.append(str(filename)) d = self._copy_attributes() From arigo at codespeak.net Sun Nov 25 19:09:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Nov 2007 19:09:29 +0100 (CET) Subject: [pypy-svn] r49098 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071125180929.B693981BA@code0.codespeak.net> Author: arigo Date: Sun Nov 25 19:09:29 2007 New Revision: 49098 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py Log: Write some comments that took me a while to "rediscover". Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Sun Nov 25 19:09:29 2007 @@ -100,6 +100,16 @@ moving_gc = True def __init__(self): + # WaRnInG! Putting GC objects as fields of the GC itself is + # basically *not* working in general! When running tests with + # the gcwrapper, there is no way they can be returned from + # get_roots_from_llinterp(). When the whole GC goes through the + # gctransformer, though, it works if the fields are read-only + # (and thus only ever reference a prebuilt list or dict). These + # prebuilt lists or dicts themselves can be mutated and point to + # more non-prebuild GC objects; this is fine because the + # internal GC ptr in the prebuilt list or dict is found by + # gctypelayout and listed in addresses_of_static_ptrs. self.wr_to_objects_with_id = [] self.object_id_dict = {} self.object_id_dict_ends_at = 0 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sun Nov 25 19:09:29 2007 @@ -223,6 +223,10 @@ self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): + # we don't need to trace prebuilt GcStructs during a minor collect: + # if a prebuilt GcStruct contains a pointer to a young object, + # then the write_barrier must have ensured that the prebuilt + # GcStruct is in the list self.old_objects_pointing_to_young. roots = self.get_roots(with_static=False) count = 0 while 1: From fijal at codespeak.net Sun Nov 25 19:29:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 19:29:17 +0100 (CET) Subject: [pypy-svn] r49099 - pypy/dist/pypy/module/thread Message-ID: <20071125182917.5C45581BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 19:29:15 2007 New Revision: 49099 Modified: pypy/dist/pypy/module/thread/ll_thread.py Log: Fix threads. Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Sun Nov 25 19:29:15 2007 @@ -4,33 +4,29 @@ from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.extfunc import genericcallable from pypy.rpython.annlowlevel import cast_instance_to_base_ptr -from pypy.translator.tool.cbuild import cache_c_module +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import llmemory import thread, py from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import typeOf from pypy.rlib.objectmodel import debug_assert +from pypy.tool import autopath error = thread.error -from pypy.tool.autopath import pypydir -pypydir = py.path.local(pypydir) -c_dir = pypydir.join('translator', 'c') -includes = ['unistd.h', 'src/thread.h'] -include_dirs = [str(c_dir)] - -def setup_thread_so(): - # XXX this is quiiiiiiiite a hack! - files = [c_dir.join('src', 'thread.c')] - modname = '_thread' - cache_c_module(files, modname, include_dirs=[str(c_dir)]) - return str(pypydir.join('_cache', modname)) + '.so' -libraries = [setup_thread_so()] +eci = ExternalCompilationInfo( + includes = ['unistd.h', 'src/thread.h'], + separate_module_sources=[''' + #include + #include + #include + '''], + include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))] +) def llexternal(name, args, result, **kwds): - return rffi.llexternal(name, args, result, includes=includes, - libraries=libraries, include_dirs=[str(c_dir)], + return rffi.llexternal(name, args, result, compilation_info=eci, **kwds) CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP], rffi.VOIDP)) @@ -38,8 +34,7 @@ c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.INT) TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock', - includes=includes, - include_dirs=include_dirs) + compilation_info=eci) c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], lltype.Void) c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], From fijal at codespeak.net Sun Nov 25 19:31:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 19:31:37 +0100 (CET) Subject: [pypy-svn] r49100 - pypy/dist/pypy/translator/c/src Message-ID: <20071125183137.0B23A81BA@code0.codespeak.net> Author: fijal Date: Sun Nov 25 19:31:37 2007 New Revision: 49100 Removed: pypy/dist/pypy/translator/c/src/ll_thread.h pypy/dist/pypy/translator/c/src/thread.c Log: Kill those files, not to tempt anyone to use them. From fijal at codespeak.net Sun Nov 25 23:01:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Nov 2007 23:01:04 +0100 (CET) Subject: [pypy-svn] r49156 - in pypy/dist/pypy/tool: . test Message-ID: <20071125220104.9DEFD81C5@code0.codespeak.net> Author: fijal Date: Sun Nov 25 23:01:02 2007 New Revision: 49156 Modified: pypy/dist/pypy/tool/gcc_cache.py pypy/dist/pypy/tool/test/test_gcc_cache.py Log: Fix gcc_cache. Not sure if it's used anymore in more than some obscure cases though.... Modified: pypy/dist/pypy/tool/gcc_cache.py ============================================================================== --- pypy/dist/pypy/tool/gcc_cache.py (original) +++ pypy/dist/pypy/tool/gcc_cache.py Sun Nov 25 23:01:02 2007 @@ -1,9 +1,10 @@ from pypy.tool.autopath import pypydir -from pypy.translator.tool.cbuild import build_executable +from pypy.translator.tool.cbuild import build_executable, ExternalCompilationInfo import md5 import py import distutils +import distutils.errors py.path.local(pypydir).join('_cache').ensure(dir=1) cache_dir = py.path.local(pypydir).join('_cache', 'gcc') Modified: pypy/dist/pypy/tool/test/test_gcc_cache.py ============================================================================== --- pypy/dist/pypy/tool/test/test_gcc_cache.py (original) +++ pypy/dist/pypy/tool/test/test_gcc_cache.py Sun Nov 25 23:01:02 2007 @@ -2,6 +2,7 @@ from pypy.tool.gcc_cache import * from pypy.tool.udir import udir import md5 +from pypy.translator.tool.cbuild import ExternalCompilationInfo def test_gcc_exec(): f = udir.join("x.c") @@ -18,8 +19,8 @@ cache_dir.join(md5.md5(f.read()).hexdigest()).remove() except: pass - assert build_executable_cache([f]) == "3\n" - assert build_executable_cache([f], compiler_exe="xxx") == "3\n" + assert build_executable_cache([f], ExternalCompilationInfo()) == "3\n" + assert build_executable_cache([f], ExternalCompilationInfo(), compiler_exe="xxx") == "3\n" def test_gcc_ask(): f = udir.join("y.c") @@ -33,5 +34,5 @@ cache_dir.join(md5.md5(f.read()).hexdigest()).remove() except: pass - assert try_compile_cache([f]) - assert try_compile_cache([f], compiler_exe="xxx") + assert try_compile_cache([f], ExternalCompilationInfo()) + assert try_compile_cache([f], ExternalCompilationInfo(), compiler_exe="xxx") From pedronis at codespeak.net Mon Nov 26 09:33:48 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 26 Nov 2007 09:33:48 +0100 (CET) Subject: [pypy-svn] r49157 - pypy/dist/pypy/interpreter/pyparser/test Message-ID: <20071126083348.BBBA38160@code0.codespeak.net> Author: pedronis Date: Mon Nov 26 09:33:47 2007 New Revision: 49157 Modified: pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Log: one more test we ended up writing around here when resuing some of this code Modified: pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Mon Nov 26 09:33:47 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \ match_encoding_declaration from pypy.interpreter.pyparser.grammar import Token, GrammarElement -from pypy.interpreter.pyparser.pythonparse import make_pyparser +from pypy.interpreter.pyparser.pythonparse import make_pyparser, _check_for_encoding P = make_pyparser('2.4') @@ -108,3 +108,17 @@ for comment, encoding in checks: res = match_encoding_declaration(comment) assert res == encoding, "Failed on (%s), %s != %s" % (comment, res, encoding) + + +def test_check_for_enconding(): + + res = _check_for_encoding("# foo") + assert res is None + res = _check_for_encoding("# -*- coding: ascii -*- ") + assert res == "ascii" + res = _check_for_encoding("# -*- coding: iso-8859-15 -*- ") + assert res == "iso-8859-15" + res = _check_for_encoding("\n # -*- coding: iso-8859-15 -*- \n") + assert res == "iso-8859-15" + res = _check_for_encoding("\n\n # -*- coding: iso-8859-15 -*- \n") + assert res is None From fijal at codespeak.net Mon Nov 26 12:49:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 12:49:38 +0100 (CET) Subject: [pypy-svn] r49160 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071126114938.F09548155@code0.codespeak.net> Author: fijal Date: Mon Nov 26 12:49:37 2007 New Revision: 49160 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: Flights are boring. Adapt llvm to eci. Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Mon Nov 26 12:49:37 2007 @@ -7,6 +7,7 @@ from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype from pypy.tool.udir import udir +from StringIO import StringIO def get_c_cpath(): from pypy.translator.c import genc @@ -35,7 +36,7 @@ # call boehm finalizers need to be fastcc -def generate_ll(ccode, default_cconv, c_include_dirs, call_funcnames=[]): +def generate_ll(ccode, default_cconv, eci, call_funcnames=[]): call_funcnames = ['@LLVM_RPython_StartupCode'] + call_funcnames define_funcnames = ['@pypy_malloc', '@pypy_malloc_atomic', @@ -51,22 +52,28 @@ f.write(ccode) f.close() - plain = filename[:-2] - includes = get_incdirs(c_include_dirs) - cmd = "llvm-gcc -emit-llvm -O0 -S %s %s.c -o %s.ll 2>&1" % ( - includes, plain, plain) + includes = get_incdirs(eci.include_dirs) + eci = eci.convert_sources_to_files() + c_files = [filename] + list(eci.separate_module_files) + read_lines = [] + for name in c_files: + # XXX this is evil... + plain = name[:-2] + cmd = "llvm-gcc -emit-llvm -O0 -S %s %s.c -o %s.ll 2>&1" % ( + includes, plain, plain) - if os.system(cmd) != 0: - raise Exception("Failed to run '%s'" % cmd) + if os.system(cmd) != 0: + raise Exception("Failed to run '%s'" % cmd) - llcode = open(plain + '.ll').read() + llcode = open(plain + '.ll').read() + read_lines += llcode.split("\n") # strip lines lines = [] calltag, declaretag, definetag = 'call ', 'declare ', 'define ' - for line in llcode.split('\n'): + for line in read_lines: # get rid of any of the structs that llvm-gcc introduces to struct types line = line.replace("%struct.", "%") @@ -109,7 +116,7 @@ lines.append("declare ccc void @abort()") return'\n'.join(lines) -def generate_c(db, entrynode, c_includes, c_sources, standalone): +def generate_c(db, entrynode, eci, standalone): ccode = [] if standalone: @@ -123,14 +130,10 @@ ccode.append('%s' % db.gcpolicy.genextern_code()) # ask rffi for includes/source - for c_include in c_includes: - ccode.append('#include <%s>' % c_include) - - ccode.append('') - - for c_source in c_sources: - ccode.append(c_source) - + s = StringIO() + eci.write_c_header(s) + ccode.append(s.getvalue()) + ccode.append('') # append our source file Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Mon Nov 26 12:49:37 2007 @@ -7,23 +7,7 @@ self.db = db self.value = value self.name = "@" + name - - def external_c_source(self): - # return a list of unique includes and sources in C - c_include_dirs = [] - c_includes = [] - c_sources = [] - - if hasattr(self.value, 'include_dirs'): - c_include_dirs = list(self.value.include_dirs) - - if hasattr(self.value, 'includes'): - c_includes = list(self.value.includes) - - if hasattr(self.value, 'sources'): - c_sources = list(self.value.sources) - - return c_include_dirs, c_includes, c_sources + self.compilation_info = getattr(value, 'compilation_info', None) def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Nov 26 12:49:37 2007 @@ -20,6 +20,7 @@ from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import rffi +from pypy.translator.tool.cbuild import ExternalCompilationInfo def augment_entrypoint(translator, entrypoint): bk = translator.annotator.bookkeeper @@ -184,21 +185,14 @@ return c.value._obj def generate_ll_externs(self, codewriter): - c_include_dirs = {} - c_includes = {} - c_sources = {} - + all = [] for node in self.db.getnodes(): - include_dirs, includes, sources = node.external_c_source() - for incdir in include_dirs: - c_include_dirs[incdir] = True - for include in includes: - c_includes[include] = True - for source in sources: - c_sources[source] = True - - ccode = generate_c(self.db, self.entrynode, c_includes, c_sources, self.standalone) - self.llcode = generate_ll(ccode, codewriter.cconv, c_include_dirs, self.db.extern_to_funcnodes) + eci = getattr(node, 'compilation_info', None) + if eci is not None: + all.append(eci) + eci = ExternalCompilationInfo().merge(*all) + ccode = generate_c(self.db, self.entrynode, eci, self.standalone) + self.llcode = generate_ll(ccode, codewriter.cconv, eci, self.db.extern_to_funcnodes) def create_codewriter(self): # prevent running the same function twice in a test Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Mon Nov 26 12:49:37 2007 @@ -32,10 +32,6 @@ def post_setup_transform(self): pass - def external_c_source(self): - " return a list of unique includes and sources in C " - return [], [], [] - def writesetupcode(self, codewriter): " pre entry-point setup " pass Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Nov 26 12:49:37 2007 @@ -69,7 +69,8 @@ raise AttributeError, name def wrapfn(fn): - def wrapped(*args): + def wrapped(*args, **kwds): + # XXX implement fishing, ie expected_extra_mallocs callargs = [] for a in args: if hasattr(a, 'chars'): From fijal at codespeak.net Mon Nov 26 12:50:44 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 12:50:44 +0100 (CET) Subject: [pypy-svn] r49161 - in pypy/dist/pypy/rpython: lltypesystem/test module/test Message-ID: <20071126115044.4452D8155@code0.codespeak.net> Author: fijal Date: Mon Nov 26 12:50:43 2007 New Revision: 49161 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: Adapt rffi tests to run on both c and llvm, probably this should be organized differently. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Mon Nov 26 12:50:43 2007 @@ -2,7 +2,8 @@ import py from pypy.rpython.lltypesystem.rffi import * from pypy.rlib.rposix import get_errno, set_errno -from pypy.translator.c.test.test_genc import compile +from pypy.translator.c.test.test_genc import compile as compile_c +from pypy.translator.llvm.test.runtest import compile_function as compile_llvm from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc from pypy.rpython.lltypesystem import lltype from pypy.tool.udir import udir @@ -15,458 +16,459 @@ from pypy.objspace.flow.model import summary from pypy.translator.tool.cbuild import ExternalCompilationInfo -def test_basic(): - c_source = py.code.Source(""" - int someexternalfunction(int x) - { - return (x + 3); - } - """) - - eci = ExternalCompilationInfo(separate_module_sources=[c_source]) - z = llexternal('someexternalfunction', [Signed], Signed, - compilation_info=eci) - - def f(): - return z(8) - - xf = compile(f, []) - assert xf() == 8+3 - -def test_hashdefine(): - h_source = """ - #define X(i) (i+3) - """ - - h_file = udir.join("stuff.h") - h_file.write(h_source) - - eci = ExternalCompilationInfo(includes=['stuff.h'], - include_dirs=[udir]) - z = llexternal('X', [Signed], Signed, compilation_info=eci) - - def f(): - return z(8) - - xf = compile(f, []) - assert xf() == 8+3 - -def test_string(): - eci = ExternalCompilationInfo(includes=['string.h']) - z = llexternal('strlen', [CCHARP], Signed, compilation_info=eci) - - def f(): - s = str2charp("xxx") - res = z(s) - free_charp(s) - return res - - xf = compile(f, [], backendopt=False) - assert xf() == 3 - -def test_string_reverse(): - c_source = py.code.Source(""" - #include - - char *f(char* arg) - { - char *ret; - ret = (char*)malloc(strlen(arg) + 1); - strcpy(ret, arg); - return ret; - } - """) - eci = ExternalCompilationInfo(separate_module_sources=[c_source]) - z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) - - def f(): - s = str2charp("xxx") - l_res = z(s) - res = charp2str(l_res) - lltype.free(l_res, flavor='raw') - free_charp(s) - return len(res) - - xf = compile(f, [], backendopt=False) - assert xf(expected_extra_mallocs=-1) == 3 - -def test_stringstar(): - c_source = """ - #include - - int f(char *args[]) { - char **p = args; - int l = 0; - while (*p) { - l += strlen(*p); - p++; +class BaseTestRffi: + def test_basic(self): + c_source = py.code.Source(""" + int someexternalfunction(int x) + { + return (x + 3); } - return (l); - } - """ - eci = ExternalCompilationInfo(separate_module_sources=[c_source]) - z = llexternal('f', [CCHARPP], Signed, compilation_info=eci) - - def f(): - l = ["xxx", "x", "xxxx"] - ss = liststr2charpp(l) - result = z(ss) - free_charpp(ss) - return result - - xf = compile(f, [], backendopt=False) - assert xf() == 8 - -def test_struct(): - h_source = """ - #ifndef _MY_SOURCE_H - #define _MY_SOURCE_H - struct xx { - int one; - char two; - int three; - }; - #endif - """ - h_file = udir.join("structxx.h") - h_file.write(h_source) - - c_source = """ - #include - - int f(struct xx* z) - { - return (z->one + z->three); - } - """ - TP = CStructPtr('xx', ('one', INT), ('two', Char), ('three', INT)) - - eci = ExternalCompilationInfo( - includes=['structxx.h'], - include_dirs=[udir], - separate_module_sources=[c_source] - ) - z = llexternal('f', [TP], INT, compilation_info=eci) - - def f(): - struct = lltype.malloc(TP.TO, flavor='raw') - struct.c_one = cast(INT, 3) - struct.c_two = '\x33' - struct.c_three = cast(INT, 5) - result = z(struct) - lltype.free(struct, flavor='raw') - return cast(LONG, result) - - fn = compile(f, [], backendopt=False) - assert fn() == 8 - -def test_externvar(): - import os - - def f(): - set_errno(12) - return get_errno() + """) - def g(): - try: - os.write(12312312, "xxx") - except OSError: - pass - return get_errno() - - fn = compile(f, []) - assert fn() == 12 - gn = compile(g, []) - import errno - assert gn() == errno.EBADF - -def test_external_callable(): - """ Try to call some llexternal function with llinterp - """ - z = llexternal('z', [Signed], Signed, _callable=lambda x:x+1) - - def f(): - return z(2) - - res = interpret(f, []) - assert res == 3 - -def test_extra_include_dirs(): - udir.ensure("incl", dir=True) - udir.join("incl", "incl.h").write("#define C 3") - c_source = py.code.Source(""" - #include - int fun () - { - return (C); - } - """) - eci = ExternalCompilationInfo( - includes=['incl.h'], - include_dirs=[str(udir.join('incl'))], - separate_module_sources=[c_source] - ) - z = llexternal('fun', [], Signed, compilation_info=eci) - - def f(): - return z() - - res = compile(f, []) - assert res() == 3 - -def test_size_t_sign(): - assert r_size_t(-1) > 0 - -def test_cast(): - res = cast(SIZE_T, -1) - assert type(res) is r_size_t - assert res == r_size_t(-1) - -def test_compile_cast(): - def f(n): - return cast(SIZE_T, n) - - f1 = compile(f, [int]) - res = f1(-1) - assert res == r_size_t(-1) - -def test_opaque_type(): - h_source = py.code.Source(""" - struct stuff { - char data[38]; - }; - - char get(struct stuff* x) - { - x->data[13] = 'a'; - return x->data[13]; - } - """) - # if it doesn't segfault, than we probably malloced it :-) - h_file = udir.join("opaque.h") - h_file.write(h_source) - - from pypy.rpython.tool import rffi_platform - eci = ExternalCompilationInfo( - includes=['opaque.h'], - include_dirs=[str(udir)] - ) - STUFFP = COpaquePtr('struct stuff', compilation_info=eci) - - ll_get = llexternal('get', [STUFFP], lltype.Char, compilation_info=eci) - - def f(): - ll_stuff = lltype.malloc(STUFFP.TO, flavor='raw') - result = ll_get(ll_stuff) - lltype.free(ll_stuff, flavor='raw') - return result - - f1 = compile(f, []) - assert f1() == 'a' - -def test_rffi_sizeof(): - try: - import ctypes - except ImportError: - py.test.skip("Cannot test without ctypes") - cache = { - lltype.Signed: ctypes.c_long, - lltype.Unsigned: ctypes.c_ulong, - lltype.UniChar: ctypes.c_uint, - lltype.Char: ctypes.c_ubyte, - DOUBLE: ctypes.c_double, - SIGNEDCHAR: ctypes.c_byte, - UCHAR: ctypes.c_ubyte, - SHORT: ctypes.c_short, - USHORT: ctypes.c_ushort, - INT: ctypes.c_int, - UINT: ctypes.c_uint, - LONG: ctypes.c_long, - ULONG: ctypes.c_ulong, - LONGLONG: ctypes.c_longlong, - ULONGLONG: ctypes.c_ulonglong, - SIZE_T: ctypes.c_size_t, - } + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('someexternalfunction', [Signed], Signed, + compilation_info=eci) + + def f(): + return z(8) + + xf = self.compile(f, []) + assert xf() == 8+3 - for ll, ctp in cache.items(): - assert sizeof(ll) == ctypes.sizeof(ctp) - assert not size_and_sign(lltype.Signed)[1] - assert not size_and_sign(lltype.Char)[1] - assert not size_and_sign(lltype.UniChar)[1] - assert size_and_sign(UINT)[1] - -def test_rffi_offsetof(): - import struct - from pypy.rpython.tool import rffi_platform - S = rffi_platform.getstruct("struct S", - """ - struct S { - short a; - int b, c; - }; """, - [("a", INT), - ("b", INT), - ("c", INT)]) - assert sizeof(S) == struct.calcsize("hii") - assert offsetof(S, "c_a") == 0 - assert offsetof(S, "c_b") == struct.calcsize("hi") - struct.calcsize("i") - assert offsetof(S, "c_c") == struct.calcsize("hii") - struct.calcsize("i") - -def test_prebuilt_constant(): - py.test.skip("Think how to do it sane") - h_source = py.code.Source(""" - int x = 3; - char** z = NULL; - #endif - """) - h_include = udir.join('constants.h') - h_include.write(h_source) - - eci = ExternalCompilationInfo(includes=['stdio.h', - str(h_include.basename)], - include_dirs=[str(udir)]) - - get_x, set_x = CExternVariable(lltype.Signed, 'x', eci) - get_z, set_z = CExternVariable(CCHARPP, 'z', eci) - - def f(): - one = get_x() - set_x(13) - return one + get_x() + def test_hashdefine(self): + h_source = """ + #define X(i) (i+3) + """ + + h_file = udir.join("stuff.h") + h_file.write(h_source) + + eci = ExternalCompilationInfo(includes=['stuff.h'], + include_dirs=[udir]) + z = llexternal('X', [Signed], Signed, compilation_info=eci) - def g(): - l = liststr2charpp(["a", "b", "c"]) + def f(): + return z(8) + + xf = self.compile(f, []) + assert xf() == 8+3 + + def test_string(self): + eci = ExternalCompilationInfo(includes=['string.h']) + z = llexternal('strlen', [CCHARP], Signed, compilation_info=eci) + + def f(): + s = str2charp("xxx") + res = z(s) + free_charp(s) + return res + + xf = self.compile(f, [], backendopt=False) + assert xf() == 3 + + def test_string_reverse(self): + c_source = py.code.Source(""" + #include + + char *f(char* arg) + { + char *ret; + ret = (char*)malloc(strlen(arg) + 1); + strcpy(ret, arg); + return ret; + } + """) + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) + + def f(): + s = str2charp("xxx") + l_res = z(s) + res = charp2str(l_res) + lltype.free(l_res, flavor='raw') + free_charp(s) + return len(res) + + xf = self.compile(f, [], backendopt=False) + assert xf(expected_extra_mallocs=-1) == 3 + + def test_stringstar(self): + c_source = """ + #include + + int f(char *args[]) { + char **p = args; + int l = 0; + while (*p) { + l += strlen(*p); + p++; + } + return (l); + } + """ + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = llexternal('f', [CCHARPP], Signed, compilation_info=eci) + + def f(): + l = ["xxx", "x", "xxxx"] + ss = liststr2charpp(l) + result = z(ss) + free_charpp(ss) + return result + + xf = self.compile(f, [], backendopt=False) + assert xf() == 8 + + def test_struct(self): + h_source = """ + #ifndef _MY_SOURCE_H + #define _MY_SOURCE_H + struct xx { + int one; + char two; + int three; + }; + #endif + """ + h_file = udir.join("structxx.h") + h_file.write(h_source) + + c_source = """ + #include + + int f(struct xx* z) + { + return (z->one + z->three); + } + """ + TP = CStructPtr('xx', ('one', INT), ('two', Char), ('three', INT)) + + eci = ExternalCompilationInfo( + includes=['structxx.h'], + include_dirs=[udir], + separate_module_sources=[c_source] + ) + z = llexternal('f', [TP], INT, compilation_info=eci) + + def f(): + struct = lltype.malloc(TP.TO, flavor='raw') + struct.c_one = cast(INT, 3) + struct.c_two = '\x33' + struct.c_three = cast(INT, 5) + result = z(struct) + lltype.free(struct, flavor='raw') + return cast(LONG, result) + + fn = self.compile(f, [], backendopt=False) + assert fn() == 8 + + def test_externvar(self): + import os + + def f(): + set_errno(12) + return get_errno() + + def g(): + try: + os.write(12312312, "xxx") + except OSError: + pass + return get_errno() + + fn = self.compile(f, []) + assert fn() == 12 + gn = self.compile(g, []) + import errno + assert gn() == errno.EBADF + + + def test_extra_include_dirs(self): + udir.ensure("incl", dir=True) + udir.join("incl", "incl.h").write("#define C 3") + c_source = py.code.Source(""" + #include + int fun () + { + return (C); + } + """) + eci = ExternalCompilationInfo( + includes=['incl.h'], + include_dirs=[str(udir.join('incl'))], + separate_module_sources=[c_source] + ) + z = llexternal('fun', [], Signed, compilation_info=eci) + + def f(): + return z() + + res = self.compile(f, []) + assert res() == 3 + + def test_compile_cast(self): + def f(n): + return cast(SIZE_T, n) + + f1 = self.compile(f, [int]) + res = f1(-1) + assert res == r_size_t(-1) + + def test_opaque_type(self): + h_source = py.code.Source(""" + struct stuff { + char data[38]; + }; + + char get(struct stuff* x) + { + x->data[13] = 'a'; + return x->data[13]; + } + """) + # if it doesn't segfault, than we probably malloced it :-) + h_file = udir.join("opaque.h") + h_file.write(h_source) + + from pypy.rpython.tool import rffi_platform + eci = ExternalCompilationInfo( + includes=['opaque.h'], + include_dirs=[str(udir)] + ) + STUFFP = COpaquePtr('struct stuff', compilation_info=eci) + + ll_get = llexternal('get', [STUFFP], lltype.Char, compilation_info=eci) + + def f(): + ll_stuff = lltype.malloc(STUFFP.TO, flavor='raw') + result = ll_get(ll_stuff) + lltype.free(ll_stuff, flavor='raw') + return result + + f1 = self.compile(f, []) + assert f1() == 'a' + + def test_prebuilt_constant(self): + py.test.skip("Think how to do it sane") + h_source = py.code.Source(""" + int x = 3; + char** z = NULL; + #endif + """) + h_include = udir.join('constants.h') + h_include.write(h_source) + + eci = ExternalCompilationInfo(includes=['stdio.h', + str(h_include.basename)], + include_dirs=[str(udir)]) + + get_x, set_x = CExternVariable(lltype.Signed, 'x', eci) + get_z, set_z = CExternVariable(CCHARPP, 'z', eci) + + def f(): + one = get_x() + set_x(13) + return one + get_x() + + def g(): + l = liststr2charpp(["a", "b", "c"]) + try: + set_z(l) + return charp2str(get_z()[2]) + finally: + free_charpp(l) + + fn = self.compile(f, []) + assert fn() == 16 + gn = self.compile(g, []) + assert gn() == "c" + +class TestRffiInternals: + def test_struct_create(self): + X = CStruct('xx', ('one', INT)) + def f(): + p = make(X, c_one=cast(INT, 3)) + res = p.c_one + lltype.free(p, flavor='raw') + return cast(LONG, res) + assert f() == 3 + assert interpret(f, []) == 3 + + def test_structcopy(self): + X2 = lltype.Struct('X2', ('x', LONG)) + X1 = lltype.Struct('X1', ('a', LONG), ('x2', X2), ('p', lltype.Ptr(X2))) + def f(): + p2 = make(X2, x=123) + p1 = make(X1, a=5, p=p2) + p1.x2.x = 456 + p1bis = make(X1) + p2bis = make(X2) + structcopy(p1bis, p1) + assert p1bis.a == 5 + assert p1bis.x2.x == 456 + assert p1bis.p == p2 + structcopy(p2bis, p2) + res = p2bis.x + lltype.free(p2bis, flavor='raw') + lltype.free(p1bis, flavor='raw') + lltype.free(p2, flavor='raw') + lltype.free(p1, flavor='raw') + return res + assert f() == 123 + res = interpret(f, []) + assert res == 123 + + def test_implicit_cast(self): + z = llexternal('z', [USHORT, ULONG, USHORT, DOUBLE], USHORT) + + def f(x, y, xx, yy): + return z(x, y, xx, yy) + + a = RPythonAnnotator() + r = a.build_types(f, [int, int, int, int]) + rtyper = RPythonTyper(a) + rtyper.specialize() + a.translator.rtyper = rtyper + backend_optimizations(a.translator) + if option.view: + a.translator.view() + graph = graphof(a.translator, f) + s = summary(graph) + # there should be not too many operations here by now + assert s == {'cast_int_to_uint': 1, 'direct_call': 1, 'cast_primitive': 2, + 'cast_int_to_float': 1} + + def test_stringpolicy1(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, compilation_info=eci) + def f(): + return cast(LONG, strlen("Xxx")) + assert interpret(f, [], backendopt=True) == 3 + + def test_stringpolicy3(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], INT, compilation_info=eci) + def f(): + ll_str = str2charp("Xxx") + res = strlen(ll_str) + lltype.free(ll_str, flavor='raw') + return res + + assert interpret(f, [], backendopt=True) == 3 + + def test_stringpolicy_mixed(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, + compilation_info=eci) + def f(): + res1 = strlen("abcd") + ll_str = str2charp("Xxx") + res2 = strlen(ll_str) + lltype.free(ll_str, flavor='raw') + return cast(LONG, res1*10 + res2) + + assert interpret(f, [], backendopt=True) == 43 + + def test_around_extcall(self): + import os + from pypy.annotation import model as annmodel + from pypy.rlib.objectmodel import invoke_around_extcall + from pypy.rpython.extfuncregistry import register_external + read_fd, write_fd = os.pipe() try: - set_z(l) - return charp2str(get_z()[2]) + # we need an external function that is not going to get wrapped around + # before()/after() calls, in order to call it from before()/after()... + def mywrite(s): + os.write(write_fd, s) + def llimpl(s): + s = ''.join(s.chars) + os.write(write_fd, s) + register_external(mywrite, [str], annmodel.s_None, 'll_mywrite', + llfakeimpl=llimpl, sandboxsafe=True) + + def before(): + mywrite("B") + def after(): + mywrite("A") + def f(): + os.write(write_fd, "-") + invoke_around_extcall(before, after) + os.write(write_fd, "E") + + interpret(f, []) + data = os.read(read_fd, 99) + assert data == "-BEA" + finally: - free_charpp(l) + os.close(write_fd) + os.close(read_fd) - fn = compile(f, []) - assert fn() == 16 - gn = compile(g, []) - assert gn() == "c" - -def test_struct_create(): - X = CStruct('xx', ('one', INT)) - def f(): - p = make(X, c_one=cast(INT, 3)) - res = p.c_one - lltype.free(p, flavor='raw') - return cast(LONG, res) - assert f() == 3 - assert interpret(f, []) == 3 - -def test_structcopy(): - X2 = lltype.Struct('X2', ('x', LONG)) - X1 = lltype.Struct('X1', ('a', LONG), ('x2', X2), ('p', lltype.Ptr(X2))) - def f(): - p2 = make(X2, x=123) - p1 = make(X1, a=5, p=p2) - p1.x2.x = 456 - p1bis = make(X1) - p2bis = make(X2) - structcopy(p1bis, p1) - assert p1bis.a == 5 - assert p1bis.x2.x == 456 - assert p1bis.p == p2 - structcopy(p2bis, p2) - res = p2bis.x - lltype.free(p2bis, flavor='raw') - lltype.free(p1bis, flavor='raw') - lltype.free(p2, flavor='raw') - lltype.free(p1, flavor='raw') - return res - assert f() == 123 - res = interpret(f, []) - assert res == 123 - -def test_implicit_cast(): - z = llexternal('z', [USHORT, ULONG, USHORT, DOUBLE], USHORT) - - def f(x, y, xx, yy): - return z(x, y, xx, yy) - - a = RPythonAnnotator() - r = a.build_types(f, [int, int, int, int]) - rtyper = RPythonTyper(a) - rtyper.specialize() - a.translator.rtyper = rtyper - backend_optimizations(a.translator) - if option.view: - a.translator.view() - graph = graphof(a.translator, f) - s = summary(graph) - # there should be not too many operations here by now - assert s == {'cast_int_to_uint': 1, 'direct_call': 1, 'cast_primitive': 2, - 'cast_int_to_float': 1} - - -def test_stringpolicy1(): - eci = ExternalCompilationInfo(includes=['string.h']) - strlen = llexternal('strlen', [CCHARP], SIZE_T, compilation_info=eci) - def f(): - return cast(LONG, strlen("Xxx")) - assert interpret(f, [], backendopt=True) == 3 - -def test_stringpolicy3(): - eci = ExternalCompilationInfo(includes=['string.h']) - strlen = llexternal('strlen', [CCHARP], INT, compilation_info=eci) - def f(): - ll_str = str2charp("Xxx") - res = strlen(ll_str) - lltype.free(ll_str, flavor='raw') - return res - - assert interpret(f, [], backendopt=True) == 3 - -def test_stringpolicy_mixed(): - eci = ExternalCompilationInfo(includes=['string.h']) - strlen = llexternal('strlen', [CCHARP], SIZE_T, - compilation_info=eci) - def f(): - res1 = strlen("abcd") - ll_str = str2charp("Xxx") - res2 = strlen(ll_str) - lltype.free(ll_str, flavor='raw') - return cast(LONG, res1*10 + res2) - - assert interpret(f, [], backendopt=True) == 43 - -def test_around_extcall(): - import os - from pypy.annotation import model as annmodel - from pypy.rlib.objectmodel import invoke_around_extcall - from pypy.rpython.extfuncregistry import register_external - read_fd, write_fd = os.pipe() - try: - # we need an external function that is not going to get wrapped around - # before()/after() calls, in order to call it from before()/after()... - def mywrite(s): - os.write(write_fd, s) - def llimpl(s): - s = ''.join(s.chars) - os.write(write_fd, s) - register_external(mywrite, [str], annmodel.s_None, 'll_mywrite', - llfakeimpl=llimpl, sandboxsafe=True) - - def before(): - mywrite("B") - def after(): - mywrite("A") - def f(): - os.write(write_fd, "-") - invoke_around_extcall(before, after) - os.write(write_fd, "E") - - interpret(f, []) - data = os.read(read_fd, 99) - assert data == "-BEA" - - finally: - os.close(write_fd) - os.close(read_fd) + def test_external_callable(self): + """ Try to call some llexternal function with llinterp + """ + z = llexternal('z', [Signed], Signed, _callable=lambda x:x+1) + + def f(): + return z(2) + + res = interpret(f, []) + assert res == 3 + def test_size_t_sign(self): + assert r_size_t(-1) > 0 + + def test_cast(self): + res = cast(SIZE_T, -1) + assert type(res) is r_size_t + assert res == r_size_t(-1) + + def test_rffi_sizeof(self): + try: + import ctypes + except ImportError: + py.test.skip("Cannot test without ctypes") + cache = { + lltype.Signed: ctypes.c_long, + lltype.Unsigned: ctypes.c_ulong, + lltype.UniChar: ctypes.c_uint, + lltype.Char: ctypes.c_ubyte, + DOUBLE: ctypes.c_double, + SIGNEDCHAR: ctypes.c_byte, + UCHAR: ctypes.c_ubyte, + SHORT: ctypes.c_short, + USHORT: ctypes.c_ushort, + INT: ctypes.c_int, + UINT: ctypes.c_uint, + LONG: ctypes.c_long, + ULONG: ctypes.c_ulong, + LONGLONG: ctypes.c_longlong, + ULONGLONG: ctypes.c_ulonglong, + SIZE_T: ctypes.c_size_t, + } + + for ll, ctp in cache.items(): + assert sizeof(ll) == ctypes.sizeof(ctp) + assert not size_and_sign(lltype.Signed)[1] + assert not size_and_sign(lltype.Char)[1] + assert not size_and_sign(lltype.UniChar)[1] + assert size_and_sign(UINT)[1] + + def test_rffi_offsetof(self): + import struct + from pypy.rpython.tool import rffi_platform + S = rffi_platform.getstruct("struct S", + """ + struct S { + short a; + int b, c; + }; """, + [("a", INT), + ("b", INT), + ("c", INT)]) + assert sizeof(S) == struct.calcsize("hii") + assert offsetof(S, "c_a") == 0 + assert offsetof(S, "c_b") == struct.calcsize("hi") - struct.calcsize("i") + assert offsetof(S, "c_c") == struct.calcsize("hii") - struct.calcsize("i") ARRAY_OF_CHAR = lltype.Array(CHAR, hints={'nolength': True}) @@ -480,6 +482,25 @@ for i in xrange(len(data) - 2): assert a2[i] == a[i + 2] lltype.free(a, flavor='raw') - + def test_ptradd_interpret(): interpret(test_ptradd, []) + + +class TestCRffi(BaseTestRffi): + def compile(self, func, args, **kwds): + return compile_c(func, args, **kwds) + +class TestLLVMRffi(BaseTestRffi): + def compile(self, func, args, **kwds): + # pfff.... + if 'backendopt' in kwds: + kwds['optimize'] = kwds['backendopt'] + del kwds['backendopt'] + return compile_llvm(func, args, **kwds) + + def test_hashdefine(self): + py.test.skip("FIXME") + + def test_opaque_type(self): + py.test.skip("FIXME") Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Mon Nov 26 12:50:43 2007 @@ -1,8 +1,6 @@ import py from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.tool.udir import udir -from pypy.translator.llvm.test.runtest import compile_function as compile_llvm -from pypy.translator.c.test.test_genc import compile import os exec 'import %s as posix' % os.name @@ -132,27 +130,24 @@ return os.getuid() assert self.interpret(f, []) == f() + + + def test_os_wstar(self): + from pypy.rpython.module.ll_os import RegisterOs + for name in RegisterOs.w_star: + if not hasattr(os, name): + continue + def fun(s): + return getattr(os, name)(s) + + for value in [0, 1, 127, 128, 255]: + res = self.interpret(fun, [value]) + assert res == fun(value) + + class TestLLtype(BaseTestPosix, LLRtypeMixin): pass class TestOOtype(BaseTestPosix, OORtypeMixin): def test_fstat(self): py.test.skip("ootypesystem does not support os.fstat") - - -def os_wstar_tester(compile): - from pypy.rpython.module.ll_os import RegisterOs - for name in RegisterOs.w_star: - if not hasattr(os, name): - continue - def fun(s): - return getattr(os, name)(s) - - fun_c = compile(fun, [int]) - for value in [0, 1, 127, 128, 255]: - res = fun_c(value) - assert res == fun(value) - -def test_os_wstar(): - yield os_wstar_tester, compile - yield os_wstar_tester, compile_llvm From fijal at codespeak.net Mon Nov 26 15:08:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 15:08:02 +0100 (CET) Subject: [pypy-svn] r49162 - pypy/dist/pypy/translator/c/src Message-ID: <20071126140802.657668157@code0.codespeak.net> Author: fijal Date: Mon Nov 26 15:08:00 2007 New Revision: 49162 Modified: pypy/dist/pypy/translator/c/src/stack.h Log: Ooops. Wrong include order Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Mon Nov 26 15:08:00 2007 @@ -2,6 +2,7 @@ /************************************************************/ /*** C header subsection: stack operations ***/ +#include #ifndef MAX_STACK_SIZE # define MAX_STACK_SIZE (1 << 19) From fijal at codespeak.net Mon Nov 26 15:08:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 15:08:13 +0100 (CET) Subject: [pypy-svn] r49163 - pypy/dist/pypy/rlib Message-ID: <20071126140813.5656E8188@code0.codespeak.net> Author: fijal Date: Mon Nov 26 15:08:12 2007 New Revision: 49163 Modified: pypy/dist/pypy/rlib/rmmap.py Log: mute gcc warnings. Modified: pypy/dist/pypy/rlib/rmmap.py ============================================================================== --- pypy/dist/pypy/rlib/rmmap.py (original) +++ pypy/dist/pypy/rlib/rmmap.py Mon Nov 26 15:08:12 2007 @@ -31,7 +31,9 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( includes=includes, - pre_include_lines=['#define _GNU_SOURCE'] + pre_include_lines=['#ifndef _GNU_SOURCE', + '#define _GNU_SOURCE', + '#endif'] ) size_t = rffi_platform.SimpleType("size_t", rffi.LONG) off_t = rffi_platform.SimpleType("off_t", rffi.LONG) From fijal at codespeak.net Mon Nov 26 17:47:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 17:47:08 +0100 (CET) Subject: [pypy-svn] r49166 - pypy/dist/pypy/module/signal Message-ID: <20071126164708.BF6788138@code0.codespeak.net> Author: fijal Date: Mon Nov 26 17:47:08 2007 New Revision: 49166 Modified: pypy/dist/pypy/module/signal/interp_signal.py Log: Make signal module not use gencapicall any more Modified: pypy/dist/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/dist/pypy/module/signal/interp_signal.py (original) +++ pypy/dist/pypy/module/signal/interp_signal.py Mon Nov 26 17:47:08 2007 @@ -2,7 +2,10 @@ from pypy.interpreter.baseobjspace import W_Root, ObjSpace from pypy.interpreter.miscutils import Action import signal as cpy_signal - +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.tool.cbuild import ExternalCompilationInfo +import py +from pypy.tool import autopath def setup(): for key, value in cpy_signal.__dict__.items(): @@ -15,6 +18,19 @@ SIG_IGN = cpy_signal.SIG_IGN signal_names = list(setup()) +eci = ExternalCompilationInfo( + includes = ['stdlib.h', 'src/signals.h'], + separate_module_sources = ['#include '], + include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))] +) + +def external(name, args, result, **kwds): + return rffi.llexternal(name, args, result, compilation_info=eci, **kwds) + +pypysig_ignore = external('pypysig_ignore', [rffi.INT], lltype.Void) +pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) +pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) +pypysig_poll = external('pypysig_poll', [], rffi.INT) class CheckSignalAction(Action): """A repeatitive action at the space level, checking if the @@ -121,62 +137,3 @@ action.handlers_w[signum] = w_handler return old_handler signal.unwrap_spec = [ObjSpace, int, W_Root] - -# ____________________________________________________________ -# CPython and LLTypeSystem implementations - -from pypy.rpython.extregistry import ExtRegistryEntry - -signal_queue = [] # only for py.py, not for translated pypy-c's - -def pypysig_poll(): - "NOT_RPYTHON" - if signal_queue: - return signal_queue.pop(0) - else: - return -1 - -def pypysig_default(signum): - "NOT_RPYTHON" - cpy_signal.signal(signum, cpy_signal.SIG_DFL) # XXX error handling - -def pypysig_ignore(signum): - "NOT_RPYTHON" - cpy_signal.signal(signum, cpy_signal.SIG_IGN) # XXX error handling - -def _queue_handler(signum, frame): - if signum not in signal_queue: - signal_queue.append(signum) - -def pypysig_setflag(signum): - "NOT_RPYTHON" - cpy_signal.signal(signum, _queue_handler) - - -# lltyping - direct mapping to the C functions defined in -# translator/c/src/signals.h - -class Entry(ExtRegistryEntry): - _about_ = pypysig_poll - def compute_result_annotation(self): - from pypy.annotation import model as annmodel - return annmodel.SomeInteger() - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - hop.exception_cannot_occur() - return hop.llops.gencapicall("pypysig_poll", [], lltype.Signed, - includes=('src/signals.h',)) - -for _fn in [pypysig_default, pypysig_ignore, pypysig_setflag]: - class Entry(ExtRegistryEntry): - _about_ = _fn - funcname = _fn.func_name - def compute_result_annotation(self, s_signum): - return None - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - vlist = hop.inputargs(lltype.Signed) - hop.exception_cannot_occur() - hop.llops.gencapicall(self.funcname, vlist, - includes=('src/signals.h',)) -del _fn From fijal at codespeak.net Mon Nov 26 17:47:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 17:47:20 +0100 (CET) Subject: [pypy-svn] r49167 - pypy/dist/pypy/translator/c/src Message-ID: <20071126164720.975C78138@code0.codespeak.net> Author: fijal Date: Mon Nov 26 17:47:20 2007 New Revision: 49167 Modified: pypy/dist/pypy/translator/c/src/signals.h Log: HaHa! Modified: pypy/dist/pypy/translator/c/src/signals.h ============================================================================== --- pypy/dist/pypy/translator/c/src/signals.h (original) +++ pypy/dist/pypy/translator/c/src/signals.h Mon Nov 26 17:47:20 2007 @@ -1,6 +1,8 @@ /* some ifdefs from CPython's signalmodule.c... */ +#include + #ifdef MS_WINDOWS #include #endif From fijal at codespeak.net Mon Nov 26 19:00:43 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 19:00:43 +0100 (CET) Subject: [pypy-svn] r49168 - pypy/dist/pypy/module/_ffi/test Message-ID: <20071126180043.B72E08188@code0.codespeak.net> Author: fijal Date: Mon Nov 26 19:00:42 2007 New Revision: 49168 Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py Log: Stupid me. Fix tests. Modified: pypy/dist/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/dist/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/dist/pypy/module/_ffi/test/test__ffi.py Mon Nov 26 19:00:42 2007 @@ -90,8 +90,7 @@ } ''')) - compile_c_module([c_file], 'x', ExternalCompilationInfo()) - return str(udir.join('x.so')) + return compile_c_module([c_file], 'x', ExternalCompilationInfo()) prepare_c_example = staticmethod(prepare_c_example) def setup_class(cls): From fijal at codespeak.net Mon Nov 26 19:16:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 19:16:10 +0100 (CET) Subject: [pypy-svn] r49169 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071126181610.DC9E08145@code0.codespeak.net> Author: fijal Date: Mon Nov 26 19:16:10 2007 New Revision: 49169 Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py Log: Checkin a test showcasing problem with strtod, not to forget. Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_standalone.py Mon Nov 26 19:16:10 2007 @@ -78,3 +78,12 @@ #assert 0 < res <= 84 +def test_strtod(): + py.test.skip("Explodes") + def entry_point(args): + print float(args[1]) + return 0 + + exe_name = 'test_strtod' + compile_standalone(entry_point, exe_name=exe_name) + data = cmdexed(exe_name, '3.13e1') From fijal at codespeak.net Mon Nov 26 19:38:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 19:38:23 +0100 (CET) Subject: [pypy-svn] r49170 - pypy/branch/cleanup-headers Message-ID: <20071126183823.89BB0814A@code0.codespeak.net> Author: fijal Date: Mon Nov 26 19:38:22 2007 New Revision: 49170 Added: pypy/branch/cleanup-headers/ - copied from r49169, pypy/dist/ Log: A new branch for cleaning up the #ifdef PYPY_NOT_MAIN_FILE mess From fijal at codespeak.net Mon Nov 26 19:45:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Nov 2007 19:45:12 +0100 (CET) Subject: [pypy-svn] r49171 - pypy/dist/pypy/translator/llvm Message-ID: <20071126184512.A852F814A@code0.codespeak.net> Author: fijal Date: Mon Nov 26 19:45:12 2007 New Revision: 49171 Modified: pypy/dist/pypy/translator/llvm/externs2ll.py Log: Test still explodes, but now I don't understand why. Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Mon Nov 26 19:45:12 2007 @@ -129,10 +129,12 @@ # ask gcpolicy for any code needed ccode.append('%s' % db.gcpolicy.genextern_code()) + ccode.append("#define PYPY_NOT_MAIN_FILE") # ask rffi for includes/source s = StringIO() eci.write_c_header(s) ccode.append(s.getvalue()) + ccode.append("#undef PYPY_NOT_MAIN_FILE") ccode.append('') From xoraxax at codespeak.net Mon Nov 26 21:46:38 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 21:46:38 +0100 (CET) Subject: [pypy-svn] r49172 - pypy/dist/pypy/translator/c/src Message-ID: <20071126204638.91C6C81C1@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 21:46:37 2007 New Revision: 49172 Modified: pypy/dist/pypy/translator/c/src/thread.h Log: This file has a Python.h dependency, lets make this explicit. Modified: pypy/dist/pypy/translator/c/src/thread.h ============================================================================== --- pypy/dist/pypy/translator/c/src/thread.h (original) +++ pypy/dist/pypy/translator/c/src/thread.h Mon Nov 26 21:46:37 2007 @@ -3,6 +3,7 @@ #ifndef __PYPY_THREAD_H #define __PYPY_THREAD_H +#include "Python.h" #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro From xoraxax at codespeak.net Mon Nov 26 21:47:03 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 21:47:03 +0100 (CET) Subject: [pypy-svn] r49173 - pypy/dist/pypy/translator/c/src Message-ID: <20071126204703.6014581C1@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 21:47:03 2007 New Revision: 49173 Modified: pypy/dist/pypy/translator/c/src/stack.h Log: Include stdio in stack.h for stderr. Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Mon Nov 26 21:47:03 2007 @@ -12,6 +12,7 @@ int LL_stack_too_big(void); #ifndef PYPY_NOT_MAIN_FILE +#include #include "thread.h" #ifndef PYPY_NOINLINE From xoraxax at codespeak.net Mon Nov 26 21:51:38 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 21:51:38 +0100 (CET) Subject: [pypy-svn] r49174 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071126205138.24FD281C4@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 21:51:37 2007 New Revision: 49174 Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py Log: Fix test: minor typo, remove skip. Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_standalone.py Mon Nov 26 21:51:37 2007 @@ -79,11 +79,10 @@ #assert 0 < res <= 84 def test_strtod(): - py.test.skip("Explodes") def entry_point(args): print float(args[1]) return 0 exe_name = 'test_strtod' compile_standalone(entry_point, exe_name=exe_name) - data = cmdexed(exe_name, '3.13e1') + data = cmdexec(exe_name, '3.13e1') From xoraxax at codespeak.net Mon Nov 26 21:52:58 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 21:52:58 +0100 (CET) Subject: [pypy-svn] r49175 - pypy/dist/pypy/translator/llvm Message-ID: <20071126205258.45C5481D1@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 21:52:57 2007 New Revision: 49175 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py Log: Teach eci to the LLVM backend. This has a different approach 49160/49171 because its doing the separate compilation and linking differently. Those changesets are reverted implicitly therefore. Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Mon Nov 26 21:52:57 2007 @@ -4,8 +4,10 @@ import py from pypy.translator.tool import stdoutcapture +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.llvm.log import log from pypy.translator.llvm.modwrapper import CtypesModule +from pypy.translator.llvm.externs2ll import get_incdirs def llvm_is_on_path(): if py.path.local.sysfind("llvm-as") is None or \ @@ -87,6 +89,19 @@ self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (base, base)) self.cmds.append("as %s.s -o %s.o" % (base, base)) + include_opts = get_incdirs(self.genllvm.eci) + # compile separate files + libraries = set() + for filename in self.genllvm.eci.separate_module_files: + assert filename.endswith(".c") + objname = filename[:-2] + ".o" + libraries.add(objname) + self.cmds.append("gcc %s -c %s -O3 -o %s" % (filename, include_opts, objname)) + + attrs = self.genllvm.eci._copy_attributes() + attrs['libraries'] = tuple(libraries) + attrs['libraries'] + self.genllvm.eci = ExternalCompilationInfo(**attrs) + # XXX support profile? # if (self.genllvm.config.translation.profopt is not None and # not self.genllvm.config.translation.noprofopt): @@ -112,50 +127,59 @@ self.dirpath.chdir() return self.genllvm.entry_name - - def make_module(self): + + def setup_linker_command(self, exename): base = self.setup() self.cmds_bytecode(base) self.cmds_objects(base) - # link (ok this is a mess!) + eci = self.genllvm.eci library_files = self.genllvm.db.gcpolicy.gc_libraries() - gc_libs = ' '.join(['-l' + lib for lib in library_files]) + library_files = list(library_files) + list(eci.libraries) + library_dirs = list(eci.library_dirs) + compiler_opts = [] if sys.platform == 'darwin': - libdir = '/sw/lib' - gc_libs_path = '-L%s -ldl' % libdir - self.cmds.append("gcc -O3 %s.o %s %s -lm -bundle -o %s.so" % (base, gc_libs_path, gc_libs, base)) + library_dirs.append('/sw/lib') + library_files.append("m") + library_files.append("dl") + if not exename: + compiler_opts.append("-bundle") else: + if not exename: + compiler_opts.append("-shared") + else: + compiler_opts.append("-static") + compiler_opts.append("-pipe") + + lib_opts = [] + for lib in library_files: + if lib[0] != "/": + lib = "-l" + lib + lib_opts.append(lib) + lib_dir_opts = ["-L" + libdir for libdir in library_dirs] + compiler_opts.extend(lib_opts) + compiler_opts.extend(lib_dir_opts) + + out = base + ".so" + if exename: + out = exename + self.cmds.append("gcc -O3 %s.o %s -o %s" % (base, " ".join(compiler_opts), out)) + return base - gc_libs_path = '-shared' - self.cmds.append("gcc -O3 %s.o %s %s -pipe -o %s.so" % (base, gc_libs_path, gc_libs, base)) + def make_module(self): + base = self.setup_linker_command(False) try: self.execute_cmds() modname = CtypesModule(self.genllvm, "%s.so" % base).create() - finally: self.lastdir.chdir() return modname, str(self.dirpath) def make_standalone(self, exename): - base = self.setup() - self.cmds_bytecode(base) - self.cmds_objects(base) - - object_files = ["-L/sw/lib"] - library_files = self.genllvm.db.gcpolicy.gc_libraries() - gc_libs = ' '.join(['-l' + lib for lib in library_files]) - - if sys.platform == 'darwin': - libdir = '/sw/' + "/lib" - gc_libs_path = '-L%s -ldl' % libdir - else: - gc_libs_path = '-static' - - self.cmds.append("gcc -O3 %s.o %s %s -lm -pipe -o %s" % (base, gc_libs_path, gc_libs, exename)) + base = self.setup_linker_command(exename) try: self.execute_cmds() Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Mon Nov 26 21:52:57 2007 @@ -3,11 +3,12 @@ import types import urllib +from StringIO import StringIO + from pypy.objspace.flow.model import FunctionGraph from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype from pypy.tool.udir import udir -from StringIO import StringIO def get_c_cpath(): from pypy.translator.c import genc @@ -19,10 +20,10 @@ def get_module_file(name): return os.path.join(get_llvm_cpath(), name) -def get_incdirs(c_include_dirs): +def get_incdirs(eci): import distutils.sysconfig - includes = tuple(c_include_dirs) + ("/sw/include", + includes = eci.include_dirs + ("/sw/include", distutils.sysconfig.EXEC_PREFIX + "/include", distutils.sysconfig.EXEC_PREFIX + "/include/gc", distutils.sysconfig.get_python_inc(), @@ -52,28 +53,22 @@ f.write(ccode) f.close() - includes = get_incdirs(eci.include_dirs) - eci = eci.convert_sources_to_files() - c_files = [filename] + list(eci.separate_module_files) - read_lines = [] - for name in c_files: - # XXX this is evil... - plain = name[:-2] - cmd = "llvm-gcc -emit-llvm -O0 -S %s %s.c -o %s.ll 2>&1" % ( - includes, plain, plain) + plain = filename[:-2] + includes = get_incdirs(eci) + cmd = "llvm-gcc -emit-llvm -O0 -S %s %s.c -o %s.ll 2>&1" % ( + includes, plain, plain) - if os.system(cmd) != 0: - raise Exception("Failed to run '%s'" % cmd) + if os.system(cmd) != 0: + raise Exception("Failed to run '%s'" % cmd) - llcode = open(plain + '.ll').read() - read_lines += llcode.split("\n") + llcode = open(plain + '.ll').read() # strip lines lines = [] calltag, declaretag, definetag = 'call ', 'declare ', 'define ' - for line in read_lines: + for line in llcode.split('\n'): # get rid of any of the structs that llvm-gcc introduces to struct types line = line.replace("%struct.", "%") @@ -123,21 +118,16 @@ ccode.append('#define __ENTRY_POINT__ %s' % entrynode.get_ref()[1:]) ccode.append('#define ENTRY_POINT_DEFINED 1') + sio = StringIO() + eci.write_c_header(sio) + ccode.extend(sio.getvalue().splitlines()) + # include python.h early ccode.append('#include ') # ask gcpolicy for any code needed ccode.append('%s' % db.gcpolicy.genextern_code()) - ccode.append("#define PYPY_NOT_MAIN_FILE") - # ask rffi for includes/source - s = StringIO() - eci.write_c_header(s) - ccode.append(s.getvalue()) - ccode.append("#undef PYPY_NOT_MAIN_FILE") - - ccode.append('') - # append our source file ccode.append(open(get_module_file('genexterns.c')).read()) return "\n".join(ccode) Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Mon Nov 26 21:52:57 2007 @@ -7,7 +7,7 @@ self.db = db self.value = value self.name = "@" + name - self.compilation_info = getattr(value, 'compilation_info', None) + self.eci = self.value.compilation_info def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Nov 26 21:52:57 2007 @@ -52,7 +52,8 @@ def __init__(self, translator, standalone): # reset counters - Node.nodename_count = {} + Node.nodename_count = {} + self.eci = ExternalCompilationInfo() self.standalone = standalone self.translator = translator @@ -187,12 +188,14 @@ def generate_ll_externs(self, codewriter): all = [] for node in self.db.getnodes(): - eci = getattr(node, 'compilation_info', None) - if eci is not None: + eci = getattr(node, 'eci', None) + if eci: all.append(eci) - eci = ExternalCompilationInfo().merge(*all) - ccode = generate_c(self.db, self.entrynode, eci, self.standalone) - self.llcode = generate_ll(ccode, codewriter.cconv, eci, self.db.extern_to_funcnodes) + self.eci = self.eci.merge(*all) + + ccode = generate_c(self.db, self.entrynode, self.eci, self.standalone) + self.llcode = generate_ll(ccode, codewriter.cconv, self.eci, self.db.extern_to_funcnodes) + self.eci = self.eci.convert_sources_to_files(being_main=True) def create_codewriter(self): # prevent running the same function twice in a test From xoraxax at codespeak.net Mon Nov 26 22:00:53 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 22:00:53 +0100 (CET) Subject: [pypy-svn] r49176 - in pypy/dist/pypy: rpython/lltypesystem/test translator/llvm Message-ID: <20071126210053.DAE8781C4@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 22:00:53 2007 New Revision: 49176 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py pypy/dist/pypy/translator/llvm/node.py Log: Fix name mangling in LLVM, unskip a test for that and change the skip message for another test. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Mon Nov 26 22:00:53 2007 @@ -500,7 +500,5 @@ return compile_llvm(func, args, **kwds) def test_hashdefine(self): - py.test.skip("FIXME") + py.test.skip("Macros cannot be called as llexternals by design, rffi does not have any special support for them") - def test_opaque_type(self): - py.test.skip("FIXME") Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Mon Nov 26 22:00:53 2007 @@ -1,11 +1,15 @@ from pypy.rpython.lltypesystem import lltype +NAME_BLACKLIST = "get".split() class Node(object): __slots__ = "name".split() prefix = '%' nodename_count = {} + for _name in NAME_BLACKLIST: + nodename_count[_name] = 1 + del _name def mangle(self, name): if name not in self.nodename_count: From xoraxax at codespeak.net Mon Nov 26 22:19:25 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 22:19:25 +0100 (CET) Subject: [pypy-svn] r49177 - pypy/dist/pypy/translator/llvm Message-ID: <20071126211925.B060F81BB@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 22:19:23 2007 New Revision: 49177 Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/genllvm.py Log: Rename eci to compilation_info on llvm nodes. Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Mon Nov 26 22:19:23 2007 @@ -7,7 +7,7 @@ self.db = db self.value = value self.name = "@" + name - self.eci = self.value.compilation_info + self.compilation_info = self.value.compilation_info def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Mon Nov 26 22:19:23 2007 @@ -188,7 +188,7 @@ def generate_ll_externs(self, codewriter): all = [] for node in self.db.getnodes(): - eci = getattr(node, 'eci', None) + eci = getattr(node, 'compilation_info', None) if eci: all.append(eci) self.eci = self.eci.merge(*all) From xoraxax at codespeak.net Mon Nov 26 22:28:47 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 26 Nov 2007 22:28:47 +0100 (CET) Subject: [pypy-svn] r49178 - pypy/dist/pypy/translator/llvm Message-ID: <20071126212847.C0FD780BB@code0.codespeak.net> Author: xoraxax Date: Mon Nov 26 22:28:47 2007 New Revision: 49178 Modified: pypy/dist/pypy/translator/llvm/node.py Log: The name mangling didn't make much sense, i must have overlooked the test failure. Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Mon Nov 26 22:28:47 2007 @@ -1,16 +1,11 @@ from pypy.rpython.lltypesystem import lltype -NAME_BLACKLIST = "get".split() class Node(object): __slots__ = "name".split() prefix = '%' nodename_count = {} - for _name in NAME_BLACKLIST: - nodename_count[_name] = 1 - del _name - def mangle(self, name): if name not in self.nodename_count: result = name From xoraxax at codespeak.net Tue Nov 27 00:27:28 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 27 Nov 2007 00:27:28 +0100 (CET) Subject: [pypy-svn] r49179 - pypy/dist/pypy/translator/llvm Message-ID: <20071126232728.D702A81C1@code0.codespeak.net> Author: xoraxax Date: Tue Nov 27 00:27:27 2007 New Revision: 49179 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/opwriter.py Log: Implement correct bit extension for small return values (as defined in the C ABI). Wrote a test that fails on genc (maybe only because of missing wrapper support) and is therefore skipped but runs on LLVM now). Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Tue Nov 27 00:27:27 2007 @@ -167,7 +167,7 @@ self._indent("unwind") def call(self, targetvar, returntype, functionref, argtypes, argrefs, - tail=None, cconv=None): + tail=None, cconv=None, ret_type_attrs=""): if tail is None: tail = self.tail if cconv is None: @@ -181,12 +181,13 @@ else: return_str = '%s = ' % targetvar - self._indent("%s%scall %s %s %s(%s)" % (return_str, + self._indent("%s%scall %s %s %s(%s) %s" % (return_str, tail, cconv, returntype, functionref, - args)) + args, + ret_type_attrs)) def alloca(self, targetvar, vartype): self._indent("%s = alloca %s" % (targetvar, vartype)) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Nov 27 00:27:27 2007 @@ -13,7 +13,7 @@ from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ VoidArrayNode, ArrayNoLengthNode, DebugStrNode -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic @@ -396,8 +396,6 @@ except ImportError: pass else: - from pypy.rpython.lltypesystem import rffi - def update(from_, type): if from_ not in self.types: self.types[from_] = type @@ -409,7 +407,23 @@ rffi.LONGLONG, rffi.ULONGLONG]: bits = rffi.size_and_sign(tp)[0] * 8 update(tp, 'i%s' % bits) - + + def get_attrs_for_type(self, type): + # because we want to bind to external functions that depend + # on sign/zero extensions, we need to use these attributes in function sigs + # note that this is not needed for internal functions because they use + # casts if necessary + type_attrs = "" + if not isinstance(type, lltype.Number): + return type_attrs + size, sign = rffi.size_and_sign(type) + if size < 4: + if not sign: + type_attrs += "signext" + else: + type_attrs += "zeroext" + return type_attrs + def __getitem__(self, key): return self.types[key] Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Tue Nov 27 00:27:27 2007 @@ -1,7 +1,8 @@ from pypy.translator.llvm.node import FuncNode -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi class ExternalFuncNode(FuncNode): + is_external = True def __init__(self, db, value): name = value._name self.db = db @@ -15,12 +16,17 @@ def getdecl_parts(self): T = self.value._TYPE rettype = self.db.repr_type(T.RESULT) + rettype_attrs = " " + + if self.is_external: + rettype_attrs = self.db.primitives.get_attrs_for_type(T.RESULT) + argtypes = [self.db.repr_type(a) for a in T.ARGS if a is not lltype.Void] - return rettype, argtypes + return rettype, argtypes, rettype_attrs def getdecl(self): - rettype, argtypes = self.getdecl_parts() - return "%s %s(%s)" % (rettype, self.ref, ", ".join(argtypes)) + rettype, argtypes, rettype_attrs = self.getdecl_parts() + return "%s %s(%s) %s" % (rettype, self.ref, ", ".join(argtypes), rettype_attrs) def writedecl(self, codewriter): codewriter.declare(self.getdecl(), cconv="ccc") Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Nov 27 00:27:27 2007 @@ -2,7 +2,7 @@ import py import ctypes -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.rstr import STR class CtypesModule: @@ -148,6 +148,8 @@ lltype.UnsignedLongLong: "ctypes.c_ulonglong", lltype.Void: None, lltype.UniChar: "ctypes.c_uint", + rffi.SIGNEDCHAR: "ctypes.c_char", + rffi.UCHAR: "ctypes.c_ubyte" } def __init__(self, genllvm, dllname): @@ -212,6 +214,8 @@ if T is lltype.Bool: action = 'bool' + elif T is rffi.UCHAR: + action = "chr" elif T is lltype.UniChar: action = 'unichr' Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Tue Nov 27 00:27:27 2007 @@ -297,6 +297,7 @@ def direct_call(self, opr): cconv = None + rettype_attrs = "" # XXX aargh - more illegal fishing # XXX sort this out later... @@ -308,13 +309,14 @@ value = opr.op.args[0].value._obj if getattr(value, 'external', None) == 'C': cconv = 'ccc' + rettype_attrs = self.db.primitives.get_attrs_for_type(T.RESULT) + #self.codewriter.debug_print(str(opr.op) + "\n") #self.codewriter.debug_print(str(cconv) + "\n") - # if we are external node - should use standard calling conventions self.codewriter.call(opr.retref, opr.rettype, opr.argrefs[0], - opr.argtypes[1:], opr.argrefs[1:], cconv=cconv) + opr.argtypes[1:], opr.argrefs[1:], cconv=cconv, ret_type_attrs=rettype_attrs) # the following works since the extra arguments that indirect_call has # is of type Void, which is removed by direct_call From xoraxax at codespeak.net Tue Nov 27 00:32:20 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 27 Nov 2007 00:32:20 +0100 (CET) Subject: [pypy-svn] r49180 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20071126233220.71D2281C1@code0.codespeak.net> Author: xoraxax Date: Tue Nov 27 00:32:19 2007 New Revision: 49180 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: Here is the test that I talked about in the last commit. Skipped the opaque test for LLVM again because another issue appeared after fixing the sign issue. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Tue Nov 27 00:32:19 2007 @@ -237,7 +237,7 @@ ) STUFFP = COpaquePtr('struct stuff', compilation_info=eci) - ll_get = llexternal('get', [STUFFP], lltype.Char, compilation_info=eci) + ll_get = llexternal('get', [STUFFP], CHAR, compilation_info=eci) def f(): ll_stuff = lltype.malloc(STUFFP.TO, flavor='raw') @@ -247,7 +247,37 @@ f1 = self.compile(f, []) assert f1() == 'a' + + def return_char(self, signed): + ctype_pref = ["un", ""][signed] + rffi_type = [UCHAR, SIGNEDCHAR][signed] + h_source = py.code.Source(""" + %ssigned char returnchar(void) + { + return 42; + } + """ % (ctype_pref, )) + h_file = udir.join("opaque2%s.h" % (ctype_pref, )) + h_file.write(h_source) + + from pypy.rpython.tool import rffi_platform + eci = ExternalCompilationInfo( + includes=[h_file.basename], + include_dirs=[str(udir)] + ) + ll_returnchar = llexternal('returnchar', [], rffi_type, compilation_info=eci) + def f(): + result = ll_returnchar() + return result + + f1 = self.compile(f, []) + assert f1() == chr(42) + + def test_generate_return_char_tests(self): + yield self.return_char, False + yield self.return_char, True + def test_prebuilt_constant(self): py.test.skip("Think how to do it sane") h_source = py.code.Source(""" @@ -491,6 +521,9 @@ def compile(self, func, args, **kwds): return compile_c(func, args, **kwds) + def test_generate_return_char_tests(self): + py.test.skip("GenC does not handle char return values correctly") + class TestLLVMRffi(BaseTestRffi): def compile(self, func, args, **kwds): # pfff.... @@ -502,3 +535,5 @@ def test_hashdefine(self): py.test.skip("Macros cannot be called as llexternals by design, rffi does not have any special support for them") + def test_opaque_type(self): + py.test.skip("GenLLVM handles opaque type defs incorrectly") From fijal at codespeak.net Tue Nov 27 09:25:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Nov 2007 09:25:58 +0100 (CET) Subject: [pypy-svn] r49181 - pypy/dist/pypy/module/thread Message-ID: <20071127082558.719488164@code0.codespeak.net> Author: fijal Date: Tue Nov 27 09:25:57 2007 New Revision: 49181 Modified: pypy/dist/pypy/module/thread/ll_thread.py Log: We should rather kill Python.h dependency. But as long as we didn't, lets include python include dir as well. Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Tue Nov 27 09:25:57 2007 @@ -12,6 +12,8 @@ from pypy.rpython.lltypesystem.lltype import typeOf from pypy.rlib.objectmodel import debug_assert from pypy.tool import autopath +from distutils import sysconfig +python_inc = sysconfig.get_python_inc() error = thread.error @@ -22,7 +24,8 @@ #include #include '''], - include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))] + include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c')), + python_inc] ) def llexternal(name, args, result, **kwds): From hpk at codespeak.net Tue Nov 27 15:46:06 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 27 Nov 2007 15:46:06 +0100 (CET) Subject: [pypy-svn] r49185 - pypy/dist/pypy/doc Message-ID: <20071127144606.066888198@code0.codespeak.net> Author: hpk Date: Tue Nov 27 15:46:03 2007 New Revision: 49185 Modified: pypy/dist/pypy/doc/news.txt Log: pretty sure that those links are outdated, not cared for. Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Tue Nov 27 15:46:03 2007 @@ -7,13 +7,6 @@ .. _Python: http://www.python.org/doc/current/ref/ref.html .. _`more...`: architecture.html#mission-statement - - The planned PyPy events announced below are also published in - `iCalendar format`_. See eventhistory_ for a list of past events. - -.. _`iCalendar format`: webcal://pypycal.sabi.net///calendars/PyPy.ics -.. _eventhistory: eventhistory.html - PyPy blog started ================= From xoraxax at codespeak.net Tue Nov 27 21:45:55 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 27 Nov 2007 21:45:55 +0100 (CET) Subject: [pypy-svn] r49187 - in pypy/dist/pypy: doc rpython/module Message-ID: <20071127204555.0255681CA@code0.codespeak.net> Author: xoraxax Date: Tue Nov 27 21:45:54 2007 New Revision: 49187 Modified: pypy/dist/pypy/doc/rffi.txt pypy/dist/pypy/rpython/module/ll_os_environ.py Log: (tav): Fixed rffi docs about llexternals and fixed os_environ on darwin. Modified: pypy/dist/pypy/doc/rffi.txt ============================================================================== --- pypy/dist/pypy/doc/rffi.txt (original) +++ pypy/dist/pypy/doc/rffi.txt Tue Nov 27 21:45:54 2007 @@ -12,23 +12,38 @@ Declaring low-level external function ------------------------------------- -Declaring external C function in RPython is easy, but one need to -remember that low level functions eats low level types (like +Declaring external C function in RPython is easy, but one needs to +remember that low level functions eat low level types (like lltype.Signed or lltype.Array) and memory management must be done by hand. To declare a function, we write:: from pypy.rpython.lltypesystem import rffi - external_function = rffi.llexternal(name, args, result, includes=[], sources=[]) + external_function = rffi.llexternal(name, args, result) where: * name - a C-level name of a function (how it would be rendered) * args - low level types of args * result - low level type of a result -* includes - additional C-level includes -* sources - additional C-level sources, needed to be compiled into - executable + +You can pass in additional information about C-level includes, +libraries and sources by passing in the optional ``compilation_info`` +parameter:: + + from pypy.rpython.lltypesystem import rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + info = ExternalCompilationInfo(includes=[], libraries=[]) + + external_function = rffi.llexternal( + name, args, result, compilation_info=info + ) + +See cbuild_ for more info on ExternalCompilationInfo. + +.. _cbuild: http://codespeak.net/svn/pypy/dist/pypy/translator/tool/cbuild.py + Types ------ Modified: pypy/dist/pypy/rpython/module/ll_os_environ.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_environ.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_environ.py Tue Nov 27 21:45:54 2007 @@ -136,8 +136,10 @@ if sys.platform.startswith('darwin'): CCHARPPP = rffi.CArrayPtr(rffi.CCHARPP) - _os_NSGetEnviron = rffi.llexternal('_NSGetEnviron', [], CCHARPPP, - includes=['crt_externs.h']) + _os_NSGetEnviron = rffi.llexternal( + '_NSGetEnviron', [], CCHARPPP, + compilation_info=ExternalCompilationInfo(includes=['crt_externs.h']) + ) def os_get_environ(): return _os_NSGetEnviron()[0] else: From fijal at codespeak.net Wed Nov 28 01:42:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Nov 2007 01:42:12 +0100 (CET) Subject: [pypy-svn] r49189 - pypy/dist/pypy/translator/goal Message-ID: <20071128004212.7579D80B3@code0.codespeak.net> Author: fijal Date: Wed Nov 28 01:42:10 2007 New Revision: 49189 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: Remove llvm backend for now as it stops bench cronjob at all (and llvm custom compilation logic is broken anyway) Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Wed Nov 28 01:42:10 2007 @@ -180,7 +180,6 @@ def main(backends=[]): if backends == []: #_ prefix means target specific option, # prefix to outcomment backends = [backend.strip() for backend in """ - llvm--_faassen c c--stackless--_faassen c--_faassen From rxe at codespeak.net Wed Nov 28 13:57:15 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 28 Nov 2007 13:57:15 +0100 (CET) Subject: [pypy-svn] r49190 - in pypy/dist/pypy/translator/llvm: . module Message-ID: <20071128125715.4A9A481B4@code0.codespeak.net> Author: rxe Date: Wed Nov 28 13:57:14 2007 New Revision: 49190 Modified: pypy/dist/pypy/translator/llvm/buildllvm.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/extfuncnode.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/support.py pypy/dist/pypy/translator/llvm/opwriter.py Log: kill and fix calling convention mess, as llvm inserts with opt anyways. (thanks AntonK and xorAxAx) Modified: pypy/dist/pypy/translator/llvm/buildllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/buildllvm.py (original) +++ pypy/dist/pypy/translator/llvm/buildllvm.py Wed Nov 28 13:57:14 2007 @@ -80,17 +80,24 @@ opts = self.optimizations() self.cmds.append("llvm-as < %s.ll | opt %s -f -o %s.bc" % (base, opts, base)) - def cmds_objects(self, base): + def cmds_objects(self, base, standalone): use_gcc = self.genllvm.config.translation.llvm_via_c if use_gcc: self.cmds.append("llc %s.bc -march=c -f -o %s.c" % (base, base)) self.cmds.append("gcc %s.c -c -O3 -fomit-frame-pointer" % base) else: - self.cmds.append("llc -relocation-model=pic %s.bc -f -o %s.s" % (base, base)) + model = '' + if not standalone: + model = ' -relocation-model=pic' + + self.cmds.append("llc %s %s.bc -f -o %s.s" % (model, base, base)) self.cmds.append("as %s.s -o %s.o" % (base, base)) include_opts = get_incdirs(self.genllvm.eci) + # compile separate files + # XXX rxe: why do we want to run a c compiler, when we run llvm + # compiler - these seems a step backwards IMHO ????? libraries = set() for filename in self.genllvm.eci.separate_module_files: assert filename.endswith(".c") @@ -128,11 +135,7 @@ return self.genllvm.entry_name - def setup_linker_command(self, exename): - base = self.setup() - self.cmds_bytecode(base) - self.cmds_objects(base) - + def setup_linker_command(self, base, exename=None): eci = self.genllvm.eci library_files = self.genllvm.db.gcpolicy.gc_libraries() library_files = list(library_files) + list(eci.libraries) @@ -165,11 +168,12 @@ if exename: out = exename self.cmds.append("gcc -O3 %s.o %s -o %s" % (base, " ".join(compiler_opts), out)) - return base - def make_module(self): - base = self.setup_linker_command(False) + base = self.setup() + self.cmds_bytecode(base) + self.cmds_objects(base, False) + self.setup_linker_command(base) try: self.execute_cmds() modname = CtypesModule(self.genllvm, "%s.so" % base).create() @@ -179,7 +183,10 @@ return modname, str(self.dirpath) def make_standalone(self, exename): - base = self.setup_linker_command(exename) + base = self.setup() + self.cmds_bytecode(base) + self.cmds_objects(base, True) + self.setup_linker_command(base, exename) try: self.execute_cmds() Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 28 13:57:14 2007 @@ -4,40 +4,18 @@ log = log.codewriter class CodeWriter(object): - tail = '' #/tail - cconv = 'ccc' #ccc/fastcc linkage = 'internal ' #/internal (disabled for now because of the JIT) - def __init__(self, file, db, tail=None, cconv=None, linkage=None): + def __init__(self, file, db, linkage=None): self.db = db self.file = file self.word_repr = db.get_machine_word() - if tail is not None: - self.tail = tail - if cconv is not None: - self.cconv = cconv if linkage is not None: self.linkage = linkage def close(self): self.file.close() - def _resolvetail(self, tail, cconv): - # from: http://llvm.org/docs/LangRef.html - # The optional "tail" marker indicates whether the callee function - # accesses any allocas or varargs in the caller. If the "tail" marker - # is present, the function call is eligible for tail call - # optimization. Note that calls may be marked "tail" even if they do - # not occur before a ret instruction. - - if cconv is not 'fastcc': - tail_ = '' - else: - tail_ = tail - if tail_: - tail_ += ' ' - return tail_ - # keep these two internal for now - incase we try a different API def _append(self, line): self.file.write(line + '\n') @@ -50,7 +28,6 @@ if patch: l = l.replace('WORD', self.word_repr) l = l.replace('POSTFIX', postfix()) - l = l.replace('CC', self.cconv) self._append(l) def comment(self, line, indent=True): @@ -82,10 +59,8 @@ def typedef(self, name, type_): self._append("%s = type %s" % (name, type_)) - def declare(self, decl, cconv=None): - if cconv is None: - cconv = self.cconv - self._append("declare %s %s" %(cconv, decl,)) + def declare(self, decl): + self._append("declare %s" % decl) def br_uncond(self, blockname): self._indent("br label %%%s" %(blockname,)) @@ -101,13 +76,11 @@ self._indent("switch %s %s, label %%%s [%s ]" % (intty, cond, defaultdest, labels)) - def openfunc(self, decl, cconv=None, linkage=None): - if cconv is None: - cconv = self.cconv + def openfunc(self, decl, linkage=None): if linkage is None: linkage = self.linkage self.newline() - self._append("define %s%s %s {" % (linkage, cconv, decl,)) + self._append("define %s %s {" % (linkage, decl,)) def closefunc(self): self._append("}") @@ -166,14 +139,7 @@ def unwind(self): self._indent("unwind") - def call(self, targetvar, returntype, functionref, argtypes, argrefs, - tail=None, cconv=None, ret_type_attrs=""): - if tail is None: - tail = self.tail - if cconv is None: - cconv = self.cconv - - tail = self._resolvetail(tail, cconv) + def call(self, targetvar, returntype, functionref, argtypes, argrefs, ret_type_attrs=""): args = ", ".join(["%s %s" % item for item in zip(argtypes, argrefs)]) if returntype == 'void': @@ -181,13 +147,11 @@ else: return_str = '%s = ' % targetvar - self._indent("%s%scall %s %s %s(%s) %s" % (return_str, - tail, - cconv, - returntype, - functionref, - args, - ret_type_attrs)) + self._indent("%s call %s %s(%s) %s" % (return_str, + returntype, + functionref, + args, + ret_type_attrs)) def alloca(self, targetvar, vartype): self._indent("%s = alloca %s" % (targetvar, vartype)) @@ -212,4 +176,4 @@ node = self.db.create_debug_string(s) self.call(var, "i32", "@write", ['i32', 'i8*', 'i32'], - ['2', node.get_childref(0), '%d' % node.get_length()], cconv='ccc') + ['2', node.get_childref(0), '%d' % node.get_length()]) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Nov 28 13:57:14 2007 @@ -39,9 +39,6 @@ self.debugstringnodes = [] - # call back into rpython code from c code - self.extern_to_funcnodes = [] - #_______debuggging______________________________________ @@ -320,9 +317,6 @@ self._tmpcount += 1 return "%tmp_" + str(count) - def add_extern_to_funcnode(self, name): - self.extern_to_funcnodes.append(name) - # __________________________________________________________ # Other helpers Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Nov 28 13:57:14 2007 @@ -37,17 +37,7 @@ # call boehm finalizers need to be fastcc -def generate_ll(ccode, default_cconv, eci, call_funcnames=[]): - call_funcnames = ['@LLVM_RPython_StartupCode'] + call_funcnames - define_funcnames = ['@pypy_malloc', - '@pypy_malloc_atomic', - '@pypy_gc__collect', - '@pypy_register_finalizer', - '@raw_malloc', - '@raw_free', - ] - declare_funcnames = call_funcnames - +def generate_ll(ccode, eci): filename = str(udir.join("ccode.c")) f = open(filename, "w") f.write(ccode) @@ -65,47 +55,7 @@ # strip lines lines = [] - - calltag, declaretag, definetag = 'call ', 'declare ', 'define ' - for line in llcode.split('\n'): - - # get rid of any of the structs that llvm-gcc introduces to struct types - line = line.replace("%struct.", "%") - - # strip comments - comment = line.find(';') - if comment >= 0: - line = line[:comment] - line = line.rstrip() - - # patch calls (upgrade to default_cconv) - i = line.find(calltag) - if i >= 0: - for funcname in call_funcnames: - if line.find(funcname) >= 0: - line = "%scall %s %s" % (line[:i], default_cconv, line[i+len(calltag):]) - break - - if line[:len(declaretag)] == declaretag: - xline = line[len(declaretag):] - for funcname in declare_funcnames: - if xline.find(funcname) != -1: - line = "declare %s %s %s" % (internal, default_cconv, xline) - break - - if line[:len(definetag)] == definetag: - xline = line[len(definetag):] - internal = '' - if xline.startswith('internal '): - internal = 'internal ' - xline = xline.replace('internal ', '') - - for funcname in define_funcnames: - if xline.find(funcname) != -1: - line = "define %s %s %s" % (internal, default_cconv, xline) - break - lines.append(line) lines.append("declare ccc void @abort()") Modified: pypy/dist/pypy/translator/llvm/extfuncnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/extfuncnode.py (original) +++ pypy/dist/pypy/translator/llvm/extfuncnode.py Wed Nov 28 13:57:14 2007 @@ -29,5 +29,5 @@ return "%s %s(%s) %s" % (rettype, self.ref, ", ".join(argtypes), rettype_attrs) def writedecl(self, codewriter): - codewriter.declare(self.getdecl(), cconv="ccc") + codewriter.declare(self.getdecl()) Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Wed Nov 28 13:57:14 2007 @@ -77,8 +77,7 @@ codewriter.malloc(targetvar, "i8", size) codewriter.call(None, 'void', '@llvm.memset' + postfix(), ['i8*', 'i8', word, word], - [targetvar, 0, size, boundary_size], - cconv='ccc') + [targetvar, 0, size, boundary_size]) class BoehmGcPolicy(GcPolicy): @@ -114,11 +113,9 @@ codewriter.call(targetvar, 'i8*', fnname, [word], [size]) if atomic: - # XXX uses own cconv codewriter.call(None, 'void', '@llvm.memset' + postfix(), ['i8*', 'i8', word, word], - [targetvar, 0, size, boundary_size], - cconv='ccc') + [targetvar, 0, size, boundary_size]) def op_set_max_heap_size(self, codewriter, opr): Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 28 13:57:14 2007 @@ -181,7 +181,6 @@ else: self.function_count[name] = 1 - self.db.add_extern_to_funcnode(name) self.entry_name = name[6:] return c.value._obj @@ -194,7 +193,7 @@ self.eci = self.eci.merge(*all) ccode = generate_c(self.db, self.entrynode, self.eci, self.standalone) - self.llcode = generate_ll(ccode, codewriter.cconv, self.eci, self.db.extern_to_funcnodes) + self.llcode = generate_ll(ccode, self.eci) self.eci = self.eci.convert_sources_to_files(being_main=True) def create_codewriter(self): @@ -204,7 +203,7 @@ if self.standalone: return CodeWriter(f, self.db), filename else: - return CodeWriter(f, self.db, cconv='ccc', linkage=''), filename + return CodeWriter(f, self.db, linkage=''), filename def write_startup_impl(self, codewriter): open_decl = "i8* @LLVM_RPython_StartupCode()" Modified: pypy/dist/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/dist/pypy/translator/llvm/module/support.py (original) +++ pypy/dist/pypy/translator/llvm/module/support.py Wed Nov 28 13:57:14 2007 @@ -1,5 +1,5 @@ extfunctions = """ -define internal CC double @pypyop_float_abs(double %x) { +define internal double @pypyop_float_abs(double %x) { block0: %cond1 = fcmp ugt double %x, 0.0 br i1 %cond1, label %return_block, label %block1 @@ -11,7 +11,7 @@ ret double %result } -define internal CC i32 @pypyop_int_abs(i32 %x) { +define internal i32 @pypyop_int_abs(i32 %x) { block0: %cond1 = icmp sge i32 %x, 0 br i1 %cond1, label %return_block, label %block1 @@ -23,7 +23,7 @@ ret i32 %result } -define internal CC i64 @pypyop_llong_abs(i64 %x) { +define internal i64 @pypyop_llong_abs(i64 %x) { block0: %cond1 = icmp sge i64 %x, 0 br i1 %cond1, label %return_block, label %block1 Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Wed Nov 28 13:57:14 2007 @@ -296,7 +296,6 @@ opr.argrefs[0], "null") def direct_call(self, opr): - cconv = None rettype_attrs = "" # XXX aargh - more illegal fishing @@ -308,15 +307,10 @@ assert isinstance(T, lltype.FuncType) value = opr.op.args[0].value._obj if getattr(value, 'external', None) == 'C': - cconv = 'ccc' rettype_attrs = self.db.primitives.get_attrs_for_type(T.RESULT) - - - #self.codewriter.debug_print(str(opr.op) + "\n") - #self.codewriter.debug_print(str(cconv) + "\n") self.codewriter.call(opr.retref, opr.rettype, opr.argrefs[0], - opr.argtypes[1:], opr.argrefs[1:], cconv=cconv, ret_type_attrs=rettype_attrs) + opr.argtypes[1:], opr.argrefs[1:], ret_type_attrs=rettype_attrs) # the following works since the extra arguments that indirect_call has # is of type Void, which is removed by direct_call From fijal at codespeak.net Wed Nov 28 18:04:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Nov 2007 18:04:54 +0100 (CET) Subject: [pypy-svn] r49193 - pypy/branch/cleanup-headers Message-ID: <20071128170454.06F3D81E4@code0.codespeak.net> Author: fijal Date: Wed Nov 28 18:04:53 2007 New Revision: 49193 Removed: pypy/branch/cleanup-headers/ Log: Remove this branch as llvm is working and I'll create a new one when I feel like working on this. From cfbolz at codespeak.net Thu Nov 29 01:59:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Nov 2007 01:59:01 +0100 (CET) Subject: [pypy-svn] r49196 - in pypy/dist/pypy/rpython: . test Message-ID: <20071129005901.C85A28160@code0.codespeak.net> Author: cfbolz Date: Thu Nov 29 01:59:00 2007 New Revision: 49196 Modified: pypy/dist/pypy/rpython/rstr.py pypy/dist/pypy/rpython/test/test_runicode.py Log: make sure that unicode(u'...') is not trying to make sure the argument is ascii. Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Thu Nov 29 01:59:00 2007 @@ -252,6 +252,9 @@ return hop.inputconst(hop.r_result, hop.s_result.const) repr = hop.args_r[0].repr v_str = hop.inputarg(repr, 0) + if repr == self.repr: # the argument is a unicode string already + hop.exception_cannot_occur() + return v_str hop.exception_is_here() return hop.gendirectcall(self.ll.ll_str2unicode, v_str) Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Thu Nov 29 01:59:00 2007 @@ -46,6 +46,20 @@ assert self.ll_to_string(self.interpret(f, [])) == 'xxx' + def test_unicode_of_unicode(self): + def f(x): + return len(unicode(unichr(x) * 3)) + assert self.interpret(f, [ord('a')]) == 3 + assert self.interpret(f, [128]) == 3 + assert self.interpret(f, [1000]) == 3 + + def test_unicode_of_unichar(self): + def f(x): + return len(unicode(unichr(x))) + assert self.interpret(f, [ord('a')]) == 1 + assert self.interpret(f, [128]) == 1 + assert self.interpret(f, [1000]) == 1 + def test_conversion_errors(self): py.test.skip("do we want this test to pass?") def f(x): From cfbolz at codespeak.net Thu Nov 29 02:05:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Nov 2007 02:05:40 +0100 (CET) Subject: [pypy-svn] r49197 - pypy/dist/pypy/rpython Message-ID: <20071129010540.94BE5817A@code0.codespeak.net> Author: cfbolz Date: Thu Nov 29 02:05:40 2007 New Revision: 49197 Modified: pypy/dist/pypy/rpython/rstr.py Log: get the condition even righter Modified: pypy/dist/pypy/rpython/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/rstr.py (original) +++ pypy/dist/pypy/rpython/rstr.py Thu Nov 29 02:05:40 2007 @@ -252,7 +252,7 @@ return hop.inputconst(hop.r_result, hop.s_result.const) repr = hop.args_r[0].repr v_str = hop.inputarg(repr, 0) - if repr == self.repr: # the argument is a unicode string already + if repr == hop.r_result: # the argument is a unicode string already hop.exception_cannot_occur() return v_str hop.exception_is_here() From niko at codespeak.net Thu Nov 29 06:44:45 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 29 Nov 2007 06:44:45 +0100 (CET) Subject: [pypy-svn] r49198 - in pypy/dist/pypy/translator/jvm: . src/pypy test Message-ID: <20071129054445.08AA18142@code0.codespeak.net> Author: niko Date: Thu Nov 29 06:44:43 2007 New Revision: 49198 Removed: pypy/dist/pypy/translator/jvm/src/pypy/RecordStringString.java Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/prebuiltnodes.py pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/typesystem.py Log: 1. implement os.dup 2. change from using custom structs (RecordStringString) to adding methods in Interlink to create records with particular field types. This avoids the need to implement equals(), toString(), hash() and all that other pain and makes fewer special cases. Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu Nov 29 06:44:43 2007 @@ -480,8 +480,14 @@ # These are some configured records that are generated by Java # code. - ootype.Record({"item0": ootype.String, "item1": ootype.String}): - jvmtype.jPyPyRecordStringString, + #ootype.Record({"item0": ootype.Signed, "item1": ootype.Signed}): + #jvmtype.jPyPyRecordSignedSigned, + #ootype.Record({"item0": ootype.Float, "item1": ootype.Signed}): + #jvmtype.jPyPyRecordFloatSigned, + #ootype.Record({"item0": ootype.Float, "item1": ootype.Float}): + #jvmtype.jPyPyRecordFloatFloat, + #ootype.Record({"item0": ootype.String, "item1": ootype.String}): + #jvmtype.jPyPyRecordStringString, } def lltype_to_cts(self, OOT): Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Thu Nov 29 06:44:43 2007 @@ -779,21 +779,31 @@ self.interlink = interlink self.name = name self.helper = helper + + # The functions in Interlink.java either return Object, + # because they are returning an instance of a class generated + # by us which the JVM doesn't know about, or they return a + # scalar. + if self.helper.return_type.descriptor.is_reference(): + self.return_type = jObject + else: + self.return_type = self.helper.return_type + self.method_obj = jvmgen.Method.v(interlink, self.name, self.helper.argument_types, - jVoid) + self.return_type) def method(self): return self.method_obj def render(self, gen): argtypes = [self.interlink] + list(self.helper.argument_types) - gen.begin_function(self.name, (), argtypes, jVoid) + gen.begin_function(self.name, (), argtypes, self.return_type) varindex = 1 for argty in self.helper.argument_types: gen.load_jvm_var(argty, varindex) varindex += argty.descriptor.type_width() gen.emit(self.helper) - gen.return_val(jVoid) + gen.return_val(self.return_type) gen.end_function() Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/jvm/prebuiltnodes.py Thu Nov 29 06:44:43 2007 @@ -2,21 +2,49 @@ # ___________________________________________________________________________ +HELPERS = {} +def with_types(type_list): + def decorator(func): + HELPERS[func] = type_list + return func + return decorator + + at with_types([]) def throwZeroDivisionError(): raise ZeroDivisionError + at with_types([]) def throwIndexError(): raise IndexError + at with_types([]) def throwOverflowError(): raise OverflowError + at with_types([]) def throwValueError(): raise ValueError + at with_types([]) def throwUnicodeDecodeError(): raise UnicodeDecodeError + at with_types([str, str]) +def recordStringString(a, b): + return (a, b) + + at with_types([float, float]) +def recordFloatFloat(a, b): + return (a, b) + + at with_types([float, int]) +def recordFloatSigned(a, b): + return (a, b) + + at with_types([int, int]) +def recordSignedSigned(a, b): + return (a, b) + # ___________________________________________________________________________ def create_interlink_node(db): @@ -24,14 +52,12 @@ a jvmgen.Method object that allows it to be called. """ translator = db.genoo.translator - HELPERS = [val for nm, val in globals().items() if nm.startswith('throw')] - - for func in HELPERS: - translator.annotator.build_types(func, []) + for func, type_list in HELPERS.items(): + translator.annotator.build_types(func, type_list) translator.rtyper.specialize_more_blocks() helpers = {} - for func in HELPERS: + for func in HELPERS.keys(): graph = graphof(translator, func) helpers[func.func_name] = db.pending_function(graph) Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Thu Nov 29 06:44:43 2007 @@ -16,4 +16,9 @@ public void throwValueError(); public void throwUnicodeDecodeError(); public void throwOSError(int errCode); + + public Object recordStringString(String a, String b); + public Object recordFloatFloat(String a, String b); + public Object recordFloatSigned(String a, String b); + public Object recordSignedSigned(String a, String b); } Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Thu Nov 29 06:44:43 2007 @@ -149,7 +149,7 @@ } -public class ll_os { +public class ll_os implements Constants { // NB: these values are those used by Windows and they differs // from the Unix ones; the os module is patched with these @@ -182,6 +182,11 @@ FileDescriptors.put(new Integer(2), new PrintStreamWrapper(System.err)); fdcount = 2; } + + public static final boolean STRACE = false; + public static void strace(String arg) { + System.err.println(arg); + } public static void throwOSError(int errno, String errText) { ErrorMessages.put(new Integer(errno), errText); @@ -245,28 +250,48 @@ fdcount++; FileDescriptors.put(new Integer(fdcount), wrapper); + + if (STRACE) strace("ll_os_open: "+name+"->"+fdcount); return fdcount; } public static void ll_os_close(int fd) { + if (STRACE) strace("ll_os_close: "+fd); FileWrapper wrapper = getfd(fd); wrapper.close(); FileDescriptors.remove(new Integer(fd)); } + public static int ll_os_dup(int fd) + { + FileWrapper wrapper = getfd(fd); + for (int i = 0; i < Integer.MAX_VALUE; i++) { + if (FileDescriptors.get(i) == null) { + FileDescriptors.put(i, wrapper); + if (STRACE) strace("ll_os_dup: "+fd+" -> "+i); + return i; + } + } + throwOSError(EMFILE, "No remaining file descriptors."); + return -1; + } + public static String ll_os_read(int fd, int count) { + if (STRACE) strace("ll_os_read: "+fd); return getfd(fd).read(count); } public static String ll_os_read(int fd, long count) { + if (STRACE) strace("ll_os_read: "+fd); return ll_os_read(fd, (int)count); } public static long ll_os_lseek(int fd, long offset, int whence) { + if (STRACE) strace("ll_os_lseek: "+fd); FileWrapper wrapper = getfd(fd); RandomAccessFile file = wrapper.getFile(); if (file == null) @@ -310,6 +335,7 @@ } public static int ll_os_write(int fd, String text) { + if (STRACE) strace("ll_os_write: "+fd+" "+text); FileWrapper f = FileDescriptors.get(new Integer(fd)); if (f == null) throwOSError(PyPy.EBADF, "Invalid fd: " + fd); @@ -356,7 +382,7 @@ { String name = (String) nameIterator.next(); String value = (String) variables.get(name); - result.add(new RecordStringString(name, value)); + result.add(PyPy.interlink.recordStringString(name, value)); } return result; Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Thu Nov 29 06:44:43 2007 @@ -20,8 +20,14 @@ def test_builtin_math_modf(self): py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") + def test_os_path_exists(self): + py.test.skip("fails in annotation stage, unrelated to JVM I think") + + def test_os_isdir(self): + py.test.skip("fails in annotation stage, unrelated to JVM I think") + def test_os_dup(self): - py.test.skip("not implemented") + py.test.skip("test N/A to jvm: replaced by test_os_dup_oo") def test_environ(self): py.test.skip('fixme! how to set environment variables in Java?') Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Thu Nov 29 06:44:43 2007 @@ -179,7 +179,10 @@ jPyPyStatResult = JvmClassType('pypy.StatResult') jPyPyWeakRef = JvmClassType('pypy.PyPyWeakRef') jll_os = JvmClassType('pypy.ll_os') +jPyPyRecordSignedSigned = JvmClassType('pypy.RecordSignedSigned') jPyPyRecordStringString = JvmClassType('pypy.RecordStringString') +jPyPyRecordFloatSigned = JvmClassType('pypy.RecordFloatSigned') +jPyPyRecordFloatFloat = JvmClassType('pypy.RecordFloatFloat') jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) From cfbolz at codespeak.net Thu Nov 29 11:19:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Nov 2007 11:19:46 +0100 (CET) Subject: [pypy-svn] r49199 - pypy/dist/pypy/objspace/std/test Message-ID: <20071129101946.A928F812F@code0.codespeak.net> Author: cfbolz Date: Thu Nov 29 11:19:45 2007 New Revision: 49199 Modified: pypy/dist/pypy/objspace/std/test/test_stringformat.py Log: test for proper str % unicode behaviour from JP Calderone - thanks Modified: pypy/dist/pypy/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_stringformat.py (original) +++ pypy/dist/pypy/objspace/std/test/test_stringformat.py Thu Nov 29 11:19:45 2007 @@ -200,6 +200,16 @@ def test_unicode_convert(self): assert isinstance("%s" % (u"x"), unicode) + def test_unicode_nonascii(self): + """ + Interpolating a unicode string with non-ascii characters in it into + a string format should decode the format string as ascii and return + unicode. + """ + result = "%s" % u'\x80' + assert isinstance(result, unicode) + assert result == u'\x80' + def test_unicode_d(self): assert u"%.1d" % 3 == '3' From cfbolz at codespeak.net Thu Nov 29 11:37:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Nov 2007 11:37:27 +0100 (CET) Subject: [pypy-svn] r49200 - in pypy/dist/pypy/annotation: . test Message-ID: <20071129103727.4A45C8129@code0.codespeak.net> Author: cfbolz Date: Thu Nov 29 11:37:26 2007 New Revision: 49200 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: best effort detection to prevent str % unicode in RPython (which is broken badly right now) Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Thu Nov 29 11:37:26 2007 @@ -438,6 +438,23 @@ def add((chr1, chr2)): return SomeUnicodeString() +class __extend__(pairtype(SomeString, SomeUnicodeString), + pairtype(SomeString, SomeUnicodeString)): + def mod((str, unistring)): + raise NotImplementedError( + "string formatting mixing strings and unicode not supported") + + +class __extend__(pairtype(SomeString, SomeTuple)): + def mod((str, s_tuple)): + for s_item in s_tuple.items: + if isinstance(s_item, (SomeUnicodeCodePoint, SomeUnicodeString)): + raise NotImplementedError( + "string formatting mixing strings and unicode not supported") + getbookkeeper().count('strformat', str, args) + return SomeString() + + class __extend__(pairtype(SomeString, SomeObject)): def mod((str, args)): Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Thu Nov 29 11:37:26 2007 @@ -2962,6 +2962,30 @@ s = a.build_types(f, [unicode, str]) assert isinstance(s, annmodel.SomeUnicodeCodePoint) + def test_strformatting_unicode(self): + def f(x): + return '%s' % unichr(x) + + a = self.RPythonAnnotator() + py.test.raises(NotImplementedError, a.build_types, f, [int]) + def f(x): + return '%s' % (unichr(x) * 3) + + a = self.RPythonAnnotator() + py.test.raises(NotImplementedError, a.build_types, f, [int]) + def f(x): + return '%s%s' % (1, unichr(x)) + + a = self.RPythonAnnotator() + py.test.raises(NotImplementedError, a.build_types, f, [int]) + def f(x): + return '%s%s' % (1, unichr(x) * 15) + + a = self.RPythonAnnotator() + py.test.raises(NotImplementedError, a.build_types, f, [int]) + + + def test_negative_slice(self): def f(s, e): return [1, 2, 3][s:e] From fijal at codespeak.net Thu Nov 29 11:41:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Nov 2007 11:41:58 +0100 (CET) Subject: [pypy-svn] r49201 - pypy/dist/pypy/translator/c Message-ID: <20071129104158.DA0068129@code0.codespeak.net> Author: fijal Date: Thu Nov 29 11:41:57 2007 New Revision: 49201 Modified: pypy/dist/pypy/translator/c/genc.py Log: Logic for creating proper makefile. untested. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Thu Nov 29 11:41:57 2007 @@ -316,10 +316,14 @@ cfiles = [] ofiles = [] for fn in compiler.cfilenames: - fn = py.path.local(fn).basename - assert fn.endswith('.c') - cfiles.append(fn) - ofiles.append(fn[:-2] + '.o') + fn = py.path.local(fn) + if fn.dirpath() == targetdir: + name = fn.basename + else: + assert fn.dirpath().dirpath() == udir + name = '../' + fn.relto(udir) + cfiles.append(name) + ofiles.append(name[:-2] + '.o') if self.config.translation.cc: cc = self.config.translation.cc From cfbolz at codespeak.net Thu Nov 29 11:54:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Nov 2007 11:54:36 +0100 (CET) Subject: [pypy-svn] r49202 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071129105436.884F08129@code0.codespeak.net> Author: cfbolz Date: Thu Nov 29 11:54:36 2007 New Revision: 49202 Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py Log: kill dead code (and a not-understandable comment) Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Thu Nov 29 11:54:36 2007 @@ -797,9 +797,6 @@ cflags = inputconst(Void, {'flavor': 'gc'}) vtemp = hop.genop("malloc_varsize", [cTEMP, cflags, size], resulttype=Ptr(TEMP)) - # XXX hash - r_tuple = hop.args_r[1] - v_tuple = hop.args_v[1] argsiter = iter(sourcevarsrepr) From arigo at codespeak.net Thu Nov 29 14:46:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Nov 2007 14:46:16 +0100 (CET) Subject: [pypy-svn] r49204 - in pypy/dist/pypy/rpython/memory: gc test Message-ID: <20071129134616.D4FE78173@code0.codespeak.net> Author: arigo Date: Thu Nov 29 14:46:14 2007 New Revision: 49204 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/test/test_gc.py Log: Test and fix for a weakref bug in the GenerationGC. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Nov 29 14:46:14 2007 @@ -167,6 +167,8 @@ if not target: freeentry = i else: + debug_assert(self.get_type_id(llmemory.cast_ptr_to_adr(target)) + > 0, "bogus weakref in compute_id()") # record this entry in the dict adr = llmemory.cast_ptr_to_adr(target) self.object_id_dict[adr] = i Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Nov 29 14:46:14 2007 @@ -293,9 +293,10 @@ if self.is_forwarded(pointing_to): (obj + offset).address[0] = self.get_forwarding_address( pointing_to) - self.objects_with_weakrefs.append(obj) else: (obj + offset).address[0] = NULL + continue # no need to remember this weakref any longer + self.objects_with_weakrefs.append(obj) def write_barrier(self, oldvalue, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Thu Nov 29 14:46:14 2007 @@ -327,6 +327,29 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == 20 + 20 + def test_young_weakref_to_old_object(self): + import weakref + class A: + pass + def f(x): + a = A() + llop.gc__collect(lltype.Void) + # 'a' is old, 'ref' is young + ref = weakref.ref(a) + # now trigger a minor collection + all = [None] * x + i = 0 + while i < x: + all[i] = [i] * i + i += 1 + # now 'a' is old, but 'ref' did not move + assert ref() is a + llop.gc__collect(lltype.Void) + # now both 'a' and 'ref' have moved + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection From exarkun at codespeak.net Thu Nov 29 14:50:22 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Thu, 29 Nov 2007 14:50:22 +0100 (CET) Subject: [pypy-svn] r49205 - in pypy/dist/pypy/annotation: . test Message-ID: <20071129135022.8B8EF8163@code0.codespeak.net> Author: exarkun Date: Thu Nov 29 14:50:22 2007 New Revision: 49205 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: fix NameError in success case for str % tuple Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Thu Nov 29 14:50:22 2007 @@ -451,7 +451,7 @@ if isinstance(s_item, (SomeUnicodeCodePoint, SomeUnicodeString)): raise NotImplementedError( "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', str, args) + getbookkeeper().count('strformat', str, s_tuple) return SomeString() Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Thu Nov 29 14:50:22 2007 @@ -2985,6 +2985,19 @@ py.test.raises(NotImplementedError, a.build_types, f, [int]) + def test_strformatting_tuple(self): + """ + A function which returns the result of interpolating a tuple of a + single str into a str format string should be annotated as returning + SomeString. + """ + def f(x): + return '%s' % (x,) + + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeString) + def test_negative_slice(self): def f(s, e): From arigo at codespeak.net Thu Nov 29 15:57:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Nov 2007 15:57:08 +0100 (CET) Subject: [pypy-svn] r49206 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071129145708.D2BFD81A7@code0.codespeak.net> Author: arigo Date: Thu Nov 29 15:57:05 2007 New Revision: 49206 Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py Log: Reference to the issue. Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Thu Nov 29 15:57:05 2007 @@ -701,7 +701,7 @@ return 0 # for None cached = ins.hash_cache if cached == 0: - cached = ins.hash_cache = intmask(id(ins)) + cached = ins.hash_cache = intmask(id(ins)) # XXX issue325 return cached def ll_inst_type(obj): From arigo at codespeak.net Thu Nov 29 16:09:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Nov 2007 16:09:23 +0100 (CET) Subject: [pypy-svn] r49207 - in pypy/dist/pypy/interpreter/pyparser: . test Message-ID: <20071129150923.0B84F817A@code0.codespeak.net> Author: arigo Date: Thu Nov 29 16:09:22 2007 New Revision: 49207 Modified: pypy/dist/pypy/interpreter/pyparser/future.py pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py Log: RPython fix: IndexErrors are implicit and not checked in RPython unless explicitly requested. Modified: pypy/dist/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/future.py (original) +++ pypy/dist/pypy/interpreter/pyparser/future.py Thu Nov 29 16:09:22 2007 @@ -31,7 +31,7 @@ futures = FutureAutomaton(futureFlags, source) try: futures.start() - except (IndexError, DoneException), e: + except DoneException, e: pass return futures.flags @@ -68,9 +68,15 @@ self.pos = 0 self.docstringConsumed = False self.flags = 0 - + + def getc(self, offset=0): + try: + return self.s[self.pos + offset] + except IndexError: + raise DoneException + def start(self): - c = self.s[self.pos] + c = self.getc() if c in ["'", '"'] and not self.docstringConsumed: self.consumeDocstring() elif c in whitespace: @@ -84,19 +90,19 @@ def consumeDocstring(self): self.docstringConsumed = True - endchar = self.s[self.pos] - if (self.s[self.pos] == self.s[self.pos+1] and - self.s[self.pos] == self.s[self.pos+2]): + endchar = self.getc() + if (self.getc() == self.getc(+1) and + self.getc() == self.getc(+2)): self.pos += 3 while 1: # Deal with a triple quoted docstring - if self.s[self.pos] == '\\': + if self.getc() == '\\': self.pos += 2 - elif self.s[self.pos] != endchar: + elif self.getc() != endchar: self.pos += 1 else: self.pos += 1 - if (self.s[self.pos] == endchar and - self.s[self.pos+1] == endchar): + if (self.getc() == endchar and + self.getc(+1) == endchar): self.pos += 2 self.consumeEmptyLine() break @@ -104,18 +110,18 @@ else: # Deal with a single quoted docstring self.pos += 1 while 1: - c = self.s[self.pos] + c = self.getc() self.pos += 1 if c == endchar: self.consumeEmptyLine() return elif c == '\\': # Deal with linefeeds - if self.s[self.pos] != '\r': + if self.getc() != '\r': self.pos += 1 else: self.pos += 1 - if self.s[self.pos] == '\n': + if self.getc() == '\n': self.pos += 1 elif c in '\r\n': # Syntax error @@ -126,31 +132,29 @@ Called when the remainder of the line can only contain whitespace and comments. """ - while self.s[self.pos] in whitespace: + while self.getc() in whitespace: self.pos += 1 - if self.s[self.pos] == '#': + if self.getc() == '#': self.consumeComment() - elif self.s[self.pos] == ';': + elif self.getc() == ';': self.pos += 1 self.consumeWhitespace() self.start() - elif self.s[self.pos] in '\r\n': + elif self.getc() in '\r\n': self.pos += 1 - if self.s[self.pos] == '\n': + if self.getc() == '\n': self.pos += 1 self.start() def consumeComment(self): self.pos += 1 - while self.s[self.pos] not in '\r\n': + while self.getc() not in '\r\n': self.pos += 1 self.consumeEmptyLine() def consumeFrom(self): self.pos += 1 - p = self.pos - s = self.s - if s[p] == 'r' and s[p+1] == 'o' and s[p+2] == 'm': + if self.getc() == 'r' and self.getc(+1) == 'o' and self.getc(+2) == 'm': self.docstringConsumed = True self.pos += 3 self.consumeMandatoryWhitespace() @@ -162,7 +166,7 @@ raise DoneException self.pos += 6 self.consumeWhitespace() - if self.s[self.pos] == '(': + if self.getc() == '(': self.pos += 1 self.consumeWhitespace() self.setFlag(self.getName()) @@ -176,25 +180,25 @@ return def consumeMandatoryWhitespace(self): - if self.s[self.pos] not in whitespace + '\\': + if self.getc() not in whitespace + '\\': raise DoneException self.consumeWhitespace() def consumeWhitespace(self): while 1: - c = self.s[self.pos] + c = self.getc() if c in whitespace: self.pos += 1 continue elif c == '\\': self.pos += 1 - c = self.s[self.pos] + c = self.getc() if c == '\n': self.pos += 1 continue elif c == '\r': self.pos += 1 - if self.s[self.pos] == '\n': + if self.getc() == '\n': self.pos += 1 else: raise DoneException @@ -202,35 +206,35 @@ return def getName(self): - if self.s[self.pos] not in letters: + if self.getc() not in letters: raise DoneException p = self.pos while 1: self.pos += 1 - if self.s[self.pos] not in alphanumerics: + if self.getc() not in alphanumerics: break name = self.s[p:self.pos] self.consumeWhitespace() return name def getMore(self, parenList=False): - if parenList and self.s[self.pos] == ')': + if parenList and self.getc() == ')': self.pos += 1 return - if (self.s[self.pos] == 'a' and - self.s[self.pos+1] == 's' and - self.s[self.pos+2] in whitespace): + if (self.getc() == 'a' and + self.getc(+1) == 's' and + self.getc(+2) in whitespace): self.getName() self.getName() self.getMore(parenList=parenList) return - elif self.s[self.pos] != ',': + elif self.getc() != ',': return else: self.pos += 1 self.consumeWhitespace() - if parenList and self.s[self.pos] == ')': + if parenList and self.getc() == ')': self.pos += 1 return # Handles trailing comma inside parenthesis self.setFlag(self.getName()) Modified: pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_futureautomaton.py Thu Nov 29 16:09:22 2007 @@ -6,7 +6,7 @@ f = future.FutureAutomaton(future.futureFlags_2_5, s) try: f.start() - except IndexError, future.DoneException: + except future.DoneException: pass return f From regmee at codespeak.net Thu Nov 29 18:07:04 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 29 Nov 2007 18:07:04 +0100 (CET) Subject: [pypy-svn] r49208 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071129170704.3CD90813B@code0.codespeak.net> Author: regmee Date: Thu Nov 29 18:07:03 2007 New Revision: 49208 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Log: some experimentation on __iter__ and next Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Thu Nov 29 18:07:03 2007 @@ -9,6 +9,7 @@ def __call__(self, *args): import clr + # now call the .NET static method call_staticmethod in interp_cl return clr.call_staticmethod(self.class_name, self.meth_name, args) def __repr__(self): @@ -91,6 +92,20 @@ def __init__(self, *args): import clr self.__cliobj__ = clr._CliObject_internal(self.__cliclass__, args) +# self.index = self.__cliobj__.__len__(self) +# self.index = self.__cliobj__.call_method('Count',1) + print self.__cliobj__ +# self.index = self.Count + + def __iter__(self): + self.index = self.Count + return self + + def next(self): + if self.index == 0: + raise StopIteration + self.index = self.index - 1 + return self.this[self.index] def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers): @@ -102,10 +117,18 @@ for name in methods: d[name] = MethodWrapper(name) + # check if there is GetEnumerator() method + for method in methods: + if method == "GetEnumerator": + print "Enumerator found .. Hurray !!!!!" + # now add the __iter__ method to the class +# d['__iter__'] = sampleIter().__iter__ + assert len(indexers) <= 1 if indexers: name, getter, setter, is_static = indexers[0] assert not is_static + print " Indexers Tuple -----> (%s,%s,%s,%s) "%(name, getter, setter, is_static) if getter: d['__getitem__'] = d[getter] if setter: @@ -115,6 +138,8 @@ # we must add properties *after* the class has been created # because we need to store UnboundMethods as getters and setters for (name, getter, setter, is_static) in properties: + print " Properties Tuple -----> (%s,%s,%s,%s) "%(name, getter, setter, is_static) + fget = None fset = None if getter: From arigo at codespeak.net Thu Nov 29 19:51:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Nov 2007 19:51:40 +0100 (CET) Subject: [pypy-svn] r49215 - pypy/dist/pypy/annotation Message-ID: <20071129185140.4E73A81BB@code0.codespeak.net> Author: arigo Date: Thu Nov 29 19:51:37 2007 New Revision: 49215 Modified: pypy/dist/pypy/annotation/bookkeeper.py Log: Don't catch and discard a RuntimeError - it could mask a real problem like a stack overflow. Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Thu Nov 29 19:51:37 2007 @@ -365,16 +365,15 @@ s_hashfn = self.immutablevalue(x.key_hash) result.dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn) - done = False - while not done: - try: - for ek, ev in x.iteritems(): - result.dictdef.generalize_key(self.immutablevalue(ek)) - result.dictdef.generalize_value(self.immutablevalue(ev)) - except RuntimeError, r: - pass - else: - done = True + seen_elements = 0 + while seen_elements != len(x): + items = x.items() + for ek, ev in items: + result.dictdef.generalize_key(self.immutablevalue(ek)) + result.dictdef.generalize_value(self.immutablevalue(ev)) + seen_elements = len(items) + # if the dictionary grew during the iteration, + # start over again result.const_box = key return result else: From niko at codespeak.net Thu Nov 29 23:20:40 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 29 Nov 2007 23:20:40 +0100 (CET) Subject: [pypy-svn] r49216 - in pypy/dist/pypy/translator: cli/test jvm jvm/src/pypy oosupport oosupport/test_template Message-ID: <20071129222040.04B1181EB@code0.codespeak.net> Author: niko Date: Thu Nov 29 23:20:37 2007 New Revision: 49216 Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/methods.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/typesystem.py pypy/dist/pypy/translator/oosupport/metavm.py pypy/dist/pypy/translator/oosupport/test_template/builtin.py pypy/dist/pypy/translator/oosupport/test_template/constant.py Log: Refactor so that we can support multiple interpreters simultaneously. Instead of storing the interlink interface in a static field of PyPy.java, we now create a new instance of PyPy.java which has a custom interlink, and then dispatch to that as needed. This means that different PyPy interpters can each have their own PyPy instance customized to their specific interlink class. Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Thu Nov 29 23:20:37 2007 @@ -12,7 +12,6 @@ py.test.skip("Doesn't work on Windows, yet") class TestCliBuiltin(CliTest, BaseTestBuiltin): - test_os_dup = skip_os test_os_path_exists = skip_os test_os_isdir = skip_os Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu Nov 29 23:20:37 2007 @@ -15,7 +15,7 @@ import pypy.translator.jvm.constant as jvmconst from pypy.translator.jvm.typesystem import \ jStringBuilder, jInt, jVoid, jString, jChar, jObject, \ - jThrowable, JvmNativeClass + jThrowable, JvmNativeClass, jPyPy, JvmClassType from pypy.translator.jvm.builtin import JvmBuiltInType from pypy.translator.oosupport.database import Database as OODatabase @@ -51,6 +51,18 @@ self._object_impl = None self._object_exc_impl = None + # Create information about the Main class we will build: + # + # note that it will have a static field called 'pypy' that + # points to a PyPy instance. This PyPy instance has been + # paired with the appropriate Interlink implementation + # which allows it to create generated structures. + # + # These are public attributes that are referenced from + # elsewhere in the code. + self.jPyPyMain = JvmClassType(self._pkg('Main')) + self.pypy_field = jvmgen.Field.s(self.jPyPyMain, 'pypy', jPyPy) + # _________________________________________________________________ # Java String vs Byte Array # @@ -108,7 +120,7 @@ for method_name, helper in methods.items(): cls.add_method(node.InterlinkFunction(cls, method_name, helper)) cls.add_interface(jvmtype.jPyPyInterlink) - self.interlink_class = cls + self.jInterlinkImplementation = cls self.pending_node(cls) def types_for_graph(self, graph): Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Thu Nov 29 23:20:37 2007 @@ -13,7 +13,7 @@ jObject, jByteArray, jPyPyExcWrap, jIntegerClass, jLongClass, \ jDoubleClass, jCharClass, jStringBuilder, JvmScalarType, jArrayList, \ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ - jPyPyHashCode, jMap, jPyPyWeakRef, jSystem, jll_os + jPyPyHashCode, jMap, jPyPyWeakRef, jSystem, jll_os, jPyPyInterlink # ___________________________________________________________________________ @@ -267,7 +267,7 @@ opcode=INVOKESPECIAL) c = staticmethod(c) - # Create a virtual method: + # Create a virtual or interface method: def v(classty, methnm, argtypes, rettype): """ Shorthand to create a virtual method. @@ -337,56 +337,56 @@ DOUBLETOSTRINGD = Method.s(jDoubleClass, 'toString', (jDouble,), jString) CHARTOSTRINGC = Method.s(jCharClass, 'toString', (jChar,), jString) MATHIABS = Method.s(jMath, 'abs', (jInt,), jInt) -IABSOVF = Method.s(jPyPy, 'abs_ovf', (jInt,), jInt) +IABSOVF = Method.v(jPyPy, 'abs_ovf', (jInt,), jInt) MATHLABS = Method.s(jMath, 'abs', (jLong,), jLong) -LABSOVF = Method.s(jPyPy, 'abs_ovf', (jLong,), jLong) +LABSOVF = Method.v(jPyPy, 'abs_ovf', (jLong,), jLong) MATHDABS = Method.s(jMath, 'abs', (jDouble,), jDouble) -INEGOVF = Method.s(jPyPy, 'negate_ovf', (jInt,), jInt) -LNEGOVF = Method.s(jPyPy, 'negate_ovf', (jLong,), jLong) -IADDOVF = Method.s(jPyPy, 'add_ovf', (jInt, jInt), jInt) -LADDOVF = Method.s(jPyPy, 'add_ovf', (jLong, jLong), jLong) -ISUBOVF = Method.s(jPyPy, 'subtract_ovf', (jInt, jInt), jInt) -LSUBOVF = Method.s(jPyPy, 'subtract_ovf', (jLong, jLong), jLong) -IMULOVF = Method.s(jPyPy, 'multiply_ovf', (jInt, jInt), jInt) -LMULOVF = Method.s(jPyPy, 'multiply_ovf', (jLong, jLong), jLong) +INEGOVF = Method.v(jPyPy, 'negate_ovf', (jInt,), jInt) +LNEGOVF = Method.v(jPyPy, 'negate_ovf', (jLong,), jLong) +IADDOVF = Method.v(jPyPy, 'add_ovf', (jInt, jInt), jInt) +LADDOVF = Method.v(jPyPy, 'add_ovf', (jLong, jLong), jLong) +ISUBOVF = Method.v(jPyPy, 'subtract_ovf', (jInt, jInt), jInt) +LSUBOVF = Method.v(jPyPy, 'subtract_ovf', (jLong, jLong), jLong) +IMULOVF = Method.v(jPyPy, 'multiply_ovf', (jInt, jInt), jInt) +LMULOVF = Method.v(jPyPy, 'multiply_ovf', (jLong, jLong), jLong) MATHFLOOR = Method.s(jMath, 'floor', (jDouble,), jDouble) -IFLOORDIVOVF = Method.s(jPyPy, 'floordiv_ovf', (jInt, jInt), jInt) -LFLOORDIVOVF = Method.s(jPyPy, 'floordiv_ovf', (jLong, jLong), jLong) -IFLOORDIVZEROVF = Method.s(jPyPy, 'floordiv_zer_ovf', (jInt, jInt), jInt) -LFLOORDIVZEROVF = Method.s(jPyPy, 'floordiv_zer_ovf', (jLong, jLong), jLong) -IREMOVF = Method.s(jPyPy, 'mod_ovf', (jInt, jInt), jInt) -LREMOVF = Method.s(jPyPy, 'mod_ovf', (jLong, jLong), jLong) -ISHLOVF = Method.s(jPyPy, 'lshift_ovf', (jInt, jInt), jInt) -LSHLOVF = Method.s(jPyPy, 'lshift_ovf', (jLong, jLong), jLong) +IFLOORDIVOVF = Method.v(jPyPy, 'floordiv_ovf', (jInt, jInt), jInt) +LFLOORDIVOVF = Method.v(jPyPy, 'floordiv_ovf', (jLong, jLong), jLong) +IFLOORDIVZEROVF = Method.v(jPyPy, 'floordiv_zer_ovf', (jInt, jInt), jInt) +LFLOORDIVZEROVF = Method.v(jPyPy, 'floordiv_zer_ovf', (jLong, jLong), jLong) +IREMOVF = Method.v(jPyPy, 'mod_ovf', (jInt, jInt), jInt) +LREMOVF = Method.v(jPyPy, 'mod_ovf', (jLong, jLong), jLong) +ISHLOVF = Method.v(jPyPy, 'lshift_ovf', (jInt, jInt), jInt) +LSHLOVF = Method.v(jPyPy, 'lshift_ovf', (jLong, jLong), jLong) MATHDPOW = Method.s(jMath, 'pow', (jDouble, jDouble), jDouble) PRINTSTREAMPRINTSTR = Method.v(jPrintStream, 'print', (jString,), jVoid) CLASSFORNAME = Method.s(jClass, 'forName', (jString,), jClass) CLASSISASSIGNABLEFROM = Method.v(jClass, 'isAssignableFrom', (jClass,), jBool) -PYPYAPPEND = Method.s(jPyPy, 'append', - (jStringBuilder, jString), jVoid) +STRINGBUILDERAPPEND = Method.v(jStringBuilder, 'append', + (jString,), jStringBuilder) PYPYUINTCMP = Method.s(jPyPy, 'uint_cmp', (jInt,jInt,), jInt) PYPYULONGCMP = Method.s(jPyPy, 'ulong_cmp', (jLong,jLong), jInt) -PYPYUINTMOD = Method.s(jPyPy, 'uint_mod', (jInt, jInt), jInt) -PYPYUINTMUL = Method.s(jPyPy, 'uint_mul', (jInt, jInt), jInt) -PYPYUINTDIV = Method.s(jPyPy, 'uint_div', (jInt, jInt), jInt) -PYPYULONGMOD = Method.s(jPyPy, 'ulong_mod', (jLong, jLong), jLong) +PYPYUINTMOD = Method.v(jPyPy, 'uint_mod', (jInt, jInt), jInt) +PYPYUINTMUL = Method.v(jPyPy, 'uint_mul', (jInt, jInt), jInt) +PYPYUINTDIV = Method.v(jPyPy, 'uint_div', (jInt, jInt), jInt) +PYPYULONGMOD = Method.v(jPyPy, 'ulong_mod', (jLong, jLong), jLong) PYPYUINTTODOUBLE = Method.s(jPyPy, 'uint_to_double', (jInt,), jDouble) PYPYDOUBLETOUINT = Method.s(jPyPy, 'double_to_uint', (jDouble,), jInt) -PYPYDOUBLETOLONG = Method.s(jPyPy, 'double_to_long', (jDouble,), jLong) #PAUL -PYPYLONGBITWISENEGATE = Method.s(jPyPy, 'long_bitwise_negate', (jLong,), jLong) -PYPYSTRTOINT = Method.s(jPyPy, 'str_to_int', (jString,), jInt) -PYPYSTRTOUINT = Method.s(jPyPy, 'str_to_uint', (jString,), jInt) -PYPYSTRTOLONG = Method.s(jPyPy, 'str_to_long', (jString,), jLong) -PYPYSTRTOULONG = Method.s(jPyPy, 'str_to_ulong', (jString,), jLong) -PYPYSTRTOBOOL = Method.s(jPyPy, 'str_to_bool', (jString,), jBool) -PYPYSTRTODOUBLE = Method.s(jPyPy, 'str_to_double', (jString,), jDouble) -PYPYSTRTOCHAR = Method.s(jPyPy, 'str_to_char', (jString,), jChar) -PYPYBOOLTODOUBLE = Method.s(jPyPy, 'bool_to_double', (jBool,), jDouble) +PYPYDOUBLETOLONG = Method.v(jPyPy, 'double_to_long', (jDouble,), jLong) #PAUL +PYPYLONGBITWISENEGATE = Method.v(jPyPy, 'long_bitwise_negate', (jLong,), jLong) +PYPYSTRTOINT = Method.v(jPyPy, 'str_to_int', (jString,), jInt) +PYPYSTRTOUINT = Method.v(jPyPy, 'str_to_uint', (jString,), jInt) +PYPYSTRTOLONG = Method.v(jPyPy, 'str_to_long', (jString,), jLong) +PYPYSTRTOULONG = Method.v(jPyPy, 'str_to_ulong', (jString,), jLong) +PYPYSTRTOBOOL = Method.v(jPyPy, 'str_to_bool', (jString,), jBool) +PYPYSTRTODOUBLE = Method.v(jPyPy, 'str_to_double', (jString,), jDouble) +PYPYSTRTOCHAR = Method.v(jPyPy, 'str_to_char', (jString,), jChar) +PYPYBOOLTODOUBLE = Method.v(jPyPy, 'bool_to_double', (jBool,), jDouble) PYPYDUMP = Method.s(jPyPy, 'dump', (jString,), jVoid) PYPYDUMPEXCWRAPPER = Method.s(jPyPy, 'dump_exc_wrapper', (jObject,), jVoid) PYPYSERIALIZEBOOLEAN = Method.s(jPyPy, 'serialize_boolean', (jBool,), jString) PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) -PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,), jString) +PYPYSERIALIZEULONG = Method.s(jPyPy, 'serialize_ulonglong', (jLong,),jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDUNICHAR = Method.s(jPyPy, 'escaped_unichar', (jChar,), jString) @@ -396,13 +396,13 @@ PYPYRUNTIMENEW = Method.s(jPyPy, 'RuntimeNew', (jClass,), jObject) PYPYSTRING2BYTES = Method.s(jPyPy, 'string2bytes', (jString,), jByteArray) PYPYARRAYTOLIST = Method.s(jPyPy, 'array_to_list', (jObjectArray,), jArrayList) -PYPYOOPARSEFLOAT = Method.s(jPyPy, 'ooparse_float', (jString,), jDouble) +PYPYOOPARSEFLOAT = Method.v(jPyPy, 'ooparse_float', (jString,), jDouble) OBJECTGETCLASS = Method.v(jObject, 'getClass', (), jClass) CLASSGETNAME = Method.v(jClass, 'getName', (), jString) CUSTOMDICTMAKE = Method.s(jPyPyCustomDict, 'make', (jPyPyEquals, jPyPyHashCode), jPyPyCustomDict) PYPYWEAKREFCREATE = Method.s(jPyPyWeakRef, 'create', (jObject,), jPyPyWeakRef) -PYPYWEAKREFGET = Method.v(jPyPyWeakRef, 'll_get', (), jObject) +PYPYWEAKREFGET = Method.s(jPyPyWeakRef, 'll_get', (), jObject) # ___________________________________________________________________________ # Fields @@ -410,6 +410,29 @@ # Field objects encode information about fields. class Field(object): + + @staticmethod + def i(classty, fieldnm, fieldty, OOTYPE=None): + """ + Shorthand to create an instance field. + 'class' - JvmType object for the class containing the field + 'fieldnm' - name of the field (Python string) + 'fieldty' - JvmType object for the type of the field + 'OOTYPE' - optional OOTYPE object for the type of the field + """ + return Field(classty.name, fieldnm, fieldty, False, OOTYPE) + + @staticmethod + def s(classty, fieldnm, fieldty, OOTYPE=None): + """ + Shorthand to create a static field. + 'class' - JvmType object for the class containing the field + 'fieldnm' - name of the field (Python string) + 'fieldty' - JvmType object for the type of the field + 'OOTYPE' - optional OOTYPE object for the type of the field + """ + return Field(classty.name, fieldnm, fieldty, True, OOTYPE) + def __init__(self, classnm, fieldnm, jtype, static, OOTYPE=None): # All fields are public self.class_name = classnm # String, ie. "java.lang.Math" @@ -456,13 +479,14 @@ # jasmin_syntax is not needed, since this object itself never appears # as an argument an Opcode -SYSTEMOUT = Field('java.lang.System', 'out', jPrintStream, True) -SYSTEMERR = Field('java.lang.System', 'err', jPrintStream, True) -DOUBLENAN = Field('java.lang.Double', 'NaN', jDouble, True) -DOUBLEPOSINF = Field('java.lang.Double', 'POSITIVE_INFINITY', jDouble, True) -DOUBLENEGINF = Field('java.lang.Double', 'NEGATIVE_INFINITY', jDouble, True) +SYSTEMOUT = Field.s(jSystem, 'out', jPrintStream) +SYSTEMERR = Field.s(jSystem, 'err', jPrintStream) +DOUBLENAN = Field.s(jDoubleClass, 'NaN', jDouble) +DOUBLEPOSINF = Field.s(jDoubleClass, 'POSITIVE_INFINITY', jDouble) +DOUBLENEGINF = Field.s(jDoubleClass, 'NEGATIVE_INFINITY', jDouble) -PYPYINTERLINK= Field(jPyPy.name, 'interlink', jPyPyInterlink, True) +PYPYINTERLINK= Field.i(jPyPy, 'interlink', jPyPyInterlink) +PYPYOS = Field.i(jPyPy, 'os', jll_os) # ___________________________________________________________________________ # Generator State @@ -941,6 +965,11 @@ fieldobj = clsobj.lookup_field(fieldname) fieldobj.store(self) + def push_pypy(self): + """ Pushes the PyPy object which contains all of our helper methods + onto the stack """ + self.db.pypy_field.load(self) + def get_field(self, CONCRETETYPE, fieldname): clsobj = self.db.pending_class(CONCRETETYPE) fieldobj = clsobj.lookup_field(fieldname) @@ -999,16 +1028,29 @@ # we have to "deal with it" self.prepare_generic_result(RETTYPE) + def prepare_call_primitive(self, op, module, name): + # Load the PyPy object pointer onto the stack: + self.push_pypy() + + # If necessary, load the ll_os object pointer instead: + if module == 'll_os': + PYPYOS.load(self) + def call_primitive(self, op, module, name): callee = op.args[0].value - argtypes, rettype = self.db.types_for_signature(callee._TYPE.ARGS, callee._TYPE.RESULT) + argtypes, rettype = self.db.types_for_signature( + callee._TYPE.ARGS, callee._TYPE.RESULT) if module == 'll_os': jcls = jll_os else: jcls = jPyPy - mthd = Method.s(jcls, name, argtypes, rettype) + mthd = Method.v(jcls, name, argtypes, rettype) self.emit(mthd) + def prepare_call_oostring(self, OOTYPE): + # Load the PyPy object pointer onto the stack: + self.push_pypy() + def call_oostring(self, OOTYPE): cts_type = self.db.lltype_to_cts(OOTYPE) @@ -1016,14 +1058,18 @@ if isinstance(cts_type, jvmtype.JvmClassType): cts_type = jObject - mthd = Method.s(jPyPy, 'oostring', [cts_type, jInt], jString) + mthd = Method.v(jPyPy, 'oostring', [cts_type, jInt], jString) self.emit(mthd) if self.db.using_byte_array: self.emit(PYPYSTRING2BYTES) + def prepare_call_oounicode(self, OOTYPE): + # Load the PyPy object pointer onto the stack: + self.push_pypy() + def call_oounicode(self, OOTYPE): cts_type = self.db.lltype_to_cts(OOTYPE) - mthd = Method.s(jPyPy, 'oounicode', [cts_type], jString) + mthd = Method.v(jPyPy, 'oounicode', [cts_type], jString) self.emit(mthd) if self.db.using_byte_array: self.emit(PYPYSTRING2BYTES) @@ -1032,8 +1078,9 @@ jtype = self.db.lltype_to_cts(TYPE) self.new_with_jtype(jtype) - def new_with_jtype(self, jtype): - ctor = Method.c(jtype, ()) + def new_with_jtype(self, jtype, ctor=None): + if ctor is None: + ctor = Method.c(jtype, ()) self.emit(NEW, jtype) self.emit(DUP) self.emit(ctor) Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Thu Nov 29 23:20:37 2007 @@ -69,11 +69,12 @@ jexc: the JvmType of the exception pexcmthd: the name of the method on the PyPy object to call. The PyPy method must take no arguments, return void, and must - always throw an exception in practice. + always throw an exception in practice. It would be better to + just find the class to throw normally, but I don't know how. """ self.java_exc = jexc - self.pypy_method = jvmgen.Method.s( - jvmtype.jPyPy, pexcmthd, [], jvmtype.jVoid) + self.pypy_method = jvmgen.Method.v( + jvmtype.jPyPyInterlink, pexcmthd, [], jvmtype.jVoid) self.instruction = inst def render(self, gen, op): @@ -87,8 +88,10 @@ gen.goto(donelbl) # } catch (JavaExceptionType) { gen.mark(catchlbl) - gen.emit(jvmgen.POP) - gen.emit(self.pypy_method) + gen.emit(jvmgen.POP) # throw away the exception object + gen.push_pypy() # load the PyPy object + gen.emit(jvmgen.PYPYINTERLINK) # load the interlink field from it + gen.emit(self.pypy_method) # invoke the method # Note: these instructions will never execute, as we expect # the pypy_method to throw an exception and not to return. We # need them here to satisfy the Java verifier, however, as it @@ -158,3 +161,10 @@ if opcode: generator.emit(opcode) CastPrimitive = _CastPrimitive() + +class _PushPyPy(MicroInstruction): + """ Pushes the PyPy instance where our helper functions are found + from the static field on the generated PyPyMain class """ + def render(self, generator, op): + generator.push_pypy() +PushPyPy = _PushPyPy() Modified: pypy/dist/pypy/translator/jvm/methods.py ============================================================================== --- pypy/dist/pypy/translator/jvm/methods.py (original) +++ pypy/dist/pypy/translator/jvm/methods.py Thu Nov 29 23:20:37 2007 @@ -22,18 +22,16 @@ self.jrettype = jvmtype.jString def _print_field_value(self, fieldnm, FIELDOOTY): - self.gen.emit(jvmgen.DUP) self.gen.load_this_ptr() fieldobj = self.clsobj.lookup_field(fieldnm) fieldobj.load(self.gen) dumpmethod = self.db.toString_method_for_ootype(FIELDOOTY) self.gen.emit(dumpmethod) - self.gen.emit(jvmgen.PYPYAPPEND) + self.gen.emit(jvmgen.STRINGBUILDERAPPEND) def _print(self, str): - self.gen.emit(jvmgen.DUP) self.gen.load_string(str) - self.gen.emit(jvmgen.PYPYAPPEND) + self.gen.emit(jvmgen.STRINGBUILDERAPPEND) def render(self, gen): self.gen = gen Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Thu Nov 29 23:20:37 2007 @@ -20,9 +20,9 @@ from pypy.rpython.ootypesystem import \ ootype, rclass from pypy.translator.jvm.typesystem import \ - JvmClassType, jString, jStringArray, jVoid, jThrowable, jInt, jPyPyMain, \ + JvmClassType, jString, jStringArray, jVoid, jThrowable, jInt, \ jObject, JvmType, jStringBuilder, jPyPyInterlink, jCallbackInterfaces, \ - JvmInterfaceType + JvmInterfaceType, jPyPy from pypy.translator.jvm.opcodes import \ opcodes from pypy.translator.jvm.option import \ @@ -91,21 +91,35 @@ } def render(self, gen): - gen.begin_class(jPyPyMain, jObject) - gen.begin_function( - 'main', (), [jStringArray], jVoid, static=True) + gen.begin_class(gen.db.jPyPyMain, jObject) + gen.add_field(gen.db.pypy_field) # Initialization: # - # 1. Setup the PyPy helper, which (for now) is a static - # variable of the PyPy class, though that precludes running - # multiple translations. + # 1. Create a PyPy helper class, passing in an appropriate + # interlink instance. # # 2. Run the initialization method for the constant class. # - gen.new_with_jtype(gen.db.interlink_class) - jvmgen.PYPYINTERLINK.store(gen) + gen.begin_function( + '', (), [], jVoid, static=True) + gen.emit(jvmgen.NEW, jPyPy) + gen.emit(jvmgen.DUP) + gen.new_with_jtype(gen.db.jInterlinkImplementation) + gen.emit(jvmgen.Method.c(jPyPy, [jPyPyInterlink])) + gen.db.pypy_field.store(gen) gen.db.constant_generator.runtime_init(gen) + gen.return_val(jVoid) + gen.end_function() + + # Main method: + # + # 1. Parse the arguments and create Python objects. + # 2. Invoke the main function. + # 3. Print the result to stdout if self.print_result is true + # + gen.begin_function( + 'main', (), [jStringArray], jVoid, static=True) if self.print_result: gen.begin_try() @@ -116,10 +130,11 @@ # invoking an appropriate helper function on each one for i, arg in enumerate(self.graph.getargs()): jty = self.db.lltype_to_cts(arg.concretetype) + conv = self._type_conversion_methods[arg.concretetype] + if conv: gen.push_pypy() gen.load_jvm_var(jStringArray, 0) gen.emit(jvmgen.ICONST, i) gen.load_from_array(jString) - conv = self._type_conversion_methods[arg.concretetype] if conv: gen.emit(conv) else: # Convert the array of strings to a List as the Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Thu Nov 29 23:20:37 2007 @@ -11,21 +11,40 @@ CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ - CastPrimitive + CastPrimitive, PushPyPy from pypy.rpython.ootypesystem import ootype import pypy.translator.jvm.generator as jvmgen import pypy.translator.jvm.typesystem as jvmtype def _proc(val): + if isinstance(val, list): + # Lists of instructions we leave alone: + return InstructionList(val) + elif isinstance(val, jvmgen.Method) and not val.is_static(): + # For virtual methods, we first push an instance of the relevant + # class, then the arguments, and then invoke the method. Note + # that we only allow virtual methods of certain pre-designated + # classes to be in the table. + if val.class_name == jvmtype.jPyPy.name: + return InstructionList( + (PushPyPy, PushAllArgs, val, StoreResult)) + else: + raise Exception("Unknown class for non-static method") + # For anything else (static methods, strings, etc) we first push + # all arguments, then invoke the emit() routine, and finally + # store the result. + return InstructionList((PushAllArgs, val, StoreResult)) + +def _proc_dict(original): + """ Function which is used to post-process each entry in the - opcodes table; it adds a PushAllArgs and StoreResult by default, - unless the entry is a list already. """ - if not isinstance(val, list): - val = InstructionList((PushAllArgs, val, StoreResult)) - else: - val = InstructionList(val) - return val + opcodes table.""" + + res = {} + for key, val in original.items(): + res[key] = _proc(val) + return res def _check_zer(op): return [TranslateException( @@ -41,7 +60,7 @@ # This table maps the opcodes to micro-ops for processing them. # It is post-processed by _proc. -_opcodes = { +opcodes = _proc_dict({ # __________ object oriented operations __________ 'new': [New, StoreResult], 'runtimenew': [RuntimeNew, StoreResult], @@ -58,16 +77,13 @@ 'oohash': [PushAllArgs, jvmgen.OBJHASHCODE, StoreResult], 'oostring': [OOString, StoreResult], 'oounicode': [OOUnicode, StoreResult], - #'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': jvmgen.PYPYOOPARSEFLOAT, 'oonewcustomdict': [NewCustomDict, StoreResult], - # 'same_as': DoNothing, 'hint': [PushArg(0), StoreResult], 'direct_call': [Call, StoreResult], 'indirect_call': [PushAllArgs, IndirectCall, StoreResult], - #'cast_ptr_to_weakadr': CastPtrToWeakAddress 'gc__collect': jvmgen.SYSTEMGC, 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, @@ -222,7 +238,7 @@ 'ullong_ge': 'ulong_greater_equals', 'ullong_lshift': [PushAllArgs, jvmgen.L2I, jvmgen.LSHL, StoreResult], 'ullong_rshift': [PushAllArgs, jvmgen.L2I, jvmgen.LUSHR, StoreResult], - 'ullong_mod_zer': _check_zer(jvmgen.PYPYULONGMOD), + 'ullong_mod_zer': jvmgen.PYPYULONGMOD, # when casting from bool we want that every truth value is casted # to 1: we can't simply DoNothing, because the CLI stack could @@ -248,10 +264,4 @@ 'cast_primitive': [PushAllArgs, CastPrimitive, StoreResult], 'is_early_constant': [PushPrimitive(ootype.Bool, False), StoreResult] -} - -opcodes = {} -for opc, val in _opcodes.items(): - opcodes[opc] = _proc(val) -del _opcodes - +}) Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Thu Nov 29 23:20:37 2007 @@ -9,21 +9,33 @@ import java.text.DecimalFormat; /** - * Class with a number of utility routines. + * Class with a number of utility routines. One instance of this is + * created by the PyPy entrypoint, and paired with an appropriate + * interlink implementation. * * I apologize for the Python-esque naming conventions, but it seems * I can't switch my mind to camelCase when working so closely with * Python mere minutes before. + * + * In general, its methods should be virtual. In some cases, however, + * they are static because it is more expedient in the generated code + * to not have to push the pypy instance before invoking the method. */ public class PyPy implements Constants { - public static Interlink interlink; + public final Interlink interlink; + public final ll_os os; - public static final long LONG_MAX = Long.MAX_VALUE; - public static final long LONG_MIN = Long.MIN_VALUE; - public static final int INT_MAX = Integer.MAX_VALUE; - public static final int INT_MIN = Integer.MIN_VALUE; - public static final double ULONG_MAX = 18446744073709551616.0; + public PyPy(Interlink interlink) { + this.interlink = interlink; + this.os = new ll_os(interlink); + } + + public final static long LONG_MAX = Long.MAX_VALUE; + public final static long LONG_MIN = Long.MIN_VALUE; + public final static int INT_MAX = Integer.MAX_VALUE; + public final static int INT_MIN = Integer.MIN_VALUE; + public final static double ULONG_MAX = 18446744073709551616.0; /** * Compares two unsigned integers (value1 and value2) and returns @@ -68,36 +80,41 @@ return VALUE2BIGGER; } - public static int uint_mod(int x, int y) { + public int uint_mod(int x, int y) { long lx = uint_to_long(x); long ly = uint_to_long(y); long lr = lx % ly; return long_to_uint(lr); } - public static int uint_mul(int x, int y) + public int uint_mul(int x, int y) { long xx = uint_to_long(x); long yy = uint_to_long(y); return long_to_uint(xx * yy); } - public static int uint_div(int x, int y) + public int uint_div(int x, int y) { long xx = uint_to_long(x); long yy = uint_to_long(y); return long_to_uint(xx / yy); } - public static long ulong_shl(long x, long y) { + public long ulong_shl(long x, long y) { int yi = (int)y; return x << yi; } - public static long ulong_mod(long x, long y) { + public long ulong_mod(long x, long y) { double dx = ulong_to_double(x); - double modulo = Math.IEEEremainder(dx, y); - return (long)modulo; + try { + double modulo = Math.IEEEremainder(dx, y); + return (long)modulo; + } catch (ArithmeticException e) { + interlink.throwZeroDivisionError(); + return 0; // never reached + } } public static int ulong_cmp(long value1, long value2) { @@ -131,7 +148,7 @@ return VALUE2BIGGER; } - public static final double BITS16 = (double)0xFFFF; + public final double BITS16 = (double)0xFFFF; public static double uint_to_double(int value) { return (double)uint_to_long(value); @@ -167,7 +184,7 @@ return res; } - public static long double_to_long(double value) + public long double_to_long(double value) { //if (value <= LONG_MAX) //{ @@ -176,11 +193,11 @@ //TODO: Add some logic here, but I don't think we'll need it } - public static long long_bitwise_negate(long value) { + public long long_bitwise_negate(long value) { return ~value; } - public static int str_to_int(String s) { + public int str_to_int(String s) { try { return Integer.parseInt(s); } catch (NumberFormatException fe) { @@ -188,7 +205,7 @@ } } - public static int str_to_uint(String s) { + public int str_to_uint(String s) { try { long l = Long.parseLong(s); if (l < Integer.MAX_VALUE) @@ -201,7 +218,7 @@ } } - public static long str_to_long(String s) { + public long str_to_long(String s) { try { return Long.parseLong(s); } catch (NumberFormatException fe) { @@ -209,7 +226,7 @@ } } - public static long str_to_ulong(String s) { + public long str_to_ulong(String s) { long res = 0; s = s.trim(); for(int i=0; i=0) || ((result^y) >=0))) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } - public static int subtract_ovf(int x, int y) + public int subtract_ovf(int x, int y) { int result = x-y; if (!(((result^x) >=0) || ((result^(~y)) >=0))) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } @@ -458,31 +478,31 @@ else return false; } - public static int multiply_ovf(int x, int y) + public int multiply_ovf(int x, int y) { if (!(int_multiply(x, y))) { - throwOverflowError(); + interlink.throwOverflowError(); } return x * y; } - public static long add_ovf(long x, long y) + public long add_ovf(long x, long y) { long result = x+y; if (!(((result^x) >=0) || ((result^y) >=0))) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } - public static long subtract_ovf(long x, long y) + public long subtract_ovf(long x, long y) { long result = x-y; if (!(((result^x) >=0) || ((result^(~y)) >=0))) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } @@ -503,54 +523,51 @@ else return false; } - public static long multiply_ovf(long x, long y) + public long multiply_ovf(long x, long y) { //if (long_multiply(x, y)) //{ // return x * y; //} //else - // throwOverflowError(); + // interlink.throwOverflowError(); if (!(long_multiply(x, y))) { - throwOverflowError(); + interlink.throwOverflowError(); } return x*y; //else - // throwOverflowError(); + // interlink.throwOverflowError(); } /* floor division */ - public static int floordiv_ovf(int x, int y) + public int floordiv_ovf(int x, int y) { if ((y == -1) && (x == INT_MIN)) { - throwOverflowError(); + interlink.throwOverflowError(); } return x/y; } - public static int floordiv_zer_ovf(int x, int y) + public int floordiv_zer_ovf(int x, int y) { - if (y != 0) - { - return floordiv_ovf(x,y); - } - else - throw new ArithmeticException("Floor Division with integer by 0"); + if (y == 0) + interlink.throwZeroDivisionError(); + return floordiv_ovf(x,y); } - public static long floordiv_ovf(long x, long y) + public long floordiv_ovf(long x, long y) { if ((y == -1) && (x == LONG_MIN)) { - throwOverflowError(); + interlink.throwOverflowError(); } return x/y; } - public static long floordiv_zer_ovf(long x, long y) + public long floordiv_zer_ovf(long x, long y) { if (y != 0) { @@ -561,41 +578,41 @@ } /* modulo */ - public static int mod_ovf(int x, int y) + public int mod_ovf(int x, int y) { if ((y == -1) && (x == INT_MIN)) { - throwOverflowError(); + interlink.throwOverflowError(); } return x%y; } - public static long mod_ovf(long x, long y) + public long mod_ovf(long x, long y) { if ((y == -1) && (x == LONG_MIN)) { - throwOverflowError(); + interlink.throwOverflowError(); } return x%y; } /* shifting */ - public static int lshift_ovf(int x, int y) // x << y + public int lshift_ovf(int x, int y) // x << y { int result = x << y; if (x != (result >> y)) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } - public static long lshift_ovf(long x, long y) // x << y + public long lshift_ovf(long x, long y) // x << y { long result = x << y; if (x != (result >> y)) { - throwOverflowError(); + interlink.throwOverflowError(); } return result; } @@ -612,30 +629,29 @@ return str.indexOf((int) char1) != -1; } - public static int ll_find(String haystack, String needle, int start, int end) { - // if it is impossible for the needle to occur: - // this deals w/ a disparity in how java and python handle when needle="" + public static int ll_find(String haystack, String needle, + int start, int end) { + // if it is impossible for the needle to occur: this deals w/ + // a disparity in how java and python handle when needle="" if (start > haystack.length()) return -1; int res = haystack.indexOf(needle, start); - //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - // " end="+end+" res="+res); if (res + needle.length() > end) return -1; return res; } - public static int ll_rfind(String haystack, String needle, int start, int end) { + public static int ll_rfind(String haystack, String needle, + int start, int end) { int res = haystack.lastIndexOf(needle, end-1); - //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - // " end="+end+" res="+res); if (res >= start) return res; return -1; } - public static int ll_count(String haystack, String needle, int start, int end) { + public static int ll_count(String haystack, String needle, + int start, int end) { haystack = haystack.substring(start, end); if (needle.length() == 0) { @@ -651,7 +667,8 @@ return cnt; } - public static int ll_find_char(String haystack, char needle, int start, int end) { + public static int ll_find_char(String haystack, char needle, + int start, int end) { // see ll_find if (start > haystack.length()) return -1; @@ -662,16 +679,16 @@ return res; } - public static int ll_rfind_char(String haystack, char needle, int start, int end) { + public static int ll_rfind_char(String haystack, char needle, + int start, int end) { int res = haystack.lastIndexOf(needle, end-1); - //System.err.println("haystack="+haystack+" needle="+needle+" start="+start+ - // " end="+end+" res="+res); if (res >= start) return res; return -1; } - public static int ll_count_char(String haystack, char needle, int start, int end) { + public static int ll_count_char(String haystack, char needle, + int start, int end) { haystack = haystack.substring(start, end); int cnt = 0; int idx = -1; @@ -681,7 +698,8 @@ return cnt; } - public static String ll_strip(String str, char ch, boolean left, boolean right) { + public static String ll_strip(String str, char ch, + boolean left, boolean right) { int start = 0; int end = str.length(); @@ -769,11 +787,6 @@ return s.getBytes(); } - public static void append(StringBuilder sb, String s) { - // avoid the annoying return value of StringBuilder.append - sb.append(s); - } - public static ArrayList array_to_list(Object[] array) { ArrayList list = new ArrayList(java.util.Arrays.asList(array)); @@ -784,7 +797,7 @@ // ---------------------------------------------------------------------- // OOString support - public static String oostring(int n, int base_) { + public String oostring(int n, int base_) { // XXX needs special case for unsigned ints if (base_ == -1) base_ = 10; @@ -794,11 +807,11 @@ return Integer.toString(n, base_); } - public static String oostring(double d, int base_) { + public String oostring(double d, int base_) { return new Double(d).toString(); } - public static String oostring(Object obj, int base_) + public String oostring(Object obj, int base_) { String clnm = obj.getClass().getName(); int underscore = clnm.lastIndexOf('_'); @@ -807,22 +820,22 @@ return String.format("<%s object>", new Object[] { clnm }); } - public static String oostring(char ch, int base_) + public String oostring(char ch, int base_) { return new Character(ch).toString(); } - public static byte[] oostring(byte[] s, int base_) + public byte[] oostring(byte[] s, int base_) { return s; } - public static String oostring(String s, int base_) + public String oostring(String s, int base_) { return s; } - public static String oostring(boolean b, int base_) + public String oostring(boolean b, int base_) { if (b) return "True"; return "False"; @@ -831,17 +844,17 @@ // ---------------------------------------------------------------------- // OOUnicode support - public static String oounicode(char ch) + public String oounicode(char ch) { return new Character(ch).toString(); } - public static String oounicode(String s) + public String oounicode(String s) { for(int i=0; i 127) - throwUnicodeDecodeError(); + interlink.throwUnicodeDecodeError(); } return s; } @@ -849,15 +862,15 @@ // ---------------------------------------------------------------------- // Primitive built-in functions - public static double ll_time_clock() { + public double ll_time_clock() { return System.currentTimeMillis()/1000.0; // XXX: processor time? } - public static double ll_time_time() { + public double ll_time_time() { return System.currentTimeMillis()/1000.0; } - public static void ll_time_sleep(double seconds) + public void ll_time_sleep(double seconds) { double startTime = ll_time_time(); double endTime = startTime + seconds; @@ -870,12 +883,12 @@ } while (startTime < endTime); } - public static String ll_join(String a, String b) + public String ll_join(String a, String b) { return a + "/" + b; // XXX } - public static String ll_strtod_formatd(String format, double d) + public String ll_strtod_formatd(String format, double d) { // XXX: this is really a quick hack to make things work. // it should disappear, because this function is not @@ -884,27 +897,6 @@ } // ---------------------------------------------------------------------- - // Exceptions - // - // If we don't use true Java exceptions, then this - -/* - static private ThreadLocal excObject = new ThreadLocal(); - - public static int startTry() { - return excCounter.get(); - } - - public void throw(Object o) { - excObject.put(o); - } - - public static Object catch(int ctr) { - return excObject.get(); - } -*/ - - // ---------------------------------------------------------------------- // Dicts // // Note: it's easier to cut and paste a few methods here than @@ -966,56 +958,29 @@ // ---------------------------------------------------------------------- // ll_math - public static double ll_math_floor(double x) + public double ll_math_floor(double x) { return Math.floor(x); } - public static double ll_math_fmod(double x, double y) + public double ll_math_fmod(double x, double y) { return x % y; } - public static double ll_math_ldexp(double v, int w) { + public double ll_math_ldexp(double v, int w) { return check(v * Math.pow(2.0, w)); } - private static double check(double v) { + private double check(double v) { if (Double.isNaN(v)) - throwValueError(); + interlink.throwValueError(); if (Double.isInfinite(v)) - throwOverflowError(); + interlink.throwOverflowError(); return v; } // ---------------------------------------------------------------------- - // Convenient Helpers for throwing exceptions - // - // Also, an abstraction barrier: at a later date we may want to - // switch to using thread-local data rather than a global variable, - // and if so we can easily do it in these functions here. - - public static void throwZeroDivisionError() { - interlink.throwZeroDivisionError(); - } - - public static void throwIndexError() { - interlink.throwIndexError(); - } - - public static void throwOverflowError() { - interlink.throwOverflowError(); - } - - public static void throwValueError() { - interlink.throwValueError(); - } - - public static void throwUnicodeDecodeError() { - interlink.throwUnicodeDecodeError(); - } - - // ---------------------------------------------------------------------- // Self Test public static int __counter = 0, __failures = 0; @@ -1033,6 +998,8 @@ public static void main(String args[]) { // Small self test: + PyPy pypy = new PyPy(null); + ensure(uint_cmp(0xFFFFFFFF, 0) > 0); ensure(uint_cmp(0, 0xFFFFFFFF) < 0); ensure(uint_cmp(0x80000000, 0) > 0); Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Thu Nov 29 23:20:37 2007 @@ -18,11 +18,13 @@ class PrintStreamWrapper extends FileWrapper { - private PrintStream stream; + private final PrintStream stream; + private final ll_os os; - public PrintStreamWrapper(PrintStream stream) + public PrintStreamWrapper(PrintStream stream, ll_os os) { this.stream = stream; + this.os = os; } public void write(String buffer) @@ -32,13 +34,13 @@ public String read(int count) { - ll_os.throwOSError(PyPy.EBADF, "Write-only fd"); + os.throwOSError(PyPy.EBADF, "Write-only fd"); return null; // never reached } public void close() { - ll_os.throwOSError(PyPy.EBADF, "Cannot close stdout or stderr"); + os.throwOSError(PyPy.EBADF, "Cannot close stdout or stderr"); } public RandomAccessFile getFile() @@ -49,16 +51,18 @@ class InputStreamWrapper extends FileWrapper { - private InputStream stream; + private final InputStream stream; + private final ll_os os; - public InputStreamWrapper(InputStream stream) + public InputStreamWrapper(InputStream stream, ll_os os) { this.stream = stream; + this.os = os; } public void write(String buffer) { - ll_os.throwOSError(PyPy.EBADF, "Read-only fd"); + os.throwOSError(PyPy.EBADF, "Read-only fd"); } public String read(int count) @@ -71,14 +75,14 @@ return new String(buf, 0, n); } catch(IOException e) { - ll_os.throwOSError(PyPy.EIO, e.getMessage()); + os.throwOSError(PyPy.EIO, e.getMessage()); return null; // never reached } } public void close() { - ll_os.throwOSError(PyPy.EBADF, "Cannot close stdin"); + os.throwOSError(PyPy.EBADF, "Cannot close stdin"); } public RandomAccessFile getFile() @@ -89,34 +93,39 @@ class RandomAccessFileWrapper extends FileWrapper { - private RandomAccessFile file; - private boolean canRead; - private boolean canWrite; - - public RandomAccessFileWrapper(RandomAccessFile file, boolean canRead, boolean canWrite) + private final RandomAccessFile file; + private final boolean canRead; + private final boolean canWrite; + private final ll_os os; + + public RandomAccessFileWrapper(RandomAccessFile file, + boolean canRead, + boolean canWrite, + ll_os os) { this.file = file; this.canRead = canRead; this.canWrite = canWrite; + this.os = os; } public void write(String buffer) { if (!this.canWrite) - ll_os.throwOSError(PyPy.EBADF, "Cannot write to this fd"); + os.throwOSError(PyPy.EBADF, "Cannot write to this fd"); try { this.file.writeBytes(buffer); } catch(IOException e) { - ll_os.throwOSError(PyPy.EIO, e.getMessage()); + os.throwOSError(PyPy.EIO, e.getMessage()); } } public String read(int count) { if (!this.canRead) - ll_os.throwOSError(PyPy.EBADF, "Cannot read from this fd"); + os.throwOSError(PyPy.EBADF, "Cannot read from this fd"); try { byte[] buffer = new byte[count]; @@ -127,7 +136,7 @@ return new String(buffer, 0, n); } catch(IOException e) { - ll_os.throwOSError(PyPy.EIO, e.getMessage()); + os.throwOSError(PyPy.EIO, e.getMessage()); return null; // never reached } } @@ -138,7 +147,7 @@ this.file.close(); } catch(IOException e) { - ll_os.throwOSError(PyPy.EIO, e.getMessage()); + os.throwOSError(PyPy.EIO, e.getMessage()); } } @@ -172,36 +181,40 @@ private static final int SEEK_CUR = 1; private static final int SEEK_END = 2; - private static int fdcount; - private static Map FileDescriptors = new HashMap(); - private static Map ErrorMessages = new HashMap(); - - static { - FileDescriptors.put(new Integer(0), new InputStreamWrapper(System.in)); - FileDescriptors.put(new Integer(1), new PrintStreamWrapper(System.out)); - FileDescriptors.put(new Integer(2), new PrintStreamWrapper(System.err)); + private int fdcount; + private final Map FileDescriptors = + new HashMap(); + private final Map ErrorMessages = + new HashMap(); + private final Interlink interlink; + + public ll_os(Interlink interlink) { + this.interlink = interlink; + FileDescriptors.put(0, new InputStreamWrapper(System.in, this)); + FileDescriptors.put(1, new PrintStreamWrapper(System.out, this)); + FileDescriptors.put(2, new PrintStreamWrapper(System.err, this)); fdcount = 2; } - + public static final boolean STRACE = false; public static void strace(String arg) { System.err.println(arg); } - public static void throwOSError(int errno, String errText) { - ErrorMessages.put(new Integer(errno), errText); - PyPy.interlink.throwOSError(errno); + public void throwOSError(int errno, String errText) { + ErrorMessages.put(errno, errText); + interlink.throwOSError(errno); } - private static FileWrapper getfd(int fd) + private FileWrapper getfd(int fd) { - FileWrapper f = FileDescriptors.get(new Integer(fd)); + FileWrapper f = FileDescriptors.get(fd); if (f == null) throwOSError(PyPy.EBADF, "Invalid file descriptor: " + fd); return f; } - private static RandomAccessFile open_file(String name, String javaMode, int flags) + private RandomAccessFile open_file(String name, String javaMode, int flags) { RandomAccessFile file; @@ -228,7 +241,7 @@ return file; } - public static int ll_os_open(String name, int flags, int mode) + public int ll_os_open(String name, int flags, int mode) { boolean canRead = false; boolean canWrite = false; @@ -246,24 +259,25 @@ // XXX: we ignore O_CREAT RandomAccessFile file = open_file(name, javaMode, flags); - RandomAccessFileWrapper wrapper = new RandomAccessFileWrapper(file, canRead, canWrite); + RandomAccessFileWrapper wrapper = + new RandomAccessFileWrapper(file, canRead, canWrite, this); fdcount++; - FileDescriptors.put(new Integer(fdcount), wrapper); + FileDescriptors.put(fdcount, wrapper); if (STRACE) strace("ll_os_open: "+name+"->"+fdcount); return fdcount; } - public static void ll_os_close(int fd) + public void ll_os_close(int fd) { if (STRACE) strace("ll_os_close: "+fd); FileWrapper wrapper = getfd(fd); wrapper.close(); - FileDescriptors.remove(new Integer(fd)); + FileDescriptors.remove(fd); } - public static int ll_os_dup(int fd) + public int ll_os_dup(int fd) { FileWrapper wrapper = getfd(fd); for (int i = 0; i < Integer.MAX_VALUE; i++) { @@ -277,19 +291,19 @@ return -1; } - public static String ll_os_read(int fd, int count) + public String ll_os_read(int fd, int count) { if (STRACE) strace("ll_os_read: "+fd); return getfd(fd).read(count); } - public static String ll_os_read(int fd, long count) + public String ll_os_read(int fd, long count) { if (STRACE) strace("ll_os_read: "+fd); return ll_os_read(fd, (int)count); } - public static long ll_os_lseek(int fd, long offset, int whence) + public long ll_os_lseek(int fd, long offset, int whence) { if (STRACE) strace("ll_os_lseek: "+fd); FileWrapper wrapper = getfd(fd); @@ -320,30 +334,30 @@ return pos; } - public static StatResult ll_os_lstat(String path) + public StatResult ll_os_lstat(String path) { return ll_os_stat(path); // XXX } - public static String ll_os_strerror(int errno) + public String ll_os_strerror(int errno) { - String msg = ErrorMessages.remove(new Integer(errno)); + String msg = ErrorMessages.remove(errno); if (msg == null) return "errno: " + errno; else return msg; } - public static int ll_os_write(int fd, String text) { + public int ll_os_write(int fd, String text) { if (STRACE) strace("ll_os_write: "+fd+" "+text); - FileWrapper f = FileDescriptors.get(new Integer(fd)); + FileWrapper f = FileDescriptors.get(fd); if (f == null) throwOSError(PyPy.EBADF, "Invalid fd: " + fd); f.write(text); return text.length(); } - public static boolean ll_os_isatty(int x) + public boolean ll_os_isatty(int x) { // XXX: this is not the right behaviour, but it's needed // to have the interactive interpreter working @@ -353,25 +367,25 @@ return false; } - public static String ll_os_getenv(String key) + public String ll_os_getenv(String key) { return System.getenv(key); } - public static void ll_os_putenv(String key, String value) + public void ll_os_putenv(String key, String value) { //System.setenv(key, value); // it appears that there is no such method??!! } - public static ArrayList ll_os_envkeys() + public ArrayList ll_os_envkeys() { Map variables = System.getenv(); Set variableNames = variables.keySet(); return new ArrayList(variableNames); } - public static ArrayList ll_os_envitems() + public ArrayList ll_os_envitems() { Map variables = System.getenv(); Set variableNames = variables.keySet(); @@ -382,13 +396,13 @@ { String name = (String) nameIterator.next(); String value = (String) variables.get(name); - result.add(PyPy.interlink.recordStringString(name, value)); + result.add(interlink.recordStringString(name, value)); } return result; } - public static ArrayList ll_os_listdir(String path) + public ArrayList ll_os_listdir(String path) { if (path == "") throwOSError(PyPy.ENOENT, "No such file or directory: ''"); @@ -400,15 +414,15 @@ return new ArrayList(Arrays.asList(f.list())); } - public static String ll_os_getcwd() + public String ll_os_getcwd() { return System.getProperty("user.dir"); } - public static StatResult ll_os_stat(String path) + public StatResult ll_os_stat(String path) { if (path.equals("")) - ll_os.throwOSError(PyPy.ENOENT, "No such file or directory: ''"); + throwOSError(PyPy.ENOENT, "No such file or directory: ''"); File f = new File(path); @@ -424,7 +438,7 @@ return res; } - ll_os.throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); + throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); return null; // never reached } } Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Thu Nov 29 23:20:37 2007 @@ -172,9 +172,8 @@ jArrayList = JvmClassType('java.util.ArrayList') jPyPy = JvmClassType('pypy.PyPy') jPyPyExcWrap = JvmClassType('pypy.ExceptionWrapper') -jPyPyMain = JvmClassType('pypy.Main') jPyPyDictItemsIterator = JvmClassType('pypy.DictItemsIterator') -jPyPyInterlink = JvmClassType('pypy.Interlink') +jPyPyInterlink = JvmInterfaceType('pypy.Interlink') jPyPyCustomDict = JvmClassType('pypy.CustomDict') jPyPyStatResult = JvmClassType('pypy.StatResult') jPyPyWeakRef = JvmClassType('pypy.PyPyWeakRef') Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Thu Nov 29 23:20:37 2007 @@ -172,13 +172,42 @@ Stack: argN, arg2, arg1, this, ... -> ret, ... """ raise NotImplementedError + def prepare_call_primitive(self, op, module, name): + """ see call_primitive: by default does nothing """ + pass + def call_primitive(self, op, module, name): """ Like call_graph, but it has been suggested that the method be - rendered as a primitive. + rendered as a primitive. The full sequence for invoking a primitive: + + self.prepare_call_primitive(op, module, name) + for each arg: self.load(arg) + self.call_primitive(op, module, name) Stack: argN...arg2, arg1, arg0, ... -> ret, ... """ raise NotImplementedError + def prepare_call_oostring(self, OOTYPE): + " see call_oostring " + pass + + def call_oostring(self, OOTYPE): + """ Invoked for the oostring opcode with both operands + (object, int base) already pushed onto the stack. + prepare_call_oostring() is invoked before the operands are + pushed.""" + raise NotImplementedError + + def prepare_call_oounicode(self, OOTYPE): + " see call_oounicode " + pass + + def call_oounicode(self, OOTYPE): + """ Invoked for the oounicode opcode with the operand already + pushed onto the stack. prepare_call_oounicode() is invoked + before the operand is pushed. """ + raise NotImplementedError + def new(self, TYPE): """ Creates a new object of the given type. @@ -415,11 +444,14 @@ callee = op.args[0].value is_primitive = self._get_primitive_name(callee) + if is_primitive: + module, name = is_primitive + generator.prepare_call_primitive(op, module, name) + for arg in op.args[1:]: generator.load(arg) - + if is_primitive: - module, name = is_primitive generator.call_primitive(op, module, name) else: generator.call_graph(callee.graph) @@ -442,6 +474,7 @@ class _OOString(MicroInstruction): def render(self, generator, op): ARGTYPE = op.args[0].concretetype + generator.prepare_call_oostring(ARGTYPE) generator.load(op.args[0]) generator.load(op.args[1]) generator.call_oostring(ARGTYPE) @@ -452,6 +485,7 @@ assert v_base.value == -1, "The second argument of oounicode must be -1" ARGTYPE = op.args[0].concretetype + generator.prepare_call_oounicode(ARGTYPE) generator.load(op.args[0]) generator.call_oounicode(ARGTYPE) Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Thu Nov 29 23:20:37 2007 @@ -32,10 +32,28 @@ res = self.ll_to_string(self.interpret(fn, [False])) assert res == file(tmpfile, 'r').read() + def test_os_dup_oo(self): + tmpdir = str(udir.udir.join("os_dup_oo")) + def fn(): + fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) + os.write(fd, "hello world") + fd2 = os.dup(fd) + os.write(fd2, " (dupped)") + os.close(fd) + try: + os.write(fd2, " (uh oh)") + except OSError, e: + return e.errno + return -1 + assert self.interpret(fn, []) == 5 # EIO + assert file(tmpdir).read() == 'hello world (dupped)' + # the following tests can't be executed with gencli because they # returns file descriptors, and cli code is executed in another - # process. Instead of those, there is a new test that opens and + # process. Instead of those, there are new tests that opens and # write to a file all in the same process. + def test_os_dup(self): + pass def test_os_write(self): pass def test_os_write_single_char(self): Modified: pypy/dist/pypy/translator/oosupport/test_template/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/constant.py Thu Nov 29 23:20:37 2007 @@ -22,6 +22,14 @@ assert res.item0 == 1 assert res.item1 == 2 + def test_tuples_of_different_types(self): + const = "1", "2" + def fn(): + return const + res = self.interpret(fn, []) + assert res.item0 == "1" + assert res.item1 == "2" + def test_list(self): const = [1, 2] def fn(): From fijal at codespeak.net Fri Nov 30 10:43:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Nov 2007 10:43:53 +0100 (CET) Subject: [pypy-svn] r49219 - pypy/dist/pypy/doc Message-ID: <20071130094353.5B9CE820D@code0.codespeak.net> Author: fijal Date: Fri Nov 30 10:43:52 2007 New Revision: 49219 Modified: pypy/dist/pypy/doc/project-ideas.txt Log: Strike outdated part. Modified: pypy/dist/pypy/doc/project-ideas.txt ============================================================================== --- pypy/dist/pypy/doc/project-ideas.txt (original) +++ pypy/dist/pypy/doc/project-ideas.txt Fri Nov 30 10:43:52 2007 @@ -159,31 +159,6 @@ Additionally, these facilities could be exposed to applications interpreted by the translated PyPy interpreter. -Extension modules ------------------ - -Rewrite one or several CPython extension modules to be based on **ctypes** -(integrated in Python 2.5): this is generally useful for Python -developers, and it is now the best path to write extension modules that are -compatible with both CPython and PyPy. This is done with the `extension -compiler`_ component of PyPy, which will likely require some attention as -well. - -Modules where some work is already done: - -* ``_socket``, ``os``, ``select`` (unfinished yet, feel free to help; - see e.g. http://codespeak.net/svn/pypy/dist/pypy/module/_socket/). - -* SSL for socket, ``bz2``, ``fcntl``, ``mmap`` and ``time``: part of the - Summer of Code project of Lawrence Oluyede - (http://codespeak.net/svn/user/rhymes/). - -You are free to pick any other CPython module, either standard or third-party -(if relatively well-known, like gtk bindings). -Note that some modules exist in a ctypes version -already, which would be a good start for porting them to PyPy's extension -compiler. - Extend py.execnet to a peer-to-peer model ----------------------------------------- From fijal at codespeak.net Fri Nov 30 10:45:15 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Nov 2007 10:45:15 +0100 (CET) Subject: [pypy-svn] r49220 - pypy/dist/pypy/doc Message-ID: <20071130094515.BA019820D@code0.codespeak.net> Author: fijal Date: Fri Nov 30 10:45:15 2007 New Revision: 49220 Modified: pypy/dist/pypy/doc/_ref.txt Log: Kill outdated links. Modified: pypy/dist/pypy/doc/_ref.txt ============================================================================== --- pypy/dist/pypy/doc/_ref.txt (original) +++ pypy/dist/pypy/doc/_ref.txt Fri Nov 30 10:45:15 2007 @@ -67,7 +67,6 @@ .. _`pypy/module/readline`: ../../pypy/module/readline .. _`objspace/`: .. _`pypy/objspace`: ../../pypy/objspace -.. _`objspace/cpy/`: ../../pypy/objspace/cpy .. _`objspace/dump.py`: ../../pypy/objspace/dump.py .. _`objspace/flow/`: ../../pypy/objspace/flow .. _`objspace/std/`: @@ -80,8 +79,6 @@ .. _`pypy/rlib`: .. _`rlib/`: ../../pypy/rlib .. _`pypy/rlib/rarithmetic.py`: ../../pypy/rlib/rarithmetic.py -.. _`pypy/rlib/rctypes/rctypesobject.py`: ../../pypy/rlib/rctypes/rctypesobject.py -.. _`pypy/rlib/rctypes/test/test_rctypesobject.py`: ../../pypy/rlib/rctypes/test/test_rctypesobject.py .. _`pypy/rlib/test`: ../../pypy/rlib/test .. _`pypy/rpython`: .. _`pypy/rpython/`: @@ -96,7 +93,6 @@ .. _`pypy/rpython/module/test`: ../../pypy/rpython/module/test .. _`rpython/ootypesystem/`: ../../pypy/rpython/ootypesystem .. _`rpython/ootypesystem/ootype.py`: ../../pypy/rpython/ootypesystem/ootype.py -.. _`pypy/rpython/rctypes/test/test_ctypes.py`: ../../pypy/rpython/rctypes/test/test_ctypes.py .. _`rpython/rint.py`: ../../pypy/rpython/rint.py .. _`rpython/rlist.py`: ../../pypy/rpython/rlist.py .. _`rpython/rmodel.py`: ../../pypy/rpython/rmodel.py From fijal at codespeak.net Fri Nov 30 10:46:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Nov 2007 10:46:07 +0100 (CET) Subject: [pypy-svn] r49221 - pypy/dist/pypy/doc/config Message-ID: <20071130094607.D64C7820D@code0.codespeak.net> Author: fijal Date: Fri Nov 30 10:46:07 2007 New Revision: 49221 Modified: pypy/dist/pypy/doc/config/objspace.usemodules._demo.txt Log: Hum. Not sure whether this module makes sense any more. Modified: pypy/dist/pypy/doc/config/objspace.usemodules._demo.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules._demo.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules._demo.txt Fri Nov 30 10:46:07 2007 @@ -1,5 +1,3 @@ Use the '_demo' module. -This is the demo module for the `extension compiler`_. Not enabled by default. - -.. _`extension compiler`: ../extcompiler.html +This is the demo module for mixed modules. Not enabled by default. From niko at codespeak.net Fri Nov 30 15:21:13 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 30 Nov 2007 15:21:13 +0100 (CET) Subject: [pypy-svn] r49226 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20071130142113.C07258213@code0.codespeak.net> Author: niko Date: Fri Nov 30 15:21:07 2007 New Revision: 49226 Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Log: introduce a bunch of forgotten Math routines Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Fri Nov 30 15:21:07 2007 @@ -18,7 +18,7 @@ public void throwOSError(int errCode); public Object recordStringString(String a, String b); - public Object recordFloatFloat(String a, String b); - public Object recordFloatSigned(String a, String b); - public Object recordSignedSigned(String a, String b); + public Object recordFloatFloat(double a, double b); + public Object recordFloatSigned(double a, int b); + public Object recordSignedSigned(int a, int b); } Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Fri Nov 30 15:21:07 2007 @@ -957,21 +957,118 @@ // ---------------------------------------------------------------------- // ll_math + // + // not sure how many of these functions are needed. we should add + // something in the backend to redirect to the Math module by + // default, perhaps. - public double ll_math_floor(double x) - { + public double ll_math_ceil(double x) { + return Math.ceil(x); + } + + public double ll_math_fabs(double x) { + return Math.abs(x); + } + + public double ll_math_floor(double x) { return Math.floor(x); } - public double ll_math_fmod(double x, double y) - { + public double ll_math_fmod(double x, double y) { return x % y; } + /* TODO + public double ll_math_frexp(double x) { + } + */ + public double ll_math_ldexp(double v, int w) { return check(v * Math.pow(2.0, w)); } + public Object ll_math_modf(double x) { + if (x >= 0) { + double floor_x = Math.floor(x); + return interlink.recordFloatFloat(floor_x, x - floor_x); + } + + double ceil_x = Math.ceil(x); + return interlink.recordFloatFloat(ceil_x, x + ceil_x); + } + + public double ll_math_exp(double x) { + return Math.exp(x); + } + + public double ll_math_log(double x, double base) { + return Math.log10(x) / Math.log10(base); + } + + public double ll_math_log10(double v) { + return check(Math.log10(v)); + } + + public double ll_math_pow(double x, double y) { + return Math.pow(x, y); + } + + public double ll_math_sqrt(double x) { + return Math.sqrt(x); + } + + public double ll_math_acos(double x) { + return Math.acos(x); + } + + public double ll_math_asin(double x) { + return Math.asin(x); + } + + public double ll_math_atan(double x) { + return Math.atan(x); + } + + public double ll_math_atan2(double x, double y) { + return Math.atan2(x, y); + } + + public double ll_math_cos(double x) { + return Math.cos(x); + } + + public double ll_math_hypot(double x, double y) { + return Math.hypot(x, y); + } + + public double ll_math_sin(double x) { + return Math.sin(x); + } + + public double ll_math_tan(double x) { + return Math.tan(x); + } + + public double ll_math_degrees(double x) { + return Math.toDegrees(x); + } + + public double ll_math_radians(double x) { + return Math.toRadians(x); + } + + public double ll_math_cosh(double x) { + return Math.cosh(x); + } + + public double ll_math_sinh(double x) { + return Math.sinh(x); + } + + public double ll_math_tanh(double x) { + return Math.tanh(x); + } + private double check(double v) { if (Double.isNaN(v)) interlink.throwValueError(); From arigo at codespeak.net Fri Nov 30 16:04:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 16:04:05 +0100 (CET) Subject: [pypy-svn] r49227 - in pypy/dist/pypy/objspace/flow: . test Message-ID: <20071130150405.3D67B8210@code0.codespeak.net> Author: arigo Date: Fri Nov 30 16:04:03 2007 New Revision: 49227 Modified: pypy/dist/pypy/objspace/flow/flowcontext.py pypy/dist/pypy/objspace/flow/test/test_objspace.py Log: In the flow objspace: don't freeze into the graph a RuntimeError caught while building the graph. Modified: pypy/dist/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/dist/pypy/objspace/flow/flowcontext.py (original) +++ pypy/dist/pypy/objspace/flow/flowcontext.py Fri Nov 30 16:04:03 2007 @@ -309,6 +309,9 @@ e.w_type is self.space.w_ImportError): raise ImportError('import statement always raises %s' % ( e,)) + if e.w_type is self.space.w_RuntimeError: + raise RuntimeError('during flow graph construction: %r' % ( + e.w_value,)) link = self.make_link([e.w_type, e.w_value], self.graph.exceptblock) self.recorder.crnt_block.closeblock(link) Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Fri Nov 30 16:04:03 2007 @@ -847,6 +847,15 @@ py.test.raises(TypeError, "self.codetest(f)") + def test_dont_capture_RuntimeError(self): + class Foo: + def __hash__(self): + return hash(self) + foolist = [Foo()] + def f(): + return foolist[0] + py.test.raises(RuntimeError, "self.codetest(f)") + class TestFlowObjSpaceDelay(Base): def setup_class(cls): cls.space = FlowObjSpace() From pypy-svn at codespeak.net Fri Nov 30 17:33:41 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Fri, 30 Nov 2007 17:33:41 +0100 (CET) Subject: [pypy-svn] November 78% OFF Message-ID: <20071130065722.14588.qmail@ip-53.net-89-2-150.rev.numericable.fr> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Fri Nov 30 17:49:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 17:49:47 +0100 (CET) Subject: [pypy-svn] r49229 - pypy/dist/pypy/annotation Message-ID: <20071130164947.5591C823E@code0.codespeak.net> Author: arigo Date: Fri Nov 30 17:49:47 2007 New Revision: 49229 Modified: pypy/dist/pypy/annotation/bookkeeper.py Log: Kill unused imports. Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Fri Nov 30 17:49:47 2007 @@ -19,8 +19,6 @@ from pypy.annotation import description from pypy.annotation.signature import annotationoftype from pypy.interpreter.argument import Arguments, ArgErr -from pypy.rlib.rarithmetic import r_int, r_uint, r_ulonglong, r_longlong -from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import r_dict, Symbolic from pypy.tool.algo.unionfind import UnionFind from pypy.rpython.lltypesystem import lltype, llmemory From arigo at codespeak.net Fri Nov 30 18:19:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 18:19:12 +0100 (CET) Subject: [pypy-svn] r49230 - pypy/dist/pypy/module/readline Message-ID: <20071130171912.641908234@code0.codespeak.net> Author: arigo Date: Fri Nov 30 18:19:10 2007 New Revision: 49230 Modified: pypy/dist/pypy/module/readline/c_readline.py Log: At least on Gentoo Linux, readline.h doesn't compile if stdio.h is not included before Modified: pypy/dist/pypy/module/readline/c_readline.py ============================================================================== --- pypy/dist/pypy/module/readline/c_readline.py (original) +++ pypy/dist/pypy/module/readline/c_readline.py Fri Nov 30 18:19:10 2007 @@ -4,8 +4,10 @@ from pypy.interpreter.gateway import ObjSpace, interp2app from pypy.translator.tool.cbuild import ExternalCompilationInfo +# at least on Gentoo Linux, readline.h doesn't compile if stdio.h is not +# included before eci = ExternalCompilationInfo( - includes = ["readline/readline.h", "readline/history.h"], + includes = ["stdio.h", "readline/readline.h", "readline/history.h"], libraries = ['readline'] ) From arigo at codespeak.net Fri Nov 30 18:25:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 18:25:15 +0100 (CET) Subject: [pypy-svn] r49231 - pypy/dist/pypy/rpython/test Message-ID: <20071130172515.280E58233@code0.codespeak.net> Author: arigo Date: Fri Nov 30 18:25:14 2007 New Revision: 49231 Modified: pypy/dist/pypy/rpython/test/test_rtyper.py Log: This was killed. Modified: pypy/dist/pypy/rpython/test/test_rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rtyper.py (original) +++ pypy/dist/pypy/rpython/test/test_rtyper.py Fri Nov 30 18:25:14 2007 @@ -144,9 +144,6 @@ assert rmodel.getgcflavor(DummyClsDescDef(A)) == 'gc' assert rmodel.getgcflavor(DummyClsDescDef(B)) == 'gc' assert rmodel.getgcflavor(DummyClsDescDef(R)) == 'raw' - cdef = DummyClsDescDef(A) - cdef._cpy_exported_type_ = type(Ellipsis) - assert rmodel.getgcflavor(cdef) == 'cpy' def test_missing_gvflavor_bug(): class MyClass: From arigo at codespeak.net Fri Nov 30 19:20:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 19:20:54 +0100 (CET) Subject: [pypy-svn] r49232 - in pypy/dist/pypy: annotation objspace/std rlib rpython rpython/lltypesystem rpython/memory/test rpython/ootypesystem rpython/ootypesystem/test rpython/test Message-ID: <20071130182054.C1AF581A5@code0.codespeak.net> Author: arigo Date: Fri Nov 30 19:20:51 2007 New Revision: 49232 Modified: pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/objspace/std/default.py pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py pypy/dist/pypy/rpython/ootypesystem/rclass.py pypy/dist/pypy/rpython/ootypesystem/test/test_ooclean.py pypy/dist/pypy/rpython/rmodel.py pypy/dist/pypy/rpython/rtuple.py pypy/dist/pypy/rpython/test/test_llinterp.py pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/rpython/test/test_rclass.py Log: issue325 testing It's too easy to use id() in RPython and forget that it's actually a pretty advanced operation on top of moving GCs. This replaces it with two functions in the objectmodel module, one which implements the full semantics of id() and another which just returns the current address as an integer. Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Fri Nov 30 19:20:51 2007 @@ -119,8 +119,11 @@ getbookkeeper().count('oct', obj) return SomeString() - def id(obj): # xxx - return SomeInteger() + def id(obj): + raise Exception("cannot use id() in RPython; pick one of:\n" + "\t\t objectmodel.compute_unique_id()\n" + "\t\t hash()\n" + "\t\t objectmodel.current_object_addr_as_int()") def int(obj): return SomeInteger() Modified: pypy/dist/pypy/objspace/std/default.py ============================================================================== --- pypy/dist/pypy/objspace/std/default.py (original) +++ pypy/dist/pypy/objspace/std/default.py Fri Nov 30 19:20:51 2007 @@ -1,6 +1,7 @@ """Default implementation for some operation.""" from pypy.objspace.std.objspace import * +from pypy.rlib import objectmodel # The following default implementations are used before delegation is tried. @@ -8,7 +9,7 @@ def id__ANY(space, w_obj): #print 'id:', w_obj - return space.wrap(id(w_obj)) + return space.wrap(objectmodel.compute_unique_id(w_obj)) # __init__ should succeed if called internally as a multimethod Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Fri Nov 30 19:20:51 2007 @@ -128,6 +128,80 @@ from pypy.rpython.extregistry import ExtRegistryEntry # ____________________________________________________________ +# +# id-like functions. +# In addition, RPython supports hash(x) on RPython instances, +# returning a number that is not guaranteed to be unique but +# that doesn't change over time for a given 'x'. + +def compute_unique_id(x): + """RPython equivalent of id(x). The 'x' must be an RPython instance. + This operation can be very costly depending on the garbage collector. + To remind you of this fact, we don't support id(x) directly. + """ + return id(x) # XXX need to return r_longlong on some platforms + +def current_object_addr_as_int(x): + """A cheap version of id(x). The current memory location of an + instance can change over time for moving GCs. Also note that on + ootypesystem this typically doesn't return the real address but + just the same as hash(x). + """ + from pypy.rlib.rarithmetic import intmask + return intmask(id(x)) + +class Entry(ExtRegistryEntry): + _about_ = compute_unique_id + + def compute_result_annotation(self, s_x): + from pypy.annotation import model as annmodel + assert isinstance(s_x, annmodel.SomeInstance) + return annmodel.SomeInteger() + + def specialize_call(self, hop): + vobj, = hop.inputargs(hop.args_r[0]) + if hop.rtyper.type_system.name == 'lltypesystem': + from pypy.rpython.lltypesystem import lltype + if isinstance(vobj.concretetype, lltype.Ptr): + return hop.genop('gc_id', [vobj], + resulttype = lltype.Signed) + elif hop.rtyper.type_system.name == 'ootypesystem': + from pypy.rpython.ootypesystem import ootype + if isinstance(vobj.concretetype, ootype.Instance): + # XXX wrong implementation for now, fix me + from pypy.rpython.rmodel import warning + warning("compute_unique_id() is not fully supported on ootype") + return hop.genop('ooidentityhash', [vobj], + resulttype = ootype.Signed) + from pypy.rpython.error import TyperError + raise TyperError("compute_unique_id() cannot be applied to %r" % ( + vobj.concretetype,)) + +class Entry(ExtRegistryEntry): + _about_ = current_object_addr_as_int + + def compute_result_annotation(self, s_x): + from pypy.annotation import model as annmodel + assert isinstance(s_x, annmodel.SomeInstance) + return annmodel.SomeInteger() + + def specialize_call(self, hop): + vobj, = hop.inputargs(hop.args_r[0]) + if hop.rtyper.type_system.name == 'lltypesystem': + from pypy.rpython.lltypesystem import lltype + if isinstance(vobj.concretetype, lltype.Ptr): + return hop.genop('cast_ptr_to_int', [vobj], + resulttype = lltype.Signed) + elif hop.rtyper.type_system.name == 'ootypesystem': + from pypy.rpython.ootypesystem import ootype + if isinstance(vobj.concretetype, ootype.Instance): + return hop.genop('ooidentityhash', [vobj], + resulttype = ootype.Signed) + from pypy.rpython.error import TyperError + raise TyperError("current_object_addr_as_int() cannot be applied to" + " %r" % (vobj.concretetype,)) + +# ____________________________________________________________ def debug_assert(x, msg): """After translation to C, this becomes an RPyAssert.""" Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Fri Nov 30 19:20:51 2007 @@ -471,7 +471,7 @@ 'instanceof': LLOp(oo=True, canfold=True), 'classof': LLOp(oo=True, canfold=True), 'subclassof': LLOp(oo=True, canfold=True), - 'ooidentityhash': LLOp(oo=True, sideeffects=False), + 'ooidentityhash': LLOp(oo=True, sideeffects=False), # not an id()! 'oostring': LLOp(oo=True, sideeffects=False), 'ooparse_int': LLOp(oo=True, canraise=(ValueError,)), 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Fri Nov 30 19:20:51 2007 @@ -701,7 +701,7 @@ return 0 # for None cached = ins.hash_cache if cached == 0: - cached = ins.hash_cache = intmask(id(ins)) # XXX issue325 + cached = ins.hash_cache = cast_ptr_to_int(ins) return cached def ll_inst_type(obj): Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Fri Nov 30 19:20:51 2007 @@ -7,6 +7,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import compute_unique_id def stdout_ignore_ll_functions(msg): @@ -262,14 +263,14 @@ def f(): a2 = A() a3 = A() - id1 = id(a1) - id2 = id(a2) - id3 = id(a3) + id1 = compute_unique_id(a1) + id2 = compute_unique_id(a2) + id3 = compute_unique_id(a3) llop.gc__collect(lltype.Void) error = 0 - if id1 != id(a1): error += 1 - if id2 != id(a2): error += 2 - if id3 != id(a3): error += 4 + if id1 != compute_unique_id(a1): error += 1 + if id2 != compute_unique_id(a2): error += 2 + if id3 != compute_unique_id(a3): error += 4 return error res = self.interpret(f, []) assert res == 0 Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Fri Nov 30 19:20:51 2007 @@ -8,6 +8,7 @@ from pypy.rpython.memory.gctransform import stacklessframework from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR +from pypy.rlib.objectmodel import compute_unique_id from pypy import conftest INT_SIZE = struct.calcsize("i") # only for estimates @@ -428,14 +429,14 @@ def func(): a2 = A() a3 = A() - id1 = id(a1) - id2 = id(a2) - id3 = id(a3) + id1 = compute_unique_id(a1) + id2 = compute_unique_id(a2) + id3 = compute_unique_id(a3) llop.gc__collect(lltype.Void) error = 0 - if id1 != id(a1): error += 1 - if id2 != id(a2): error += 2 - if id3 != id(a3): error += 4 + if id1 != compute_unique_id(a1): error += 1 + if id2 != compute_unique_id(a2): error += 2 + if id3 != compute_unique_id(a3): error += 4 return error run = self.runner(func) res = run([]) @@ -456,7 +457,7 @@ # remember the ids, it will trigger some collections itself i = 0 while i < len(alist): - idarray[i] = id(alist[i]) + idarray[i] = compute_unique_id(alist[i]) i += 1 j = 0 while j < 2: @@ -464,7 +465,7 @@ [A() for i in range(20)] i = 0 while i < len(alist): - assert idarray[i] == id(alist[i]) + assert idarray[i] == compute_unique_id(alist[i]) i += 1 j += 1 lltype.free(idarray, flavor='raw') Modified: pypy/dist/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rclass.py Fri Nov 30 19:20:51 2007 @@ -449,10 +449,6 @@ cmeta = inputconst(ootype.Void, "meta") return hop.genop('oogetfield', [vinst, cmeta], resulttype=CLASSTYPE) - def rtype_id(self, hop): - vinst, = hop.inputargs(self) - return hop.genop('ooidentityhash', [vinst], resulttype=ootype.Signed) - def null_instance(self): return ootype.null(self.lowleveltype) Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_ooclean.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_ooclean.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_ooclean.py Fri Nov 30 19:20:51 2007 @@ -437,6 +437,7 @@ assert res == ~0x0200 & 0x3ff def test_hash_preservation(): + from pypy.rlib.objectmodel import current_object_addr_as_int class C: pass class D(C): @@ -445,7 +446,7 @@ d2 = D() # xxx we assume that the identityhash doesn't change from # one line to the next - current_identityhash = id(d2) + current_identityhash = current_object_addr_as_int(d2) instance_hash = hash(d2) return ((current_identityhash & sys.maxint) == (instance_hash & sys.maxint)) Modified: pypy/dist/pypy/rpython/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rmodel.py (original) +++ pypy/dist/pypy/rpython/rmodel.py Fri Nov 30 19:20:51 2007 @@ -215,13 +215,6 @@ else: return hop.genop('int_is_true', [vlen], resulttype=Bool) - def rtype_id(self, hop): - if not isinstance(self.lowleveltype, Ptr): - raise TyperError('id() of an instance of the non-pointer %r' % ( - self,)) - vobj, = hop.inputargs(self) - return hop.genop('gc_id', [vobj], resulttype=Signed) - def rtype_hash(self, hop): ll_hash = self.get_ll_hash_function() v, = hop.inputargs(self) Modified: pypy/dist/pypy/rpython/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/rtuple.py (original) +++ pypy/dist/pypy/rpython/rtuple.py Fri Nov 30 19:20:51 2007 @@ -204,9 +204,6 @@ def rtype_len(self, hop): return hop.inputconst(Signed, len(self.items_r)) - def rtype_id(self, hop): - raise TyperError("cannot ask for the id() of a tuple") - def get_ll_eq_function(self): return gen_eq_function(self.items_r) Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Fri Nov 30 19:20:51 2007 @@ -409,12 +409,13 @@ assert res == 5 def test_id(): + from pypy.rlib.objectmodel import compute_unique_id def getids(i, j): e1 = ExampleClass(1) e2 = ExampleClass(2) a = [e1, e2][i] b = [e1, e2][j] - return (id(a) == id(b)) == (a is b) + return (compute_unique_id(a) == compute_unique_id(b)) == (a is b) for i in [0, 1]: for j in [0, 1]: result = interpret(getids, [i, j]) Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Fri Nov 30 19:20:51 2007 @@ -4,7 +4,7 @@ from pypy.rlib.objectmodel import running_on_llinterp, debug_llinterpcall from pypy.rpython.lltypesystem import lltype from pypy.tool import udir -from pypy.rlib.rarithmetic import r_uint, intmask +from pypy.rlib.rarithmetic import r_uint, intmask, r_longlong from pypy.annotation.builtin import * from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.lltypesystem.rffi import SHORT @@ -429,6 +429,26 @@ res = f(7) assert res == 321 + def test_id(self): + from pypy.rlib.objectmodel import compute_unique_id + from pypy.rlib.objectmodel import current_object_addr_as_int + class A: + pass + def fn(): + a1 = A() + a2 = A() + return (compute_unique_id(a1), current_object_addr_as_int(a1), + compute_unique_id(a2), current_object_addr_as_int(a2)) + res = self.interpret(fn, []) + x0, x1, x2, x3 = self.ll_unpack_tuple(res, 4) + assert isinstance(x0, (int, r_longlong)) + assert isinstance(x1, int) + assert isinstance(x2, (int, r_longlong)) + assert isinstance(x3, int) + assert x1 == intmask(x0) # at least on top of llinterp + assert x3 == intmask(x2) # at least on top of llinterp + assert x0 != x2 + class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): def test_isinstance_obj(self): Modified: pypy/dist/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rclass.py (original) +++ pypy/dist/pypy/rpython/test/test_rclass.py Fri Nov 30 19:20:51 2007 @@ -390,6 +390,7 @@ assert res == ~0x0200 & 0x3ff def test_hash_preservation(self): + from pypy.rlib.objectmodel import current_object_addr_as_int class C: pass class D(C): @@ -399,7 +400,9 @@ def f(): d2 = D() # xxx check for this CPython peculiarity for now: - x = (hash(d2) & sys.maxint) == (id(d2) & sys.maxint) + # (this is true on top of the llinterp too) + x = ((hash(d2) & sys.maxint) == + (current_object_addr_as_int(d2) & sys.maxint)) return x, hash(c)+hash(d) res = self.interpret(f, []) @@ -688,11 +691,14 @@ assert summary(graph) == {"setfield": 2} def test_instance_repr(self): + from pypy.rlib.objectmodel import current_object_addr_as_int class FooBar(object): pass def f(): x = FooBar() - return id(x), str(x) + # on lltype, the RPython-level repr of an instance contains the + # current object address + return current_object_addr_as_int(x), str(x) res = self.interpret(f, []) xid, xstr = self.ll_unpack_tuple(res, 2) From arigo at codespeak.net Fri Nov 30 19:50:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Nov 2007 19:50:10 +0100 (CET) Subject: [pypy-svn] r49233 - pypy/dist/pypy/objspace/std Message-ID: <20071130185010.7207081A6@code0.codespeak.net> Author: arigo Date: Fri Nov 30 19:50:10 2007 New Revision: 49233 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/typeobject.py Log: Forgotten two more places using id(). Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Fri Nov 30 19:50:10 2007 @@ -1011,8 +1011,10 @@ continue bodySrc.append(tmpl%locals()) exec py.code.Source(''' +from pypy.rlib.objectmodel import current_object_addr_as_int def _report_one(fd, info): - os.write(fd, "_address" + ": " + str(id(info)) + "\\n") + os.write(fd, "_address" + ": " + str(current_object_addr_as_int(info)) + + "\\n") %s '''%'\n '.join(bodySrc)).compile() Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Fri Nov 30 19:50:10 2007 @@ -7,6 +7,7 @@ from pypy.objspace.std.objecttype import object_typedef from pypy.objspace.std.dictproxyobject import W_DictProxyObject from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import current_object_addr_as_int from pypy.rlib.jit import hint from pypy.rlib.rarithmetic import intmask, r_uint @@ -335,7 +336,14 @@ tup = w_self._lookup_where(name) return tup SHIFT = r_uint.BITS - space.config.objspace.std.methodcachesizeexp - method_hash = r_uint(intmask(id(version_tag) * hash(name))) >> SHIFT + version_tag_as_int = current_object_addr_as_int(version_tag) + # ^^^Note: if the version_tag object is moved by a moving GC, the + # existing method cache entries won't be found any more; new + # entries will be created based on the new address. The + # assumption is that the version_tag object won't keep moving all + # the time - so using the fast current_object_addr_as_int() instead + # of a slower solution like hash() is still a good trade-off. + method_hash = r_uint(intmask(version_tag_as_int * hash(name))) >> SHIFT cached_version_tag = ec.method_cache_versions[method_hash] if cached_version_tag is version_tag: cached_name = ec.method_cache_names[method_hash]