From mwh at codespeak.net Tue Aug 1 12:24:02 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 1 Aug 2006 12:24:02 +0200 (CEST) Subject: [pypy-svn] r30833 - pypy/dist/pypy/doc Message-ID: <20060801102402.6F28E10077@code0.codespeak.net> Author: mwh Date: Tue Aug 1 12:24:00 2006 New Revision: 30833 Modified: pypy/dist/pypy/doc/translation.txt Log: very small wording tweak Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Tue Aug 1 12:24:00 2006 @@ -49,9 +49,10 @@ It is helpful to consider translation as being made up of the following steps: -1. The complete program is imported, arbitrary run-time initialization can - be performed. Once this is done, the program must be present in memory - as a form that is "static enough" in the sense of RPython_. +1. The complete program is imported, at which time arbitrary run-time + initialization can be performed. Once this is done, the program must + be present in memory as a form that is "static enough" in the sense of + RPython_. 2. The Annotator_ performs a global analysis starting from an specified entry point to deduce type and other information about what each From arigo at codespeak.net Tue Aug 1 13:34:10 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Aug 2006 13:34:10 +0200 (CEST) Subject: [pypy-svn] r30838 - pypy/dist/pypy/interpreter/test Message-ID: <20060801113410.559C610078@code0.codespeak.net> Author: arigo Date: Tue Aug 1 13:34:08 2006 New Revision: 30838 Modified: pypy/dist/pypy/interpreter/test/test_gateway.py Log: Remove dead code in test file. Modified: pypy/dist/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_gateway.py (original) +++ pypy/dist/pypy/interpreter/test/test_gateway.py Tue Aug 1 13:34:08 2006 @@ -233,9 +233,3 @@ assert space.eq_w(space.call_function(w_app_g_id,w("foo")),w("foo")) assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - - - - -def app_g3(b): - return 'foo'+b From auc at codespeak.net Tue Aug 1 13:54:20 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 1 Aug 2006 13:54:20 +0200 (CEST) Subject: [pypy-svn] r30840 - in pypy/dist/pypy/objspace: . cclp test Message-ID: <20060801115420.8E43D10078@code0.codespeak.net> Author: auc Date: Tue Aug 1 13:54:16 2006 New Revision: 30840 Modified: pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: misc changes, transl. fixes, better thread accounting Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 1 13:54:16 2006 @@ -24,6 +24,14 @@ self._traced = {} w (".. MAIN THREAD = ", str(id(self._main))) + def get_threads(self): + threads = [self._head] + curr = self._head.next + while curr != self._head: + threads.append(curr) + curr = curr.next + return threads + def _init_blocked(self): self._blocked = {} # thread set self._blocked_on = {} # var -> threads @@ -88,6 +96,7 @@ to_be_run = self._select_next() w(".. SWITCHING", str(id(ClonableCoroutine.w_getcurrent(self.space))), "=>", str(id(to_be_run))) self._switch_count += 1 + assert isinstance(to_be_run, ClonableCoroutine) to_be_run.w_switch() def schedule_or_pass(self): @@ -98,6 +107,7 @@ return w(".. SWITCHING", str(id(curr)), "=>", str(id(to_be_run))) self._switch_count += 1 + assert isinstance(to_be_run, ClonableCoroutine) to_be_run.w_switch() def _select_next(self, dont_pass=True): @@ -213,16 +223,20 @@ scheduler[0] = Scheduler(space) app_reset_scheduler = gateway.interp2app(reset_scheduler) -def sched_stats(space): +def sched_info(space): sched = scheduler[0] w_ret = space.newdict([]) space.setitem(w_ret, space.wrap('switches'), space.wrap(sched._switch_count)) - space.setitem(w_ret, space.wrap('threads'), space.wrap(sched.__len__())) - space.setitem(w_ret, space.wrap('blocked'), space.wrap(len(sched._blocked))) - space.setitem(w_ret, space.wrap('blocked_on'), space.wrap(len(sched._blocked_on))) - space.setitem(w_ret, space.wrap('blocked_byneed'), space.wrap(len(sched._blocked_byneed))) + space.setitem(w_ret, space.wrap('threads'), + space.wrap([id(th) for th in sched.get_threads()])) + space.setitem(w_ret, space.wrap('blocked'), + space.wrap([id(th) for th in sched._blocked.keys()])) + space.setitem(w_ret, space.wrap('blocked_on'), + space.wrap([id(th) for th in sched._blocked_on.keys()])) + space.setitem(w_ret, space.wrap('blocked_byneed'), + space.wrap([id(th) for th in sched._blocked_byneed.keys()])) return w_ret -app_sched_stats = gateway.interp2app(sched_stats) +app_sched_info = gateway.interp2app(sched_info) def schedule(space): Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Tue Aug 1 13:54:16 2006 @@ -1,18 +1,28 @@ +from pypy.interpreter import baseobjspace from pypy.interpreter.error import OperationError + from pypy.objspace.cclp.misc import ClonableCoroutine -class CSpace: +## def newspace(w_callable, __args__): +## w_coro = stacklet(w_callable, __args__) +## w_space = CSpace(coro, parent=coro.space) +## w_coro.space = space +## return w_space + +class CSpace(baseobjspace.Wrappable): def __init__(self, space, distributor, parent=None): - assert isinstance(distributor, ClonableCoroutine) - assert (parent is None) or isinstance(parent, CSpace) - self.space = space # the object space ;-) - self.parent = parent - self.distributor = distributor - self.threads = {} # the eventual other threads + pass +## assert isinstance(distributor, ClonableCoroutine) +## assert (parent is None) or isinstance(parent, CSpace) +## self.space = space # the object space ;-) +## self.parent = parent +## self.distributor = distributor +## self.threads = {} # the eventual other threads + - def is_top_level(self): - return self.parent is None +## def is_top_level(self): +## return self.parent is None ## def current_space(): ## #XXX return w_getcurrent().cspace @@ -22,16 +32,16 @@ ## #XXX fork ? ## pass - def clone(self): - if self.is_top_level(): - raise OperationError(self.space.w_RuntimeError, - self.space.wrap("Clone"+forbidden_boilerplate)) - new = CSpace(self.distributor.clone(), parent=self) - new.distributor.cspace = new - for thread in self.threads: - tclone = thread.clone() - tclone.cspace = new - new.threads[tclone] = True +## def clone(self): +## if self.is_top_level(): +## raise OperationError(self.space.w_RuntimeError, +## self.space.wrap("Clone"+forbidden_boilerplate)) +## new = CSpace(self.distributor.clone(), parent=self) +## new.distributor.cspace = new +## for thread in self.threads: +## tclone = thread.clone() +## tclone.cspace = new +## new.threads[tclone] = True ## def choose(self, n): ## if self.is_top_level(): Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Tue Aug 1 13:54:16 2006 @@ -1,5 +1,4 @@ from pypy.interpreter import gateway, baseobjspace, argument -from pypy.interpreter.error import OperationError from pypy.rpython.objectmodel import we_are_translated from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue @@ -24,7 +23,7 @@ w_Future = W_Future(space) thunk = FutureThunk(space, w_callable, args, w_Future, coro) coro.bind(thunk) - w("THREAD", str(id(coro))) + w("FUTURE", str(id(coro))) scheduler[0].add_new_thread(coro) return w_Future app_future = gateway.interp2app(future, unwrap_spec=[baseobjspace.ObjSpace, @@ -44,7 +43,7 @@ coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) coro.bind(thunk) - w("THREAD", str(id(coro))) + w("STACKLET", str(id(coro))) scheduler[0].add_new_thread(coro) scheduler[0].schedule() return coro Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Tue Aug 1 13:54:16 2006 @@ -6,7 +6,7 @@ #-- Types -------------------------------------------------- -class W_Var(W_Root, object): +class W_Var(W_Root): def __init__(w_self, space): # ring of aliases or bound value w_self.w_bound_to = w_self Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Tue Aug 1 13:54:16 2006 @@ -16,7 +16,7 @@ from pypy.objspace.cclp.thread import app_future, app_stacklet, app_this_thread -from pypy.objspace.cclp.scheduler import Scheduler, scheduler, app_sched_stats, \ +from pypy.objspace.cclp.scheduler import Scheduler, scheduler, app_sched_info, \ app_schedule, app_reset_scheduler #-- VARIABLE ------------------------------------------------ @@ -254,8 +254,8 @@ space.wrap(app_wait)) space.setitem(space.builtin.w_dict, space.wrap('wait_needed'), space.wrap(app_wait_needed)) - space.setitem(space.builtin.w_dict, space.wrap('sched_stats'), - space.wrap(app_sched_stats)) + space.setitem(space.builtin.w_dict, space.wrap('sched_info'), + space.wrap(app_sched_info)) space.setitem(space.builtin.w_dict, space.wrap('schedule'), space.wrap(app_schedule)) space.setitem(space.builtin.w_dict, space.wrap('this_thread'), Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Tue Aug 1 13:54:16 2006 @@ -202,7 +202,7 @@ cls.space = gettestobjspace('logic', usemodules=("_stackless",)) def test_future_value(self): - print "future value", sched_stats() + print "future value", sched_info() def poop(X): return X + 1 @@ -211,18 +211,16 @@ Y = future(poop, X) unify(X, 42) assert Y == 43 - assert sched_stats()['threads'] == 1 X = newvar() T = future(poop, X) raises(Exception, unify, T, 42) - print sched_stats() - assert sched_stats()['threads'] == 2 + bind(X, 42); schedule() # helps the gc X, Y = newvar(), newvar() T = future(poop, X) raises(Exception, unify, T, Y) - assert sched_stats()['threads'] == 3 + bind(X, 42); schedule() # gc ... assert is_free(Y) X = newvar() @@ -230,10 +228,10 @@ unify(Y, T) unify(X, 42) assert Y == 43 - assert sched_stats()['threads'] == 3 + bind(X, 42); schedule() def test_one_future_exception(self): - print "one future exception", sched_stats() + print "one future exception", sched_info() class FooException(Exception): pass def poop(X): @@ -250,7 +248,7 @@ assert False def test_exception_in_chain(self): - print "exception in chain", sched_stats() + print "exception in chain", sched_info() class FooException(Exception): pass def raise_foo(): @@ -274,7 +272,7 @@ assert False def test_exception_in_group(self): - print "exception in groups", sched_stats() + print "exception in groups", sched_info() class FooException(Exception): pass def loop_or_raise(Canary, crit, Bomb_signal): @@ -306,7 +304,7 @@ """check that a wait nested in a tree of threads works correctly """ - print "nested threads", sched_stats() + print "nested threads", sched_info() def sleep(X): wait(X) return X @@ -322,7 +320,7 @@ assert v == 42 def test_wait_needed(self): - print "wait_needed", sched_stats() + print "wait_needed", sched_info() X = newvar() def binder(V): @@ -337,9 +335,10 @@ future(binder, X) assert X == 42 + schedule() # gc help def test_eager_producer_consummer(self): - print "eager_producer_consummer", sched_stats() + print "eager_producer_consummer", sched_info() def generate(n, limit): if n < limit: @@ -361,10 +360,11 @@ def test_lazy_producer_consummer(self): - print "lazy_producer_consummer", sched_stats() + print "lazy_producer_consummer", sched_info() def lgenerate(n, L): """wait-needed version of generate""" + #XXX how is this ever collected ? wait_needed(L) Tail = newvar() bind(L, (n, Tail)) @@ -388,9 +388,10 @@ wait(T) assert T == 45 + assert len(sched_info()['blocked_byneed']) == 1 + reset_scheduler() def test_wait_two(self): - print "wait_two", sched_stats() def sleep(X, Barrier): wait(X) @@ -411,6 +412,8 @@ unify(Y, 42) assert X == Y == 42 assert o == 2 + schedule() # give a chance to the second thread to exit + assert len(sched_info()['threads']) == 1 def test_fib(self): skip("recursion limits breakage") @@ -436,12 +439,7 @@ def test_stacklet(self): - print "stacklet", sched_stats() reset_scheduler() - #XXX each of the previous test decorates - # the scheduler with unreclaimed stuff - # In this case, side-effect happen. Nasty. - count = [0] def inc_and_greet(count, max_, Finished, Failed): @@ -462,7 +460,7 @@ try: wait(Failed) except Exception, e: # Unification Failure - assert sched_stats()['threads'] == 1 + assert len(sched_info()['threads']) == 1 return assert False From arigo at codespeak.net Tue Aug 1 13:56:00 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Aug 2006 13:56:00 +0200 (CEST) Subject: [pypy-svn] r30841 - in pypy/dist/pypy/interpreter: . test Message-ID: <20060801115600.9301D1007B@code0.codespeak.net> Author: arigo Date: Tue Aug 1 13:55:59 2006 New Revision: 30841 Modified: pypy/dist/pypy/interpreter/argument.py pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/test/test_gateway.py Log: To allow *args, **kwargs in app2interp-ed functions, the interp-level gateway hook function must accept a general Arguments object as its last argument. (This will be necessary for dict.update(), but so far it's not used, so I can only say that it *should* annotate properly.) Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Tue Aug 1 13:55:59 2006 @@ -3,7 +3,6 @@ """ from pypy.interpreter.error import OperationError -import os class AbstractArguments: Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Tue Aug 1 13:55:59 2006 @@ -17,7 +17,7 @@ from pypy.interpreter.function import Function, Method from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache -from pypy.interpreter.argument import Arguments +from pypy.interpreter.argument import Arguments, AbstractArguments from pypy.tool.sourcetools import NiceCompile, compile2 # internal non-translatable parts: @@ -788,7 +788,16 @@ ret_w = sc[ApplevelClass](space, self, name, args_w) if ret_w is not None: # it was RPython return ret_w - args = Arguments(space, list(args_w)) + # the last argument can be an Arguments + if args_w and isinstance(args_w[-1], AbstractArguments): + # ...which is merged with the previous arguments, if any + args = args_w[-1] + if len(args_w) > 1: + more_args_w, more_kwds_w = args.unpack() + args = Arguments(space, list(args_w[:-1]) + more_args_w, + more_kwds_w) + else: + args = Arguments(space, list(args_w)) w_func = self.wget(space, name) return space.call_args(w_func, args) def get_function(space): Modified: pypy/dist/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_gateway.py (original) +++ pypy/dist/pypy/interpreter/test/test_gateway.py Tue Aug 1 13:55:59 2006 @@ -91,7 +91,20 @@ return a+b g3 = gateway.app2interp_temp(noapp_g3, gateway.applevelinterp_temp) assert self.space.eq_w(g3(self.space, w('foo'), w('bar')), w('foobar')) - + + def test_app2interp_general_args(self): + w = self.space.wrap + def app_general(x, *args, **kwds): + assert type(args) is tuple + assert type(kwds) is dict + return x + 10 * len(args) + 100 * len(kwds) + gg = gateway.app2interp_temp(app_general) + args = gateway.Arguments(self.space, [w(6), w(7)]) + assert self.space.int_w(gg(self.space, w(3), args)) == 23 + args = gateway.Arguments(self.space, [w(6)], {'hello': w(7), + 'world': w(8)}) + assert self.space.int_w(gg(self.space, w(3), args)) == 213 + def test_interp2app(self): space = self.space w = space.wrap From antocuni at codespeak.net Tue Aug 1 14:06:49 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Aug 2006 14:06:49 +0200 (CEST) Subject: [pypy-svn] r30843 - in pypy/dist/pypy/translator/cli: . src src/stub test Message-ID: <20060801120649.71E5C1007A@code0.codespeak.net> Author: antocuni Date: Tue Aug 1 14:06:42 2006 New Revision: 30843 Modified: pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/cli/src/ll_os.cs pypy/dist/pypy/translator/cli/src/stub/main.il pypy/dist/pypy/translator/cli/test/test_exception.py Log: Don't longer ignore last_exception when catching an exception. Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Tue Aug 1 14:06:42 2006 @@ -125,7 +125,10 @@ self.store(link.target.inputargs[1]) else: # the exception value is on the stack, store it in the proper place + self.ilasm.opcode('dup') self.store(link.last_exc_value) + self.ilasm.get_field(('Object_meta', 'Object', 'meta')) + self.store(link.last_exception) self._setup_link(link) target_label = self._get_block_name(target) Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_os.cs Tue Aug 1 14:06:42 2006 @@ -46,6 +46,46 @@ } } + class CRLFFile: IFile + { + private FileStream stream; + private TextWriter writer; + private TextReader reader; + public CRLFFile(FileStream stream, TextReader reader, TextWriter writer) + { + this.stream = stream; + this.writer = writer; + this.reader = reader; + } + + public FileStream GetStream() + { + return stream; + } + + public void Write(string buffer) + { + Debug.Assert(writer != null); // XXX: raise OSError? + writer.Write(buffer); + writer.Flush(); + } + + public string Read(int count) + { + Debug.Assert(reader != null); // XXX: raise OSError? + System.Text.StringBuilder builder = new System.Text.StringBuilder(count); + while (count-- > 0) { + int ch = reader.Read(); + if (ch == -1) + break; + if (ch == '\r' && reader.Peek() == '\n') + ch = reader.Read(); + builder.Append((char)ch); + } + return builder.ToString(); + } + } + class BinaryFile: IFile { private FileStream stream; @@ -124,7 +164,7 @@ private static FileMode get_file_mode(int flags) { if ((flags & O_APPEND) !=0 ) return FileMode.Append; if ((flags & O_TRUNC) !=0 ) return FileMode.Truncate; - if ((flags & O_CREAT) !=0 ) return FileMode.CreateNew; + if ((flags & O_CREAT) !=0 ) return FileMode.OpenOrCreate; return FileMode.Open; } @@ -143,8 +183,15 @@ if (f_access == FileAccess.Read || f_access == FileAccess.ReadWrite) reader = new StreamReader(stream); if (f_access == FileAccess.Write || f_access == FileAccess.ReadWrite) - writer = new StreamWriter(stream); - f = new TextFile(stream, reader, writer); + { + Console.Error.WriteLine("opening {0} for writing", name); + writer = new StreamWriter(stream); + } + + if (System.Environment.NewLine == "\r\n") + f = new CRLFFile(stream, reader, writer); + else + f = new TextFile(stream, reader, writer); } fdcount++; @@ -194,7 +241,8 @@ return res; } // path is not a file nor a dir, raise OSError - Helpers.raise_OSError(2); // ENOENT + //Helpers.raise_OSError(2); // ENOENT + PrebuiltGraphs.raiseOSError(2); return null; // never reached } Modified: pypy/dist/pypy/translator/cli/src/stub/main.il ============================================================================== --- pypy/dist/pypy/translator/cli/src/stub/main.il (original) +++ pypy/dist/pypy/translator/cli/src/stub/main.il Tue Aug 1 14:06:42 2006 @@ -36,3 +36,15 @@ } } } + +/// XXX: remove +.class public PrebuiltGraphs +{ + .method public static void raiseOSError(int32 errno_1) il managed + { + ldstr "This is only a stub, it should not be called" + newobj instance void class [mscorlib]System.ApplicationException::.ctor(string) + throw + ret + } +} Modified: pypy/dist/pypy/translator/cli/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_exception.py (original) +++ pypy/dist/pypy/translator/cli/test/test_exception.py Tue Aug 1 14:06:42 2006 @@ -3,4 +3,19 @@ from pypy.rpython.test.test_exception import BaseTestException class TestCliException(CliTest, BaseTestException): - pass + def test_nested_try(self): + def helper(x): + if x == 0: + raise ValueError + def dummy(): + pass + def fn(x): + try: + try: + helper(x) + finally: + dummy() + except ValueError, e: + raise + + self.interpret_raises(ValueError, fn, [0]) From antocuni at codespeak.net Tue Aug 1 15:15:44 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Aug 2006 15:15:44 +0200 (CEST) Subject: [pypy-svn] r30845 - pypy/dist/pypy/translator/cli Message-ID: <20060801131544.3E0F110081@code0.codespeak.net> Author: antocuni Date: Tue Aug 1 15:15:43 2006 New Revision: 30845 Modified: pypy/dist/pypy/translator/cli/function.py Log: store last_exception only if it is a Variable. Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Tue Aug 1 15:15:43 2006 @@ -125,10 +125,13 @@ self.store(link.target.inputargs[1]) else: # the exception value is on the stack, store it in the proper place - self.ilasm.opcode('dup') - self.store(link.last_exc_value) - self.ilasm.get_field(('Object_meta', 'Object', 'meta')) - self.store(link.last_exception) + if isinstance(link.last_exception, flowmodel.Variable): + self.ilasm.opcode('dup') + self.store(link.last_exc_value) + self.ilasm.get_field(('Object_meta', 'Object', 'meta')) + self.store(link.last_exception) + else: + self.store(link.last_exc_value) self._setup_link(link) target_label = self._get_block_name(target) From rhymes at codespeak.net Tue Aug 1 16:25:38 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Tue, 1 Aug 2006 16:25:38 +0200 (CEST) Subject: [pypy-svn] r30849 - pypy/dist/pypy/module/_ssl Message-ID: <20060801142538.EC84C1007E@code0.codespeak.net> Author: rhymes Date: Tue Aug 1 16:25:36 2006 New Revision: 30849 Modified: pypy/dist/pypy/module/_ssl/ssl.py Log: ssl3_enc_method struct definition was missing. that doesn't change the fact the module still doesn't compile though Modified: pypy/dist/pypy/module/_ssl/ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/ssl.py (original) +++ pypy/dist/pypy/module/_ssl/ssl.py Tue Aug 1 16:25:36 2006 @@ -1167,6 +1167,21 @@ pass class ssl3_enc_method(Structure): pass +ssl3_enc_method._fields_ = [ + ('enc', CFUNCTYPE(POINTER(SSL), c_int)), + ('mac', CFUNCTYPE(POINTER(SSL), STRING, c_int)), + ('setup_key_block', CFUNCTYPE(POINTER(SSL))), + ('generate_master_secret', CFUNCTYPE(POINTER(SSL), STRING, STRING, c_int)), + ('change_cipher_state', CFUNCTYPE(POINTER(SSL), c_int)), + ('final_finish_mac', CFUNCTYPE(POINTER(SSL), POINTER(EVP_MD_CTX), POINTER(EVP_MD_CTX), STRING, c_int, STRING)), + ('finish_mac_length', c_int), + ('cert_verify_mac', CFUNCTYPE(POINTER(SSL), POINTER(EVP_MD_CTX), STRING)), + ('client_finished_label', STRING), + ('client_finished_label_len', c_int), + ('server_finished_label', STRING), + ('server_finished_label_len', c_int), + ('alert_value', CFUNCTYPE(c_int)), +] ssl_method_st._fields_ = [ ('version', c_int), ('ssl_new', CFUNCTYPE(c_int, POINTER(SSL))), From auc at codespeak.net Tue Aug 1 16:42:49 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 1 Aug 2006 16:42:49 +0200 (CEST) Subject: [pypy-svn] r30850 - pypy/extradoc/eu-report Message-ID: <20060801144249.F23D810083@code0.codespeak.net> Author: auc Date: Tue Aug 1 16:42:43 2006 New Revision: 30850 Modified: pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf Log: Modified: pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf ============================================================================== Files pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf (original) and pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf Tue Aug 1 16:42:43 2006 differ From arigo at codespeak.net Tue Aug 1 17:34:20 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Aug 2006 17:34:20 +0200 (CEST) Subject: [pypy-svn] r30852 - in pypy/dist/pypy: doc rpython rpython/rctypes rpython/rctypes/test Message-ID: <20060801153420.36FF41007E@code0.codespeak.net> Author: arigo Date: Tue Aug 1 17:34:17 2006 New Revision: 30852 Added: pypy/dist/pypy/rpython/rctypes/atype.py (contents, props changed) pypy/dist/pypy/rpython/rctypes/rtype.py (contents, props changed) Modified: pypy/dist/pypy/doc/rctypes.txt pypy/dist/pypy/rpython/extregistry.py pypy/dist/pypy/rpython/rctypes/aarray.py pypy/dist/pypy/rpython/rctypes/astringbuf.py pypy/dist/pypy/rpython/rctypes/implementation.py pypy/dist/pypy/rpython/rctypes/rarray.py pypy/dist/pypy/rpython/rctypes/rmodel.py pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Log: Support for 'ctype*int' at run-time, to build arrays of a length which is not a constant during annotation. Modified: pypy/dist/pypy/doc/rctypes.txt ============================================================================== --- pypy/dist/pypy/doc/rctypes.txt (original) +++ pypy/dist/pypy/doc/rctypes.txt Tue Aug 1 17:34:17 2006 @@ -68,10 +68,11 @@ rule that types should not be manipulated at run-time. You can only call it with a constant type, as in ``POINTER(c_int)``. -The create_string_buffer() function allows you to build variable-size -arrays of chars, but there is no way to build other variable-sized -arrays: an expression like ``c_int * n`` can only appear at -bootstrapping-time. (This restriction should be lifted at some point.) +*New:* an expression like ``c_int * n`` can appear at run-time. It can +be used to create variable-sized array instances, i.e. arrays whose +length is not a static constant, as in ``my_array = (c_int * n)()``. +Similarly, the create_string_buffer() function returns a variable-size +array of chars. NOTE: in order to translate an RPython program using ctypes, the module ``pypy.rpython.rctypes.implementation`` must be imported! This is Modified: pypy/dist/pypy/rpython/extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extregistry.py (original) +++ pypy/dist/pypy/rpython/extregistry.py Tue Aug 1 17:34:17 2006 @@ -19,16 +19,28 @@ del selfcls._metatype_ def _register_value(selfcls, key): - assert key not in EXT_REGISTRY_BY_VALUE - EXT_REGISTRY_BY_VALUE[key] = selfcls + if isinstance(key, (tuple, list)): + for k in key: + selfcls._register_value(k) + else: + assert key not in EXT_REGISTRY_BY_VALUE + EXT_REGISTRY_BY_VALUE[key] = selfcls def _register_type(selfcls, key): - assert key not in EXT_REGISTRY_BY_TYPE - EXT_REGISTRY_BY_TYPE[key] = selfcls + if isinstance(key, (tuple, list)): + for k in key: + selfcls._register_type(k) + else: + assert key not in EXT_REGISTRY_BY_TYPE + EXT_REGISTRY_BY_TYPE[key] = selfcls def _register_metatype(selfcls, key): - assert key not in EXT_REGISTRY_BY_METATYPE - EXT_REGISTRY_BY_METATYPE[key] = selfcls + if isinstance(key, (tuple, list)): + for k in key: + selfcls._register_metatype(k) + else: + assert key not in EXT_REGISTRY_BY_METATYPE + EXT_REGISTRY_BY_METATYPE[key] = selfcls class ExtRegistryEntry(object): Modified: pypy/dist/pypy/rpython/rctypes/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/aarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/aarray.py Tue Aug 1 17:34:17 2006 @@ -5,6 +5,28 @@ ArrayType = type(ARRAY(c_int, 10)) +class VarSizedArrayType(object): + """Placeholder for ctypes array types whose size is not an + annotation-time constant. + """ + def __init__(self, itemtype): + self._type_ = itemtype + #self._length_ = unspecified + self.__name__ = itemtype.__name__ + '_Array' + + def get_instance_annotation(self, *args_s): + return SomeCTypesObject(self, ownsmemory=True) + + def __eq__(self, other): + return (self.__class__ is other.__class__ and + self._type_ == other._type_) + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self._type_) + class CallEntry(CTypesCallEntry): "Annotation and rtyping of calls to array types." @@ -19,16 +41,14 @@ if hop.nb_args > r_array.length: raise TyperError("too many arguments for an array of length %d" % ( r_array.length,)) - for i in range(hop.nb_args): - v_item = hop.inputarg(r_array.r_item, arg=i) - c_index = inputconst(lltype.Signed, i) - r_array.setitem(hop.llops, v_result, c_index, v_item) + items_v = hop.inputargs(*[r_array.r_item] * hop.nb_args) + r_array.initializeitems(hop.llops, v_result, items_v) return v_result class ObjEntry(CTypesObjEntry): "Annotation and rtyping of array instances." - _metatype_ = ArrayType + _metatype_ = ArrayType, VarSizedArrayType def get_field_annotation(self, s_array, fieldname): assert fieldname == 'value' Modified: pypy/dist/pypy/rpython/rctypes/astringbuf.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/astringbuf.py (original) +++ pypy/dist/pypy/rpython/rctypes/astringbuf.py Tue Aug 1 17:34:17 2006 @@ -4,6 +4,12 @@ from ctypes import create_string_buffer, c_char, sizeof +###################################################################### +# NOTE: astringbuf and rstringbuf should be removed and replaced # +# with a regular var-sized array of char, now that we # +# support var-sized arrays. # +###################################################################### + class StringBufferType(object): """Placeholder for the result type of create_string_buffer(), @@ -68,7 +74,7 @@ return r_arg.rtype_len(hop) else: if not s_arg.is_constant(): - raise TyperError("ctypes.sizeof(non_ctypes_object)") + raise TyperError("ctypes.sizeof(non_constant_type)") # XXX check that s_arg.const is really a ctypes type ctype = s_arg.const s_arg = SomeCTypesObject(ctype, ownsmemory=True) Added: pypy/dist/pypy/rpython/rctypes/atype.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/rctypes/atype.py Tue Aug 1 17:34:17 2006 @@ -0,0 +1,47 @@ +""" +Var-sized arrays, i.e. arrays whose size is not known at annotation-time. +""" + +from pypy.annotation.model import SomeCTypesObject +from pypy.annotation.model import SomeBuiltin, SomeInteger, SomeString +from pypy.annotation.pairtype import pair, pairtype +from pypy.rpython.extregistry import ExtRegistryEntry + + +class SomeCTypesType(SomeBuiltin): + """A ctypes type behaves like a built-in function, because it can only + be called -- with the exception of 'ctype*int' to build array types. + """ + def rtyper_makerepr(self, rtyper): + from pypy.rpython.rctypes.rtype import TypeRepr + return TypeRepr(self) + + def rtyper_makekey(self): + return SomeCTypesType, getattr(self, 'const', None) + + +class SomeVarSizedCTypesType(SomeBuiltin): + """A ctypes built at runtime as 'ctype*int'. + Note that at the moment 'ctype*int*int' is not supported. + """ + def __init__(self, ctype_item): + from pypy.rpython.rctypes.aarray import VarSizedArrayType + ctype_array = VarSizedArrayType(ctype_item) + SomeBuiltin.__init__(self, ctype_array.get_instance_annotation) + self.ctype_array = ctype_array + + def rtyper_makerepr(self, rtyper): + assert self.s_self is None + from pypy.rpython.rctypes.rtype import VarSizedTypeRepr + return VarSizedTypeRepr() + + def rtyper_makekey(self): + return SomeVarSizedCTypesType, self.ctype_array + + +class __extend__(pairtype(SomeCTypesType, SomeInteger)): + def mul((s_ctt, s_int)): + entry = s_ctt.analyser.im_self # fish fish + ctype_item = entry.instance + return SomeVarSizedCTypesType(ctype_item) + mul.can_only_throw = [] Modified: pypy/dist/pypy/rpython/rctypes/implementation.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/implementation.py (original) +++ pypy/dist/pypy/rpython/rctypes/implementation.py Tue Aug 1 17:34:17 2006 @@ -92,7 +92,14 @@ ## annotation, e.g. callback functions.""" class CTypesCallEntry(CTypesEntry): - "Annotation and rtyping of calls to ctypes types." + "Annotation and rtyping of ctypes types (mostly their calls)." + + def compute_annotation(self): + ctype = self.instance + assert ctype is not None + analyser = self.compute_result_annotation + methodname = ctype.__name__ + return SomeCTypesType(analyser, methodname=methodname) def compute_result_annotation(self, *args_s, **kwds_s): ctype = self.instance # the ctype is the called object @@ -108,6 +115,7 @@ # Importing for side effect of registering types with extregistry +from pypy.rpython.rctypes.atype import SomeCTypesType import pypy.rpython.rctypes.aprimitive import pypy.rpython.rctypes.apointer import pypy.rpython.rctypes.aarray Modified: pypy/dist/pypy/rpython/rctypes/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/rarray.py Tue Aug 1 17:34:17 2006 @@ -7,6 +7,7 @@ from pypy.rpython.rctypes.rmodel import genreccopy_arrayitem, reccopy, C_ZERO from pypy.rpython.rctypes.rprimitive import PrimitiveRepr from pypy.rpython.rctypes.rpointer import PointerRepr +from pypy.rpython.rctypes.aarray import VarSizedArrayType from pypy.annotation.model import SomeCTypesObject from pypy.objspace.flow.model import Constant @@ -18,15 +19,22 @@ array_ctype = s_array.knowntype item_ctype = array_ctype._type_ - self.length = array_ctype._length_ + if isinstance(array_ctype, VarSizedArrayType): + self.length = None + else: + self.length = array_ctype._length_ # Find the repr and low-level type of items from their ctype self.r_item = rtyper.getrepr(SomeCTypesObject(item_ctype, ownsmemory=False)) # Here, self.c_data_type == self.ll_type - c_data_type = lltype.FixedSizeArray(self.r_item.ll_type, - self.length) + if self.length is not None: + c_data_type = lltype.FixedSizeArray(self.r_item.ll_type, + self.length) + else: + c_data_type = lltype.Array(self.r_item.ll_type, + hints={'nolength': True}) super(ArrayRepr, self).__init__(rtyper, s_array, c_data_type) @@ -35,8 +43,11 @@ item_keepalive_type = self.r_item.get_content_keepalive_type() if not item_keepalive_type: return None - else: + elif self.length is not None: return lltype.FixedSizeArray(item_keepalive_type, self.length) + else: + raise NotImplementedError("XXX not supported yet: " + "var-sized arrays of pointers") def initialize_const(self, p, value): for i in range(self.length): @@ -100,6 +111,11 @@ genreccopy_arrayitem(llops, v_newkeepalive, v_keepalive_array, v_index) + def initializeitems(self, llops, v_array, items_v): + for i, v_item in enumerate(items_v): + c_index = inputconst(lltype.Signed, i) + self.setitem(llops, v_array, c_index, v_item) + class __extend__(pairtype(ArrayRepr, IntegerRepr)): def rtype_getitem((r_array, r_int), hop): Modified: pypy/dist/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/dist/pypy/rpython/rctypes/rmodel.py Tue Aug 1 17:34:17 2006 @@ -115,9 +115,20 @@ self.r_memoryowner.lowleveltype) def allocate_instance(self, llops): - c1 = inputconst(lltype.Void, self.lowleveltype.TO) + TYPE = self.lowleveltype.TO + if TYPE._is_varsize(): + raise TyperError("allocating array with unknown length") + c1 = inputconst(lltype.Void, TYPE) return llops.genop("malloc", [c1], resulttype=self.lowleveltype) + def allocate_instance_varsize(self, llops, v_length): + TYPE = self.lowleveltype.TO + if not TYPE._is_varsize(): + raise TyperError("allocating non-array with a specified length") + c1 = inputconst(lltype.Void, TYPE) + return llops.genop("malloc_varsize", [c1, v_length], + resulttype=self.lowleveltype) + def allocate_instance_ref(self, llops, v_c_data, v_c_data_owner=None): """Only if self.ownsmemory is false. This allocates a new instance and initialize its c_data pointer.""" Added: pypy/dist/pypy/rpython/rctypes/rtype.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/rctypes/rtype.py Tue Aug 1 17:34:17 2006 @@ -0,0 +1,42 @@ +from pypy.objspace.flow.model import Constant +from pypy.annotation.pairtype import pairtype +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst +from pypy.rpython.error import TyperError +from pypy.rpython.rbuiltin import BuiltinFunctionRepr + + +class TypeRepr(BuiltinFunctionRepr): + + def __init__(self, s_ctype): + assert s_ctype.s_self is None + if not s_ctype.is_constant(): + raise TyperError("non-constant ctypes type object") + ctype = s_ctype.const + BuiltinFunctionRepr.__init__(self, ctype) + + +class __extend__(pairtype(TypeRepr, IntegerRepr)): + + def rtype_mul((r_ctype, r_int), hop): + v_ctype, v_repeatcount = hop.inputargs(r_ctype, lltype.Signed) + assert isinstance(v_ctype, Constant) + return v_repeatcount + + +class VarSizedTypeRepr(Repr): + """Repr of the var-sized array type built at runtime as 'ctype*int'. + The ctype must be a real constant ctype, so the var-sized type can + be represented as just the runtime length. + """ + lowleveltype = lltype.Signed + + def rtype_simple_call(self, hop): + r_array = hop.r_result + args_r = [self] + [r_array.r_item] * (hop.nb_args-1) + args_v = hop.inputargs(*args_r) + v_repeatcount = args_v[0] + hop.exception_cannot_occur() + v_result = r_array.allocate_instance_varsize(hop.llops, v_repeatcount) + r_array.initializeitems(hop.llops, v_result, args_v[1:]) + return v_result Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Tue Aug 1 17:34:17 2006 @@ -163,6 +163,18 @@ a.translator.view() assert s == annmodel.SomeString() + def test_annotate_varsize_array(self): + def func(n): + a = (c_int * n)() + a[n//2] = 12 + return a[n//3] + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(func, [int]) + if conftest.option.view: + a.translator.view() + assert s.knowntype == int + class Test_specialization: def test_specialize_array(self): def create_array(): @@ -282,6 +294,27 @@ assert res.c_data[3] == 0 assert res.c_data[4] == 0 + def test_specialize_varsize_array_constructor(self): + def func(n): + return (c_int * n)() + res = interpret(func, [12]) + py.test.raises(TypeError, "len(res.c_data)") # nolength hint + assert res.c_data[0] == 0 + assert res.c_data[11] == 0 + py.test.raises(IndexError, "res.c_data[12]") + + def test_specialize_varsize_array(self): + def func(n): + a = (c_int * n)(5) + for i in range(1, n): + a[i] = i + res = 0 + for i in range(n): + res += a[i] + return res + res = interpret(func, [10]) + assert res == 50 + class Test_compilation: def setup_class(self): if not test_c_compile: From auc at codespeak.net Tue Aug 1 17:37:41 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 1 Aug 2006 17:37:41 +0200 (CEST) Subject: [pypy-svn] r30853 - in pypy/dist/pypy: lib module/_stackless objspace/cclp objspace/test Message-ID: <20060801153741.C97441007E@code0.codespeak.net> Author: auc Date: Tue Aug 1 17:37:37 2006 New Revision: 30853 Modified: pypy/dist/pypy/lib/_exceptions.py pypy/dist/pypy/module/_stackless/clonable.py pypy/dist/pypy/module/_stackless/interp_clonable.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: proper exceptions types are now raised futures read-onlyness more correctly enforced Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Tue Aug 1 17:37:37 2006 @@ -436,3 +436,16 @@ badchar, self.start, self.reason) return "%r codec can't encode characters in position %d-%d: %s" % ( self.encoding, self.start, self.end - 1, self.reason) + + + +#-- Logic object space specific stuff +#XXX conditionalize me on '-o logic' + +class LOError(Exception): pass + +class UnificationError(LOError): pass +class RebindingError(LOError): pass +class FutureBindingError(LOError): pass + +class AllBlockedError(LOError): pass Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Tue Aug 1 17:37:37 2006 @@ -57,7 +57,7 @@ raise OperationError(space.w_ValueError, space.wrap( "cannot switch to an unbound Coroutine")) self.switch() - rstack.resume_point("w_switch", self, space) + #rstack.resume_point("w_switch", self, space) state = self.costate w_ret, state.w_tempval = state.w_tempval, space.w_None return w_ret Modified: pypy/dist/pypy/module/_stackless/interp_clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/interp_clonable.py (original) +++ pypy/dist/pypy/module/_stackless/interp_clonable.py Tue Aug 1 17:37:37 2006 @@ -4,7 +4,6 @@ from pypy.interpreter.error import OperationError -from pypy.rpython import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode class InterpClonableCoroutine(Coroutine): Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 1 17:37:37 2006 @@ -126,7 +126,7 @@ self._init_head(self._main) self._init_blocked() w(".. SCHEDULER reinitialized") - raise OperationError(self.space.w_RuntimeError, + raise OperationError(self.space.w_AllBlockedError, self.space.wrap("can't schedule, possible deadlock in sight")) return to_be_run Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Tue Aug 1 17:37:37 2006 @@ -1,10 +1,10 @@ -from pypy.interpreter import baseobjspace +from pypy.interpreter import baseobjspace, typedef from pypy.objspace.cclp.misc import w, ClonableCoroutine W_Root = baseobjspace.W_Root -#-- Types -------------------------------------------------- +#-- Variables types ---------------------------------------- class W_Var(W_Root): def __init__(w_self, space): @@ -27,15 +27,16 @@ w_self.client = ClonableCoroutine.w_getcurrent(space) w("FUT", str(w_self)) +#-- Exception types ---------------------------------------- -class W_FailedValue(W_Root, object): +class W_FailedValue(W_Root): """wraps an exception raised in some coro, to be re-raised in some dependant coro sometime later """ def __init__(w_self, exc): w_self.exc = exc -#-- Misc ---------------------------------------------------- +#-- Misc --------------------------------------------------- def deref(space, w_var): #XXX kill me ? Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Tue Aug 1 17:37:37 2006 @@ -55,7 +55,7 @@ scheduler[0].add_to_blocked_byneed(w_var, ClonableCoroutine.w_getcurrent(space)) scheduler[0].schedule() else: - raise OperationError(space.w_RuntimeError, + raise OperationError(space.w_TypeError, space.wrap("wait_needed only supported on unbound variables")) def wait_needed(space, w_var): @@ -140,18 +140,21 @@ w_curr = w_next -def fail(space, w_obj1, w_obj2): +def raise_unification_failure(space): """raises a specific exception for bind/unify should fail the current comp. space at some point""" - #FIXME : really raise some specific exception - assert isinstance(w_obj1, W_Root) - assert isinstance(w_obj2, W_Root) - raise OperationError(space.w_RuntimeError, + raise OperationError(space.w_UnificationError, space.wrap("Unification failure")) +# to signal a future binding exception +def raise_future_binding(space): + raise OperationError(space.w_FutureBindingError, + space.wrap("This future is read-only for you, pal")) + #-- BIND ----------------------------- + def bind(space, w_var, w_obj): """1. aliasing of unbound variables 2. assign bound var to unbound var @@ -170,15 +173,15 @@ return _assign(space, w_var, w_obj) if space.is_true(space.eq(w_var.w_bound_to, w_obj)): return - raise OperationError(space.w_RuntimeError, - space.wrap("Cannot bind twice")) + raise OperationError(space.w_RebindingError, + space.wrap("Cannot bind twice but two identical values")) + def bind__Future_Root(space, w_fut, w_obj): #v("future val", str(id(w_fut))) if w_fut.client == ClonableCoroutine.w_getcurrent(space): - raise OperationError(space.w_RuntimeError, - space.wrap("This future is read-only for you, pal")) - bind__Var_Root(space, w_fut, w_obj) # call-next-method ? + raise_future_binding(space) + return bind__Var_Root(space, w_fut, w_obj) # call-next-method ? def bind__Var_Var(space, w_v1, w_v2): #w("var var") @@ -199,17 +202,20 @@ def bind__Future_Var(space, w_fut, w_var): #v("future var") if w_fut.client == ClonableCoroutine.w_getcurrent(space): - raise OperationError(space.w_RuntimeError, - space.wrap("This future is read-only for you, pal")) - bind__Var_Var(space, w_fut, w_var) + raise_future_binding(space) + return bind__Var_Var(space, w_fut, w_var) -#XXX Var_Future would just alias or assign, this is ok +def bind__Var_Future(space, w_var, w_fut): + if space.is_true(space.is_bound(w_fut)): + return bind__Var_Root(w_var, w_fut.w_bound_to) + raise_future_binding(space) bind_mm = StdObjSpaceMultiMethod('bind', 2) bind_mm.register(bind__Var_Root, W_Var, W_Root) bind_mm.register(bind__Var_Var, W_Var, W_Var) bind_mm.register(bind__Future_Root, W_Future, W_Root) bind_mm.register(bind__Future_Var, W_Future, W_Var) +bind_mm.register(bind__Var_Future, W_Var, W_Future) all_mms['bind'] = bind_mm def _assign(space, w_var, w_val): @@ -281,7 +287,7 @@ w_d1 = w_x.getdict() #returns wrapped dict or unwrapped None ... w_d2 = w_y.getdict() if None in [w_d1, w_d2]: - fail(space, w_x, w_y) + raise_unification_failure(space) else: return space.unify(w_d1, w_d2) return space.w_None @@ -308,7 +314,7 @@ def unify__Tuple_Tuple(space, w_i1, w_i2): if len(w_i1.wrappeditems) != len(w_i2.wrappeditems): - fail(space, w_i1, w_i2) + raise_unification_failure(space) idx, top = (-1, space.int_w(space.len(w_i1))-1) while idx < top: idx += 1 @@ -321,7 +327,7 @@ def unify__List_List(space, w_i1, w_i2): if len(w_i1.wrappeditems) != len(w_i2.wrappeditems): - fail(space, w_i1, w_i2) + raise_unification_failure(space) idx, top = (-1, space.int_w(space.len(w_i1))-1) while idx < top: idx += 1 Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Tue Aug 1 17:37:37 2006 @@ -17,7 +17,7 @@ assert is_bound(1) # FIXME : propagate proper # FailureException - raises(Exception, bind, X, 2) + raises(RebindingError, bind, X, 2) def test_bind_to_self(self): X = newvar() @@ -51,7 +51,6 @@ assert alias_of(X, Y) assert alias_of(X, Y) unify(X, 1) - # what about is_alias, then ? assert X == 1 assert Y == 1 @@ -131,7 +130,7 @@ X = newvar() def f(x): return x + 1 - raises(Exception, f, X) + raises(AllBlockedError, f, X) def test_eq_unifies_simple(self): X = newvar() @@ -171,7 +170,7 @@ unify(X, (1, (2, None))) assert X == (1, (2, None)) unify(X, (1, (2, None))) - raises(Exception, unify, X, (1, 2)) + raises(UnificationError, unify, X, (1, 2)) def test_unify_dict(self): Z, W = newvar(), newvar() @@ -193,7 +192,7 @@ assert alias_of(f1.b, f2.b) unify(f2.b, 'foo') assert f1.b == f2.b == 'foo' - raises(Exception, unify, f1.b, 24) + raises(UnificationError, unify, f1.b, 24) class AppTest_LogicFutures(object): @@ -202,8 +201,6 @@ cls.space = gettestobjspace('logic', usemodules=("_stackless",)) def test_future_value(self): - print "future value", sched_info() - def poop(X): return X + 1 @@ -214,22 +211,18 @@ X = newvar() T = future(poop, X) - raises(Exception, unify, T, 42) + raises(FutureBindingError, unify, T, 42) bind(X, 42); schedule() # helps the gc - X, Y = newvar(), newvar() + X, Y, Z = newvar(), newvar(), newvar() + bind(Z, 42) T = future(poop, X) - raises(Exception, unify, T, Y) + raises(FutureBindingError, unify, T, Z) + raises(FutureBindingError, unify, Z, T) + raises(FutureBindingError, unify, Y, T) + raises(FutureBindingError, unify, T, Y) bind(X, 42); schedule() # gc ... - assert is_free(Y) - X = newvar() - T = future(poop, X) - unify(Y, T) - unify(X, 42) - assert Y == 43 - bind(X, 42); schedule() - def test_one_future_exception(self): print "one future exception", sched_info() class FooException(Exception): pass @@ -340,22 +333,26 @@ def test_eager_producer_consummer(self): print "eager_producer_consummer", sched_info() - def generate(n, limit): + def generate(n, limit, R): if n < limit: - return (n, generate(n + 1, limit)) - return None + Tail = newvar() + unify(R, (n, Tail)) + return generate(n + 1, limit, Tail) + bind(R, None) + return - def sum(L, a): + def sum(L, a, R): Head, Tail = newvar(), newvar() unify(L, (Head, Tail)) if Tail != None: - return sum(Tail, Head + a) - return a + Head + return sum(Tail, Head + a, R) + bind(R, a + Head) + return X = newvar() S = newvar() - unify(S, future(sum, X, 0)) - unify(X, future(generate, 0, 10)) + stacklet(sum, X, 0, S) + stacklet(generate, 0, 10, X) assert S == 45 @@ -365,28 +362,32 @@ def lgenerate(n, L): """wait-needed version of generate""" #XXX how is this ever collected ? + # should be when L becomes unreferenced from + # outside this thread ... But this can't + # happen right now because we keep + # references also of this thread in the + # scheduler. wait_needed(L) Tail = newvar() bind(L, (n, Tail)) lgenerate(n+1, Tail) - def lsum(L, a, limit): + def lsum(L, a, limit, R): """this summer controls the generator""" if limit > 0: Head, Tail = newvar(), newvar() - wait(L) + wait(L) # send needed signal to generator unify(L, (Head, Tail)) - return lsum(Tail, a+Head, limit-1) + return lsum(Tail, a+Head, limit-1, R) else: - return a + bind(R, a) Y = newvar() T = newvar() future(lgenerate, 0, Y) - unify(T, future(lsum, Y, 0, 10)) + stacklet(lsum, Y, 0, 10, T) - wait(T) assert T == 45 assert len(sched_info()['blocked_byneed']) == 1 reset_scheduler() From auc at codespeak.net Tue Aug 1 17:54:34 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 1 Aug 2006 17:54:34 +0200 (CEST) Subject: [pypy-svn] r30854 - pypy/dist/pypy/objspace/cclp Message-ID: <20060801155434.941A610088@code0.codespeak.net> Author: auc Date: Tue Aug 1 17:54:33 2006 New Revision: 30854 Modified: pypy/dist/pypy/objspace/cclp/variable.py Log: transl. fix Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Tue Aug 1 17:54:33 2006 @@ -207,7 +207,7 @@ def bind__Var_Future(space, w_var, w_fut): if space.is_true(space.is_bound(w_fut)): - return bind__Var_Root(w_var, w_fut.w_bound_to) + return bind__Var_Root(w_var, deref(space, w_fut)) raise_future_binding(space) bind_mm = StdObjSpaceMultiMethod('bind', 2) From arigo at codespeak.net Tue Aug 1 18:03:04 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Aug 2006 18:03:04 +0200 (CEST) Subject: [pypy-svn] r30856 - in pypy/dist/pypy/rpython/rctypes: . test Message-ID: <20060801160304.BF94D10089@code0.codespeak.net> Author: arigo Date: Tue Aug 1 18:03:02 2006 New Revision: 30856 Modified: pypy/dist/pypy/rpython/rctypes/apointer.py pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Log: Support for 'POINTER(ctype * length)'. In rctypes it actually returns the same as 'POINTER(ctype * 0)', because rctypes performs no index checking so it should be equivalent. But both have to be supported, otherwise we can't run the code on top of the normal ctypes without getting IndexErrors. Modified: pypy/dist/pypy/rpython/rctypes/apointer.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/apointer.py (original) +++ pypy/dist/pypy/rpython/rctypes/apointer.py Tue Aug 1 18:03:02 2006 @@ -63,9 +63,18 @@ def compute_result_annotation(self, s_arg): from pypy.annotation.bookkeeper import getbookkeeper - assert s_arg.is_constant(), ( - "POINTER(%r): argument must be constant" % (s_arg,)) - RESTYPE = POINTER(s_arg.const) + from atype import SomeVarSizedCTypesType + if isinstance(s_arg, SomeVarSizedCTypesType): + # POINTER(varsized_array_type): given that rctypes performs + # no index checking, this pointer-to-array type is equivalent + # to a pointer to an array of whatever size. + RESTYPE = POINTER(s_arg.ctype_array._type_ * 0) + else: + # POINTER(constant_ctype) returns the constant annotation + # corresponding to the POINTER(ctype). + assert s_arg.is_constant(), ( + "POINTER(%r): argument must be constant" % (s_arg,)) + RESTYPE = POINTER(s_arg.const) return getbookkeeper().immutablevalue(RESTYPE) def specialize_call(self, hop): Modified: pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py Tue Aug 1 18:03:02 2006 @@ -218,3 +218,12 @@ assert PyString_FromString((c_char * 6)(*"hello")) == "hello" assert PyString_FromString((c_byte * 6)(104,101,108,108,111)) =="hello" assert PyString_FromString(create_string_buffer("hello")) == "hello" + +def test_varsize_cast(): + import struct + N = struct.calcsize("l") + x = c_long() + p = cast(pointer(x), POINTER(c_ubyte*N)) + for i, c in enumerate(struct.pack("l", 12345678)): + p.contents[i] = ord(c) + assert x.value == 12345678 Modified: pypy/dist/pypy/rpython/rctypes/test/test_rarray.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rarray.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rarray.py Tue Aug 1 18:03:02 2006 @@ -11,7 +11,7 @@ import sys from pypy.rpython.test.test_llinterp import interpret -from ctypes import c_int, c_short, c_char_p, c_char, pointer +from ctypes import c_int, c_short, c_char_p, c_char, c_ubyte, pointer, cast from ctypes import ARRAY, POINTER, Structure c_int_10 = ARRAY(c_int,10) @@ -364,6 +364,22 @@ fn = self.compile(func, []) assert fn() == "xy" + def test_compile_varsize_cast(self): + # this cannot work on top of lltype, but only in unsafe C + import struct + N = struct.calcsize("i") + BYTES = list(enumerate(struct.pack("i", 12345678))) + def func(n): + x = c_int() + arraytype = c_ubyte * n # runtime length + p = cast(pointer(x), POINTER(arraytype)) + for i, c in BYTES: + p.contents[i] = ord(c) + return x.value + fn = self.compile(func, [int]) + res = fn(N) + assert res == 12345678 + class Test_compilation_llvm(Test_compilation): def setup_class(self): if not test_llvm_compile: From mwh at codespeak.net Tue Aug 1 18:32:44 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 1 Aug 2006 18:32:44 +0200 (CEST) Subject: [pypy-svn] r30860 - pypy/dist/pypy/translator/stackless Message-ID: <20060801163244.929AC10080@code0.codespeak.net> Author: mwh Date: Tue Aug 1 18:32:43 2006 New Revision: 30860 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: reuse some resume blocks, saves another ~130k in pypy-c-stackless. obscurity++ though. next: reusing resume points when only the erased types are the same, cleanups? Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Tue Aug 1 18:32:43 2006 @@ -449,6 +449,7 @@ assert self.curr_graph is None self.curr_graph = graph self.curr_graph_save_blocks = {} + self.curr_graph_resume_blocks = {} if SAVE_STATISTICS: self.stats.cur_rp_exact_types = {} self.stats.cur_rp_erased_types = {} @@ -482,6 +483,7 @@ self.curr_graph = None self.curr_graph_save_blocks = None + self.curr_graph_resume_blocks = None def ops_read_global_state_field(self, targetvar, fieldname): ops = [] @@ -519,8 +521,12 @@ not_resuming_link.llexitcase = -1 resuming_links = [] for resume_index, resume_block in enumerate(self.resume_blocks): + if resume_block.inputargs: + args = [var_resume_state] + else: + args = [] resuming_links.append( - model.Link([], resume_block, resume_index)) + model.Link(args, resume_block, resume_index)) resuming_links[-1].llexitcase = resume_index new_start_block.exitswitch = var_resume_state @@ -897,68 +903,111 @@ def _generate_resume_block(self, varsinfieldorder, frame_type, var_result, links_to_resumption): - newblock = model.Block([]) - newargs = [] - llops = LowLevelOpList() - llops.genop("setfield", - [self.ll_global_state, - self.c_restart_substate_name, - self.c_minus_one]) - frame_top = varoftype(lltype.Ptr(frame_type)) - llops.extend(self.ops_read_global_state_field(frame_top, "top")) - llops.genop("setfield", - [self.ll_global_state, - self.c_inst_top_name, - self.c_null_state]) - varmap = {} - fielditer = iter(frame_type._names[1:]) - for arg in varsinfieldorder: - assert arg is not var_result - t = storage_type(arg.concretetype) - if t is lltype.Void: - v_newarg = model.Constant(None, lltype.Void) - else: - fname = model.Constant(fielditer.next(), lltype.Void) - assert frame_type._flds[fname.value] is t - v_newarg = llops.genop('getfield', [frame_top, fname], - resulttype = t) - v_newarg = gen_cast(llops, arg.concretetype, v_newarg) - varmap[arg] = v_newarg - - rettype = storage_type(var_result.concretetype) - getretval = self.fetch_retvals[rettype] - retval = llops.genop("direct_call", [getretval], - resulttype = rettype) - varmap[var_result] = retval - - newblock.operations.extend(llops) - - def rename(arg): - if isinstance(arg, model.Variable): - if arg in varmap: + typekey = [v.concretetype for v in varsinfieldorder] + linkkey = [(link.target, link.exitcase) for link in links_to_resumption[1:]] + key = tuple([var_result.concretetype] + typekey + linkkey) + if key in self.curr_graph_resume_blocks: + newblock, switchblock, newargs, newresult = self.curr_graph_resume_blocks[key] + if switchblock is None: + newblock.inputargs = [varoftype(lltype.Signed)] + switchblock = unsimplify.insert_empty_block(None, newblock.exits[0], []) + newblock.exits[0].args.append(newblock.inputargs[0]) + switchblock.inputargs.append(varoftype(lltype.Signed)) + switchblock.exitswitch = switchblock.inputargs[-1] + link, = switchblock.exits + link.exitcase = link.llexitcase = self.resume_blocks.index(newblock) + mapping = {} + for i in range(len(newblock.exits[0].args)): + mapping[newblock.exits[0].args[i]] = switchblock.inputargs[i] + if newresult in mapping: + newresult = mapping[newresult] + newnewargs = [] + for arg in newargs: + newnewargs.append(mapping[arg]) + newargs = newnewargs + self.curr_graph_resume_blocks[key] = newblock, switchblock, newnewargs, newresult + oldlink = links_to_resumption[0] + varmap = {} + for old, new in zip(varsinfieldorder, newargs): + varmap[old] = new + varmap[var_result] = newresult + def rename(arg): + if isinstance(arg, model.Variable): return varmap[arg] else: - assert arg in [l.last_exception, l.last_exc_value] - r = unsimplify.copyvar(None, arg) - varmap[arg] = r - return r - else: - return arg - - newblock.closeblock(*[l.copy(rename) - for l in links_to_resumption]) - # this check is a bit implicit! - if len(links_to_resumption) > 1: - newblock.exitswitch = model.c_last_exception + return arg + newlink = oldlink.copy(rename) + newlink.exitcase = newlink.llexitcase = len(self.resume_blocks) + switchblock.recloseblock(*(switchblock.exits + (newlink,))) + rettype = newresult.concretetype + retval = newresult + retlink = newlink else: - newblock.exitswitch = None + newblock = model.Block([]) + newargs = [] + llops = LowLevelOpList() + llops.genop("setfield", + [self.ll_global_state, + self.c_restart_substate_name, + self.c_minus_one]) + frame_top = varoftype(lltype.Ptr(frame_type)) + llops.extend(self.ops_read_global_state_field(frame_top, "top")) + llops.genop("setfield", + [self.ll_global_state, + self.c_inst_top_name, + self.c_null_state]) + varmap = {} + newargs = [] + fielditer = iter(frame_type._names[1:]) + for arg in varsinfieldorder: + assert arg is not var_result + t = storage_type(arg.concretetype) + if t is lltype.Void: + v_newarg = model.Constant(None, lltype.Void) + else: + fname = model.Constant(fielditer.next(), lltype.Void) + assert frame_type._flds[fname.value] is t + v_newarg = llops.genop('getfield', [frame_top, fname], + resulttype = t) + v_newarg = gen_cast(llops, arg.concretetype, v_newarg) + newargs.append(v_newarg) + varmap[arg] = v_newarg + + rettype = storage_type(var_result.concretetype) + getretval = self.fetch_retvals[rettype] + retval = llops.genop("direct_call", [getretval], + resulttype = rettype) + varmap[var_result] = retval + + newblock.operations.extend(llops) + + def rename(arg): + if isinstance(arg, model.Variable): + if arg in varmap: + return varmap[arg] + else: + assert arg in [l.last_exception, l.last_exc_value] + r = unsimplify.copyvar(None, arg) + varmap[arg] = r + return r + else: + return arg + newblock.closeblock(*[l.copy(rename) + for l in links_to_resumption]) + # this check is a bit implicit! + if len(links_to_resumption) > 1: + newblock.exitswitch = model.c_last_exception + else: + newblock.exitswitch = None + self.curr_graph_resume_blocks[key] = newblock, None, newargs, retval + retlink = newblock.exits[0] + if SAVE_STATISTICS: + self.stats.resumeops += len(newblock.operations) + if var_result.concretetype != rettype: - self.insert_return_conversion( - newblock.exits[0], var_result.concretetype, retval) + self.insert_return_conversion(retlink, var_result.concretetype, retval) - if SAVE_STATISTICS: - self.stats.resumeops += len(newblock.operations) return newblock def generate_restart_infos(self, graph): From auc at codespeak.net Tue Aug 1 18:58:41 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 1 Aug 2006 18:58:41 +0200 (CEST) Subject: [pypy-svn] r30863 - in pypy/dist/pypy: module/_stackless objspace/cclp Message-ID: <20060801165841.EF83510080@code0.codespeak.net> Author: auc Date: Tue Aug 1 18:58:38 2006 New Revision: 30863 Modified: pypy/dist/pypy/module/_stackless/clonable.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/variable.py Log: translation & other fixes Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Tue Aug 1 18:58:38 2006 @@ -25,6 +25,7 @@ if not is_main: space.getexecutioncontext().subcontext_new(self) self._dead = False + self._next = self._prev = self # help the annotator with the scheduling ring def hello(self): if we_are_translated(): Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 1 18:58:38 2006 @@ -1,3 +1,4 @@ +from pypy.rpython.objectmodel import we_are_translated from pypy.interpreter.error import OperationError from pypy.interpreter import gateway @@ -26,10 +27,10 @@ def get_threads(self): threads = [self._head] - curr = self._head.next + curr = self._head._next while curr != self._head: threads.append(curr) - curr = curr.next + curr = curr._next return threads def _init_blocked(self): @@ -39,14 +40,15 @@ def _init_head(self, coro): self._head = coro - self._head.next = self._head.prev = self._head + self._head._next = self._head._prev = self._head def _set_head(self, thread): + assert isinstance(thread, ClonableCoroutine) self._head = thread def _check_initial_conditions(self): try: - assert self._head.next == self._head.prev == self._head + assert self._head._next == self._head._prev == self._head assert self._head not in self._blocked assert self._head not in self._blocked_on assert self._head not in self._blocked_byneed @@ -60,34 +62,37 @@ raise def _chain_insert(self, thread): - assert thread.next is None - assert thread.prev is None + assert thread._next is thread + assert thread._prev is thread + assert isinstance(thread, ClonableCoroutine) + assert isinstance(thread._next, ClonableCoroutine) + assert isinstance(thread._prev, ClonableCoroutine) if self._head is None: - thread.next = thread - thread.prev = thread + thread._next = thread + thread._prev = thread self._set_head(thread) else: r = self._head - l = r.prev - l.next = thread - r.prev = thread - thread.prev = l - thread.next = r + l = r._prev + l._next = thread + r._prev = thread + thread._prev = l + thread._next = r def remove_thread(self, thread): w(".. REMOVING", str(id(thread))) assert thread not in self._blocked del self._traced[thread] - l = thread.prev - r = thread.next - l.next = r - r.prev = l - if r == thread: - if not we_are_translated(): + l = thread._prev + r = thread._next + l._next = r + r._prev = l + if r == thread: #XXX write a test for me ! + if not we_are_translated(): import traceback traceback.print_exc() self.display_head() - thread.next = thread.next = None + thread._next = thread._next = None return thread #-- to be used by logic objspace @@ -117,7 +122,7 @@ while (to_be_run in self._blocked) \ or to_be_run.is_dead() \ or (to_be_run == current): - to_be_run = to_be_run.next + to_be_run = to_be_run._next if to_be_run == sentinel: if not dont_pass: return ClonableCoroutine.w_getcurrent(self.space) @@ -136,16 +141,16 @@ curr = self._head sentinel = curr count = 1 # there is always a main thread - while curr.next != sentinel: - curr = curr.next + while curr._next != sentinel: + curr = curr._next count += 1 return count def display_head(self): curr = self._head v('Threads : [', '-'.join([str(id(curr)), str(curr in self._blocked)])) - while curr.next != self._head: - curr = curr.next + while curr._next != self._head: + curr = curr._next v('-'.join([str(id(curr)), str(curr in self._blocked)])) w(']') Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Tue Aug 1 18:58:38 2006 @@ -15,8 +15,6 @@ args = __args__.normalize() # coro init coro = ClonableCoroutine(space) - # prepare thread chaining, create missing slots - coro.next = coro.prev = None # computation space is the same as in the parent coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace # feed the coro @@ -37,8 +35,6 @@ args = __args__.normalize() # coro init coro = ClonableCoroutine(space) - # prepare thread chaining, create missing slots - coro.next = coro.prev = None # computation space is the same as in the parent coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Tue Aug 1 18:58:38 2006 @@ -205,10 +205,12 @@ raise_future_binding(space) return bind__Var_Var(space, w_fut, w_var) -def bind__Var_Future(space, w_var, w_fut): - if space.is_true(space.is_bound(w_fut)): - return bind__Var_Root(w_var, deref(space, w_fut)) - raise_future_binding(space) +def bind__Var_Future(space, w_var, w_fut): + if space.is_true(space.is_bound(w_fut)): #XXX write a test for me ! + return bind__Var_Root(space, w_var, deref(space, w_fut)) + if w_fut.client == ClonableCoroutine.w_getcurrent(space): + raise_future_binding(space) + return bind__Var_Var(space, w_var, w_fut) #and for me ... bind_mm = StdObjSpaceMultiMethod('bind', 2) bind_mm.register(bind__Var_Root, W_Var, W_Root) From ale at codespeak.net Wed Aug 2 09:59:10 2006 From: ale at codespeak.net (ale at codespeak.net) Date: Wed, 2 Aug 2006 09:59:10 +0200 (CEST) Subject: [pypy-svn] r30875 - pypy/dist/pypy/lib/pyontology Message-ID: <20060802075910.0D76610080@code0.codespeak.net> Author: ale Date: Wed Aug 2 09:59:06 2006 New Revision: 30875 Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py Log: You have to check if values are different in order to fail a FunctionaProperty test Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/constraint_classes.py (original) +++ pypy/dist/pypy/lib/pyontology/constraint_classes.py Wed Aug 2 09:59:06 2006 @@ -212,7 +212,10 @@ domain_dict = Linkeddict(domain) for cls, val in domain_dict.items(): if len(val) != 1: - raise ConsistencyFailure("FunctionalCardinality error") + for item in val: + for otheritem in val: + if (otheritem == item) == False: + raise ConsistencyFailure("FunctionalCardinality error") else: return 0 From ale at codespeak.net Wed Aug 2 10:26:26 2006 From: ale at codespeak.net (ale at codespeak.net) Date: Wed, 2 Aug 2006 10:26:26 +0200 (CEST) Subject: [pypy-svn] r30876 - in pypy/dist/pypy/lib/pyontology: . test Message-ID: <20060802082626.2948510081@code0.codespeak.net> Author: ale Date: Wed Aug 2 10:26:25 2006 New Revision: 30876 Modified: pypy/dist/pypy/lib/pyontology/pyontology.py pypy/dist/pypy/lib/pyontology/test/test_ontology.py Log: Better support for literals, bugfixes Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Wed Aug 2 10:26:25 2006 @@ -6,9 +6,9 @@ from constraint_classes import * import sys, py import time -from pypy.tool.ansi_print import ansi_log log = py.log.Producer("Pyontology") -py.log.setconsumer("Pyontology", ansi_log) +#from pypy.tool.ansi_print import ansi_log +py.log.setconsumer("Pyontology", None) namespaces = { @@ -76,7 +76,7 @@ def finish(self, variables, glob_constraints): # The finish method constructs the constraints if not self.finished: - log("FINISH %s" % self.name) + log.finish("%s" % self.name) # Try to initialise constraints for this class if len(self.type) > 1: #try to merge the domains of the types @@ -95,11 +95,12 @@ self.domains.update(dom) self.in_constraint.extend(constraints) # Initialise constraints from the base classes + self.finished = True for cls in self.bases: cls = variables[cls] dom,constraint = cls.finish(variables, glob_constraints) - log("DOM %r "%dom) + log.finish("DOM %r "%dom) # if the base class is a Restriction we shouldnt add the constraints to the store if not isinstance(cls, Restriction): self.domains.update(dom) @@ -110,7 +111,6 @@ self.domains.update(dom) log("Updating constraints %r" % constraints) self.in_constraint.extend(constraints) - self.finished = True log("RESULT of finish %r, %r" %(self.domains,self.in_constraint)) # update the store if ('owl_Thing' in variables.keys() and isinstance(self, ClassDomain) @@ -145,6 +145,7 @@ self._bases = bases def addValue(self, value): +# assert isinstance(value, URIRef) self.values[value] = True def getValues(self): @@ -154,7 +155,8 @@ return iter(self.values.keys()) def setValues(self, values): - self.values = dict.fromkeys(values) + for val in values: + self.addValue(val) #self.values = dict.fromkeys(values) class FixedClassDomain(ClassDomain): @@ -181,7 +183,8 @@ return "<%s( %s, %s)>"%(self.__class__.__name__, self.name, self.uri) def __hash__(self): - return hash(self.uri) + return hash(self.uri) + def __eq__(self, other): log("CMP %r,%r"%(self,other)) if ((hasattr(other,'uri') and self.uri == other.uri) or @@ -465,8 +468,10 @@ a = uris[ns] + '_' + name var = str(a.replace('.','_')) var = str(a.replace('-','_')) - else: + elif type(a) == BNode: var = str(a) + else: + return a if not cls: return var if not var in self.variables: @@ -550,7 +555,7 @@ if isinstance(self.variables[avar], Thing): self.variables[avar].addValue(Individual(svar, s)) else: - self.variables[avar].addValue(s) + self.variables[avar].addValue(svar) def first(self, s, var): pass @@ -611,6 +616,8 @@ var = self.flatten_rdf_list(var) svar = self.make_var(FixedClassDomain, s) res = self.variables[var].getValues() + for ind in res: + self.make_var(Thing, ind) self.variables[svar].setValues([ Individual(self.make_var(None,x),x) for x in res]) @@ -728,7 +735,7 @@ avar = self.make_var(None, var) scls = self.variables[svar] scls.un_constraint.append(constraint) - scls.value = var + scls.value = avar def hasValue(self, s, var): """ The hasValue restriction defines a class having as an extension all @@ -762,7 +769,7 @@ var = "%s_%s_allvalue" %(cls, prop.name) dom = {var : fd(prop.getValues( ))} # The condition should return true if - cons = Expression([cls, var], " %s[1] in %s and %s == %s[0]" %(var, val, cls, var)) + cons = Expression([cls, var, val], " %s[1] in %s and %s == %s[0]" %(var, val, cls, var)) return dom, [cons] self.value_helper(s, var, somevalue) Modified: pypy/dist/pypy/lib/pyontology/test/test_ontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_ontology.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_ontology.py Wed Aug 2 10:26:25 2006 @@ -52,7 +52,7 @@ def test_addvalue(): O = Ontology() - a = O.make_var(Property, 'a') + a = O.make_var(Property, URIRef('a')) O.variables[a].addValue('key', 42) assert O.variables[a].getValues() == [('key', 42)] O.variables[a].addValue('key', 43) @@ -331,8 +331,8 @@ O.onProperty(restrict,p) O.consider_triple((cls, p, 1)) O.hasValue(restrict, 2) - O.type(2, URIRef(namespaces['owl']+'#Thing')) - O.type(1, URIRef(namespaces['owl']+'#Thing')) +# O.type(2, URIRef(namespaces['owl']+'#Thing')) +# O.type(1, URIRef(namespaces['owl']+'#Thing')) cls2 = URIRef('class2') obj = URIRef(namespaces['owl']+'#Thing') @@ -364,7 +364,7 @@ restrict = BNode('anon') own = [UR('first'), UR('second'), UR('third')] O.oneOf(restrict, own) - O.type(restrict, namespaces['owl']+'#Class') + O.type(restrict, UR(namespaces['owl']+'#Class')) O.consistency() assert len(O.rep._domains[restrict].getValues()) == 3 assert set(O.rep._domains[restrict].getValues()) == set(own) @@ -392,7 +392,7 @@ restrict = BNode('anon') own = ['1','2','3'] O.oneOf(restrict, own) - O.type(restrict, namespaces['owl']+'#DataRange') + O.type(restrict, UR(namespaces['owl']+'#DataRange')) O.consistency() assert len(O.rep._domains[restrict].getValues()) == 3 assert set(O.rep._domains[restrict].getValues()) == set(own) @@ -478,9 +478,9 @@ O.differentFrom(own1, own2) O.differentFrom(cls, own2) O.differentFrom(own2,cls) - O.type(cls, namespaces['owl']+'#Thing') - O.type(own1, namespaces['owl']+'#Thing') - O.type(own2, namespaces['owl']+'#Thing') + O.type(cls, UR(namespaces['owl']+'#Thing')) + O.type(own1, UR(namespaces['owl']+'#Thing')) + O.type(own2, UR(namespaces['owl']+'#Thing')) O.consistency() assert len(O.rep._constraints) == 4 @@ -488,7 +488,7 @@ O = Ontology() cls = BNode('anon') O.differentFrom(cls, cls) - O.type(cls, namespaces['owl']+'#Thing') + O.type(cls, UR(namespaces['owl']+'#Thing')) py.test.raises(ConsistencyFailure, O.consistency) def test_sameas(): @@ -499,9 +499,9 @@ O.sameAs(cls, own1) O.sameAs(own1, own2) O.sameAs(cls, own2) - O.type(cls, namespaces['owl']+'#Thing') - O.type(own1, namespaces['owl']+'#Thing') - O.type(own2, namespaces['owl']+'#Thing') + O.type(cls, UR(namespaces['owl']+'#Thing')) + O.type(own1, UR(namespaces['owl']+'#Thing')) + O.type(own2, UR(namespaces['owl']+'#Thing')) sub = URIRef('a') obj = URIRef(namespaces['owl']+'#ObjectProperty') O.type(sub, obj) @@ -514,8 +514,8 @@ cls = BNode('anon') own1 = BNode('liist1') O.sameAs(cls, own1) - O.type(cls, namespaces['owl']+'#Thing') - O.type(own1, namespaces['owl']+'#Thing') + O.type(cls, UR(namespaces['owl']+'#Thing')) + O.type(own1, UR(namespaces['owl']+'#Thing')) sub = URIRef('a') obj = URIRef(namespaces['owl']+'#ObjectProperty') O.type(sub, obj) @@ -530,22 +530,22 @@ # predicate p cls = URIRef('cls') O = Ontology() - O.add((cls, namespaces['rdfs']+'#type', namespaces['owl']+'#Class' )) + O.add((cls, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Class'))) p = O.make_var(Property,URIRef('p')) p = URIRef('p') - O.add((p, namespaces['rdfs']+'#type', namespaces['owl']+'#ObjectProperty' )) + O.add((p, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#ObjectProperty'))) restr = BNode('anon') - O.add((restr, namespaces['rdfs']+'#type', namespaces['owl']+'#Restriction' )) - O.add((restr, namespaces['rdfs']+'#onProperty', p )) - O.add((cls, namespaces['rdfs']+'#subClassOf',restr )) - O.add((restr, namespaces['rdfs']+'#maxCardinality', 2 )) + O.add((restr, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Restriction') )) + O.add((restr, UR(namespaces['rdfs']+'#onProperty'), p )) + O.add((cls, UR(namespaces['rdfs']+'#subClassOf'),restr )) + O.add((restr, UR(namespaces['rdfs']+'#maxCardinality'), 2 )) restr2 = BNode('anon2') - O.add((restr2, namespaces['rdfs']+'#type', namespaces['owl']+'#Restriction' )) - O.add((restr2, namespaces['rdfs']+'#onProperty', p )) - O.add((cls, namespaces['rdfs']+'#subClassOf',restr2 )) - O.add((restr2, namespaces['rdfs']+'#minCardinality', 3 )) + O.add((restr2, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Restriction') )) + O.add((restr2, UR(namespaces['rdfs']+'#onProperty'), p )) + O.add((cls, UR(namespaces['rdfs']+'#subClassOf'),restr2 )) + O.add((restr2, UR(namespaces['rdfs']+'#minCardinality'), 3 )) O.attach_fd() py.test.raises(ConsistencyFailure, O.consistency) @@ -553,24 +553,24 @@ cls = URIRef('cls') cls2 = URIRef('cls2') O = Ontology() - O.add((cls, namespaces['rdfs']+'#type', namespaces['owl']+'#Class' )) - O.add((cls2, namespaces['rdfs']+'#type', namespaces['owl']+'#Class' )) + O.add((cls, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Class'))) + O.add((cls2, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Class'))) p = O.make_var(Property,URIRef('p')) p = URIRef('p') - O.add((p, namespaces['rdfs']+'#type', namespaces['owl']+'#ObjectProperty' )) + O.add((p, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#ObjectProperty'))) restr = BNode('anon') - O.add((restr, namespaces['rdfs']+'#type', namespaces['owl']+'#Restriction' )) - O.add((restr, namespaces['rdfs']+'#onProperty', p )) - O.add((cls, namespaces['rdfs']+'#subClassOf',restr )) - O.add((restr, namespaces['rdfs']+'#maxCardinality', 2 )) + O.add((restr, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Restriction'))) + O.add((restr, UR(namespaces['rdfs']+'#onProperty'), p )) + O.add((cls, UR(namespaces['rdfs']+'#subClassOf'),restr )) + O.add((restr, UR(namespaces['rdfs']+'#maxCardinality'), 2 )) restr2 = BNode('anon2') - O.add((restr2, namespaces['rdfs']+'#type', namespaces['owl']+'#Restriction' )) - O.add((restr2, namespaces['rdfs']+'#onProperty', p )) - O.add((cls, namespaces['rdfs']+'#subClassOf',restr2 )) - O.add((restr2, namespaces['rdfs']+'#minCardinality', 3 )) - O.add((cls2, namespaces['rdfs']+'#subClassOf', cls )) + O.add((restr2, UR(namespaces['rdfs']+'#type'), UR(namespaces['owl']+'#Restriction'))) + O.add((restr2, UR(namespaces['rdfs']+'#onProperty'), p )) + O.add((cls, UR(namespaces['rdfs']+'#subClassOf'),restr2 )) + O.add((restr2, UR(namespaces['rdfs']+'#minCardinality'), 3 )) + O.add((cls2, UR(namespaces['rdfs']+'#subClassOf'), cls )) O.attach_fd() py.test.raises(ConsistencyFailure, O.consistency) From auc at codespeak.net Wed Aug 2 11:37:33 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 2 Aug 2006 11:37:33 +0200 (CEST) Subject: [pypy-svn] r30881 - in pypy/dist/pypy/lib/logic: . basic_store computation_space Message-ID: <20060802093733.E27DF10083@code0.codespeak.net> Author: auc Date: Wed Aug 2 11:37:32 2006 New Revision: 30881 Removed: pypy/dist/pypy/lib/logic/basic_store/ pypy/dist/pypy/lib/logic/computation_space/ pypy/dist/pypy/lib/logic/oz-clp.txt pypy/dist/pypy/lib/logic/oz-dataflow-concurrency.txt Log: remove old obsolete unused cruft From auc at codespeak.net Wed Aug 2 11:38:25 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 2 Aug 2006 11:38:25 +0200 (CEST) Subject: [pypy-svn] r30882 - pypy/dist/pypy/module/_stackless Message-ID: <20060802093825.2376A10083@code0.codespeak.net> Author: auc Date: Wed Aug 2 11:38:24 2006 New Revision: 30882 Modified: pypy/dist/pypy/module/_stackless/clonable.py Log: make clone applevel available Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Wed Aug 2 11:38:24 2006 @@ -41,6 +41,13 @@ ec = self.space.getexecutioncontext() ec.subcontext_leave(self) + def w_clone(self): + try: + return InterpClonableCoroutine.clone(self) + except NotImplementedError: + raise OperationError(self.space.w_NotImplementedError, + self.space.wrap("clone() is only available in translated PyPy")) + def w_getcurrent(space): return space.wrap(ClonableCoroutine._get_state(space).current) w_getcurrent = staticmethod(w_getcurrent) @@ -219,6 +226,7 @@ __new__ = interp2app(ClonableCoroutine.descr_method__new__.im_func), _framestack = GetSetProperty(w_descr__framestack), getcurrent = interp2app(ClonableCoroutine.w_getcurrent), + clone = interp2app(ClonableCoroutine.w_clone), __reduce__ = interp2app(ClonableCoroutine.descr__reduce__, unwrap_spec=['self', ObjSpace]), __setstate__ = interp2app(ClonableCoroutine.descr__setstate__, From fijal at codespeak.net Wed Aug 2 14:41:32 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Aug 2006 14:41:32 +0200 (CEST) Subject: [pypy-svn] r30893 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060802124132.1F2D610088@code0.codespeak.net> Author: fijal Date: Wed Aug 2 14:41:31 2006 New Revision: 30893 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: Added my start date Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Wed Aug 2 14:41:31 2006 @@ -13,7 +13,7 @@ Michael Hudson ? ? Armin Rigo ? ? Holger Krekel 18-28 v/dublin on-campus (faxed form) -Maciej Fijalkowski ? ? +Maciej Fijalkowski 21-?/8 no idea ==================== ============== ======================== People on the following list were present at previous sprints: From antocuni at codespeak.net Wed Aug 2 14:53:55 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 2 Aug 2006 14:53:55 +0200 (CEST) Subject: [pypy-svn] r30894 - in pypy/dist/pypy/translator/cli: . src src/stub test Message-ID: <20060802125355.5823010088@code0.codespeak.net> Author: antocuni Date: Wed Aug 2 14:53:48 2006 New Revision: 30894 Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs pypy/dist/pypy/translator/cli/src/stub/main.il pypy/dist/pypy/translator/cli/support.py pypy/dist/pypy/translator/cli/test/test_constant.py Log: string literals with non-printable characters was not rendered properly, and the corresponding test was buggy because it didn't test anything due to constant folding. Both issues have been fixed. Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_os.cs Wed Aug 2 14:53:48 2006 @@ -163,7 +163,7 @@ private static FileMode get_file_mode(int flags) { if ((flags & O_APPEND) !=0 ) return FileMode.Append; - if ((flags & O_TRUNC) !=0 ) return FileMode.Truncate; + if ((flags & O_TRUNC) !=0 ) return FileMode.Create; if ((flags & O_CREAT) !=0 ) return FileMode.OpenOrCreate; return FileMode.Open; } @@ -189,9 +189,15 @@ } if (System.Environment.NewLine == "\r\n") - f = new CRLFFile(stream, reader, writer); + { + Console.Error.WriteLine("opening {0} for reading (with CRLF conversion)", name); + f = new CRLFFile(stream, reader, writer); + } else - f = new TextFile(stream, reader, writer); + { + Console.Error.WriteLine("opening {0} for reading (without CRLF conversion)", name); + f = new TextFile(stream, reader, writer); + } } fdcount++; @@ -208,10 +214,22 @@ public static int ll_os_write(int fd, string buffer) { + PrintString("ll_os_write", buffer); getfd(fd).Write(buffer); return buffer.Length; } + private static void PrintString(string source, string s) + { + Console.WriteLine(source); + Console.WriteLine(s); + Console.WriteLine("Length: {0}", s.Length); + for (int i=0; i Author: auc Date: Wed Aug 2 15:28:59 2006 New Revision: 30895 Modified: pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: transl. fixes Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Wed Aug 2 15:28:59 2006 @@ -2,9 +2,9 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import gateway -from pypy.objspace.cclp.types import W_Var, aliases +from pypy.objspace.cclp.types import W_Var, W_FailedValue, aliases from pypy.objspace.cclp.misc import w, v, ClonableCoroutine -from pypy.objspace.cclp.space import CSpace +#from pypy.objspace.cclp.space import CSpace scheduler = [] @@ -16,14 +16,14 @@ self.space = space self._main = ClonableCoroutine.w_getcurrent(space) # link top_level space to main coroutine - self.top_space = CSpace(space, self._main) - self._main.cspace = self.top_space + #self.top_space = CSpace(space, self._main) + #self._main.cspace = self.top_space # ... self._init_head(self._main) self._init_blocked() self._switch_count = 0 self._traced = {} - w (".. MAIN THREAD = ", str(id(self._main))) + w ("MAIN THREAD = ", str(id(self._main))) def get_threads(self): threads = [self._head] @@ -38,8 +38,10 @@ self._blocked_on = {} # var -> threads self._blocked_byneed = {} # var -> threads - def _init_head(self, coro): - self._head = coro + def _init_head(self, thread): + assert isinstance(thread, ClonableCoroutine) + self._head = thread + # for the reset case self._head._next = self._head._prev = self._head def _set_head(self, thread): @@ -67,19 +69,20 @@ assert isinstance(thread, ClonableCoroutine) assert isinstance(thread._next, ClonableCoroutine) assert isinstance(thread._prev, ClonableCoroutine) - if self._head is None: - thread._next = thread - thread._prev = thread - self._set_head(thread) - else: - r = self._head - l = r._prev - l._next = thread - r._prev = thread - thread._prev = l - thread._next = r +## if self._head is None: +## thread._next = thread +## thread._prev = thread +## self._set_head(thread) +## else: + r = self._head + l = r._prev + l._next = thread + r._prev = thread + thread._prev = l + thread._next = r def remove_thread(self, thread): + assert isinstance(thread, ClonableCoroutine) w(".. REMOVING", str(id(thread))) assert thread not in self._blocked del self._traced[thread] @@ -99,20 +102,20 @@ def schedule(self): to_be_run = self._select_next() + assert isinstance(to_be_run, ClonableCoroutine) w(".. SWITCHING", str(id(ClonableCoroutine.w_getcurrent(self.space))), "=>", str(id(to_be_run))) self._switch_count += 1 - assert isinstance(to_be_run, ClonableCoroutine) to_be_run.w_switch() def schedule_or_pass(self): to_be_run = self._select_next(dont_pass=False) + assert isinstance(to_be_run, ClonableCoroutine) curr = ClonableCoroutine.w_getcurrent(self.space) if to_be_run == curr: w(".. PASS") return w(".. SWITCHING", str(id(curr)), "=>", str(id(to_be_run))) self._switch_count += 1 - assert isinstance(to_be_run, ClonableCoroutine) to_be_run.w_switch() def _select_next(self, dont_pass=True): @@ -147,6 +150,9 @@ return count def display_head(self): + if we_are_translated(): + w("") + return curr = self._head v('Threads : [', '-'.join([str(id(curr)), str(curr in self._blocked)])) while curr._next != self._head: @@ -159,18 +165,18 @@ assert isinstance(thread, ClonableCoroutine) self._chain_insert(thread) - def add_to_blocked_on(self, w_var, uthread): - w(".. we BLOCK thread", str(id(uthread)), "on var", str(w_var)) + def add_to_blocked_on(self, w_var, thread): + w(".. we BLOCK thread", str(id(thread)), "on var", str(w_var)) assert isinstance(w_var, W_Var) - assert isinstance(uthread, ClonableCoroutine) - assert uthread not in self._blocked + assert isinstance(thread, ClonableCoroutine) + assert thread not in self._blocked if w_var in self._blocked_on: blocked = self._blocked_on[w_var] else: blocked = [] self._blocked_on[w_var] = blocked - blocked.append(uthread) - self._blocked[uthread] = True + blocked.append(thread) + self._blocked[thread] = True def unblock_on(self, w_var): v(".. we UNBLOCK threads dependants of var", str(w_var)) @@ -182,17 +188,17 @@ w(str([id(thr) for thr in blocked])) for thr in blocked: del self._blocked[thr] - def add_to_blocked_byneed(self, w_var, uthread): - w(".. we BLOCK BYNEED thread", str(id(uthread)), "on var", str(w_var)) + def add_to_blocked_byneed(self, w_var, thread): + w(".. we BLOCK BYNEED thread", str(id(thread)), "on var", str(w_var)) assert isinstance(w_var, W_Var) - assert isinstance(uthread, ClonableCoroutine) + assert isinstance(thread, ClonableCoroutine) if w_var in self._blocked_byneed: blocked = self._blocked_byneed[w_var] else: blocked = [] self._blocked_byneed[w_var] = blocked - blocked.append(uthread) - self._blocked[uthread] = True + blocked.append(thread) + self._blocked[thread] = True def unblock_byneed_on(self, w_var): v(".. we UNBLOCK BYNEED dependants of var", str(w_var)) @@ -216,6 +222,8 @@ self._traced[thread] = lvars def dirty_traced_vars(self, thread, failed_value): + assert isinstance(thread, ClonableCoroutine) + assert isinstance(failed_value, W_FailedValue) w(".. DIRTYING traced vars") for w_var in self._traced[thread]: if self.space.is_true(self.space.is_free(w_var)): @@ -226,20 +234,22 @@ def reset_scheduler(space): "garbage collection of threads might pose some problems" scheduler[0] = Scheduler(space) + scheduler[0]._check_initial_conditions() app_reset_scheduler = gateway.interp2app(reset_scheduler) def sched_info(space): sched = scheduler[0] w_ret = space.newdict([]) - space.setitem(w_ret, space.wrap('switches'), space.wrap(sched._switch_count)) - space.setitem(w_ret, space.wrap('threads'), - space.wrap([id(th) for th in sched.get_threads()])) - space.setitem(w_ret, space.wrap('blocked'), - space.wrap([id(th) for th in sched._blocked.keys()])) - space.setitem(w_ret, space.wrap('blocked_on'), - space.wrap([id(th) for th in sched._blocked_on.keys()])) - space.setitem(w_ret, space.wrap('blocked_byneed'), - space.wrap([id(th) for th in sched._blocked_byneed.keys()])) + if not we_are_translated(): # XXX and otherwise, WTF ??? + space.setitem(w_ret, space.wrap('switches'), space.wrap(sched._switch_count)) + space.setitem(w_ret, space.wrap('threads'), + space.wrap([id(th) for th in sched.get_threads()])) + space.setitem(w_ret, space.wrap('blocked'), + space.wrap([id(th) for th in sched._blocked.keys()])) + space.setitem(w_ret, space.wrap('blocked_on'), + space.wrap([id(th) for th in sched._blocked_on.keys()])) + space.setitem(w_ret, space.wrap('blocked_byneed'), + space.wrap([id(th) for th in sched._blocked_byneed.keys()])) return w_ret app_sched_info = gateway.interp2app(sched_info) Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Wed Aug 2 15:28:59 2006 @@ -16,7 +16,7 @@ # coro init coro = ClonableCoroutine(space) # computation space is the same as in the parent - coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace + #coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace # feed the coro w_Future = W_Future(space) thunk = FutureThunk(space, w_callable, args, w_Future, coro) @@ -36,7 +36,7 @@ # coro init coro = ClonableCoroutine(space) # computation space is the same as in the parent - coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace + #coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) coro.bind(thunk) w("STACKLET", str(id(coro))) Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Wed Aug 2 15:28:59 2006 @@ -24,7 +24,7 @@ "a read-only-by-its-consummer variant of logic. var" def __init__(w_self, space): W_Var.__init__(w_self, space) - w_self.client = ClonableCoroutine.w_getcurrent(space) + w_self._client = ClonableCoroutine.w_getcurrent(space) w("FUT", str(w_self)) #-- Exception types ---------------------------------------- @@ -36,10 +36,53 @@ def __init__(w_self, exc): w_self.exc = exc +#-- Something to hold the ring of coros -------------------------------- + +## class Triple(object): + +## def __init__(self, thread): +## assert isinstance(thread, ClonableCoroutine) +## self._thread = thread +## self._prev = self._next = self + +## def get_next(self): +## return self._next + +## def get_prev(self): +## return self._prev + +## def set_next(self, triple): +## assert isinstance(triple, Triple) +## self._next = triple + +## def set_prev(self, triple): +## assert isinstance(triple, Triple) +## self._prev = triple + +## next = property(get_next, set_next) +## prev = property(get_prev, set_prev) + +## def insert_before(self, triple): +## assert isinstance(triple, Triple) +## before = self.prev +## # ... +## before.next = triple +## triple.prev = before +## # ... +## self.prev = triple +## triple.next = self + +## def __str__(self): +## curr = self +## out = ['[', str(id(self._thread))] +## while curr != self: +## curr = self.next +## out.append(str(id(curr._thread))) +## return ''.join(out) + #-- Misc --------------------------------------------------- def deref(space, w_var): - #XXX kill me ? "gets the value/next alias of a variable" assert isinstance(w_var, W_Var) return w_var.w_bound_to Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Wed Aug 2 15:28:59 2006 @@ -179,7 +179,7 @@ def bind__Future_Root(space, w_fut, w_obj): #v("future val", str(id(w_fut))) - if w_fut.client == ClonableCoroutine.w_getcurrent(space): + if w_fut._client == ClonableCoroutine.w_getcurrent(space): raise_future_binding(space) return bind__Var_Root(space, w_fut, w_obj) # call-next-method ? @@ -201,14 +201,14 @@ def bind__Future_Var(space, w_fut, w_var): #v("future var") - if w_fut.client == ClonableCoroutine.w_getcurrent(space): + if w_fut._client == ClonableCoroutine.w_getcurrent(space): raise_future_binding(space) return bind__Var_Var(space, w_fut, w_var) def bind__Var_Future(space, w_var, w_fut): if space.is_true(space.is_bound(w_fut)): #XXX write a test for me ! return bind__Var_Root(space, w_var, deref(space, w_fut)) - if w_fut.client == ClonableCoroutine.w_getcurrent(space): + if w_fut._client == ClonableCoroutine.w_getcurrent(space): raise_future_binding(space) return bind__Var_Var(space, w_var, w_fut) #and for me ... Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 2 15:28:59 2006 @@ -391,6 +391,8 @@ assert T == 45 assert len(sched_info()['blocked_byneed']) == 1 reset_scheduler() + assert len(sched_info()['blocked_byneed']) == 0 + assert len(sched_info()['threads']) == 1 def test_wait_two(self): From mwh at codespeak.net Wed Aug 2 16:00:49 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 2 Aug 2006 16:00:49 +0200 (CEST) Subject: [pypy-svn] r30898 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060802140049.B91BC10034@code0.codespeak.net> Author: mwh Date: Wed Aug 2 16:00:48 2006 New Revision: 30898 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: my dates Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Wed Aug 2 16:00:48 2006 @@ -10,7 +10,7 @@ ==================== ============== ======================== Anders Chrigstroem 20-28/8 ? Samuele Pedroni 20-28/8 ? -Michael Hudson ? ? +Michael Hudson 20-27 no idea Armin Rigo ? ? Holger Krekel 18-28 v/dublin on-campus (faxed form) Maciej Fijalkowski 21-?/8 no idea From mwh at codespeak.net Wed Aug 2 16:03:31 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 2 Aug 2006 16:03:31 +0200 (CEST) Subject: [pypy-svn] r30899 - pypy/dist/pypy/translator/stackless Message-ID: <20060802140331.0D43610089@code0.codespeak.net> Author: mwh Date: Wed Aug 2 16:03:31 2006 New Revision: 30899 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: revert r30860. it's makes confusing code even more confusing and breaks the build, occasionally. will try to do something cleaner. Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Wed Aug 2 16:03:31 2006 @@ -449,7 +449,6 @@ assert self.curr_graph is None self.curr_graph = graph self.curr_graph_save_blocks = {} - self.curr_graph_resume_blocks = {} if SAVE_STATISTICS: self.stats.cur_rp_exact_types = {} self.stats.cur_rp_erased_types = {} @@ -483,7 +482,6 @@ self.curr_graph = None self.curr_graph_save_blocks = None - self.curr_graph_resume_blocks = None def ops_read_global_state_field(self, targetvar, fieldname): ops = [] @@ -521,12 +519,8 @@ not_resuming_link.llexitcase = -1 resuming_links = [] for resume_index, resume_block in enumerate(self.resume_blocks): - if resume_block.inputargs: - args = [var_resume_state] - else: - args = [] resuming_links.append( - model.Link(args, resume_block, resume_index)) + model.Link([], resume_block, resume_index)) resuming_links[-1].llexitcase = resume_index new_start_block.exitswitch = var_resume_state @@ -903,111 +897,68 @@ def _generate_resume_block(self, varsinfieldorder, frame_type, var_result, links_to_resumption): - typekey = [v.concretetype for v in varsinfieldorder] - linkkey = [(link.target, link.exitcase) for link in links_to_resumption[1:]] - key = tuple([var_result.concretetype] + typekey + linkkey) - if key in self.curr_graph_resume_blocks: - newblock, switchblock, newargs, newresult = self.curr_graph_resume_blocks[key] - if switchblock is None: - newblock.inputargs = [varoftype(lltype.Signed)] - switchblock = unsimplify.insert_empty_block(None, newblock.exits[0], []) - newblock.exits[0].args.append(newblock.inputargs[0]) - switchblock.inputargs.append(varoftype(lltype.Signed)) - switchblock.exitswitch = switchblock.inputargs[-1] - link, = switchblock.exits - link.exitcase = link.llexitcase = self.resume_blocks.index(newblock) - mapping = {} - for i in range(len(newblock.exits[0].args)): - mapping[newblock.exits[0].args[i]] = switchblock.inputargs[i] - if newresult in mapping: - newresult = mapping[newresult] - newnewargs = [] - for arg in newargs: - newnewargs.append(mapping[arg]) - newargs = newnewargs - self.curr_graph_resume_blocks[key] = newblock, switchblock, newnewargs, newresult - oldlink = links_to_resumption[0] - varmap = {} - for old, new in zip(varsinfieldorder, newargs): - varmap[old] = new - varmap[var_result] = newresult - def rename(arg): - if isinstance(arg, model.Variable): + newblock = model.Block([]) + newargs = [] + llops = LowLevelOpList() + llops.genop("setfield", + [self.ll_global_state, + self.c_restart_substate_name, + self.c_minus_one]) + frame_top = varoftype(lltype.Ptr(frame_type)) + llops.extend(self.ops_read_global_state_field(frame_top, "top")) + llops.genop("setfield", + [self.ll_global_state, + self.c_inst_top_name, + self.c_null_state]) + varmap = {} + fielditer = iter(frame_type._names[1:]) + for arg in varsinfieldorder: + assert arg is not var_result + t = storage_type(arg.concretetype) + if t is lltype.Void: + v_newarg = model.Constant(None, lltype.Void) + else: + fname = model.Constant(fielditer.next(), lltype.Void) + assert frame_type._flds[fname.value] is t + v_newarg = llops.genop('getfield', [frame_top, fname], + resulttype = t) + v_newarg = gen_cast(llops, arg.concretetype, v_newarg) + varmap[arg] = v_newarg + + rettype = storage_type(var_result.concretetype) + getretval = self.fetch_retvals[rettype] + retval = llops.genop("direct_call", [getretval], + resulttype = rettype) + varmap[var_result] = retval + + newblock.operations.extend(llops) + + def rename(arg): + if isinstance(arg, model.Variable): + if arg in varmap: return varmap[arg] else: - return arg - newlink = oldlink.copy(rename) - newlink.exitcase = newlink.llexitcase = len(self.resume_blocks) - switchblock.recloseblock(*(switchblock.exits + (newlink,))) - rettype = newresult.concretetype - retval = newresult - retlink = newlink + assert arg in [l.last_exception, l.last_exc_value] + r = unsimplify.copyvar(None, arg) + varmap[arg] = r + return r + else: + return arg + + newblock.closeblock(*[l.copy(rename) + for l in links_to_resumption]) + # this check is a bit implicit! + if len(links_to_resumption) > 1: + newblock.exitswitch = model.c_last_exception else: - newblock = model.Block([]) - newargs = [] - llops = LowLevelOpList() - llops.genop("setfield", - [self.ll_global_state, - self.c_restart_substate_name, - self.c_minus_one]) - frame_top = varoftype(lltype.Ptr(frame_type)) - llops.extend(self.ops_read_global_state_field(frame_top, "top")) - llops.genop("setfield", - [self.ll_global_state, - self.c_inst_top_name, - self.c_null_state]) - varmap = {} - newargs = [] - fielditer = iter(frame_type._names[1:]) - for arg in varsinfieldorder: - assert arg is not var_result - t = storage_type(arg.concretetype) - if t is lltype.Void: - v_newarg = model.Constant(None, lltype.Void) - else: - fname = model.Constant(fielditer.next(), lltype.Void) - assert frame_type._flds[fname.value] is t - v_newarg = llops.genop('getfield', [frame_top, fname], - resulttype = t) - v_newarg = gen_cast(llops, arg.concretetype, v_newarg) - newargs.append(v_newarg) - varmap[arg] = v_newarg - - rettype = storage_type(var_result.concretetype) - getretval = self.fetch_retvals[rettype] - retval = llops.genop("direct_call", [getretval], - resulttype = rettype) - varmap[var_result] = retval - - newblock.operations.extend(llops) - - def rename(arg): - if isinstance(arg, model.Variable): - if arg in varmap: - return varmap[arg] - else: - assert arg in [l.last_exception, l.last_exc_value] - r = unsimplify.copyvar(None, arg) - varmap[arg] = r - return r - else: - return arg + newblock.exitswitch = None - newblock.closeblock(*[l.copy(rename) - for l in links_to_resumption]) - # this check is a bit implicit! - if len(links_to_resumption) > 1: - newblock.exitswitch = model.c_last_exception - else: - newblock.exitswitch = None - self.curr_graph_resume_blocks[key] = newblock, None, newargs, retval - retlink = newblock.exits[0] - if SAVE_STATISTICS: - self.stats.resumeops += len(newblock.operations) - if var_result.concretetype != rettype: - self.insert_return_conversion(retlink, var_result.concretetype, retval) + self.insert_return_conversion( + newblock.exits[0], var_result.concretetype, retval) + if SAVE_STATISTICS: + self.stats.resumeops += len(newblock.operations) return newblock def generate_restart_infos(self, graph): From auc at codespeak.net Wed Aug 2 16:09:12 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 2 Aug 2006 16:09:12 +0200 (CEST) Subject: [pypy-svn] r30901 - pypy/dist/pypy/objspace/cclp Message-ID: <20060802140912.A72711008E@code0.codespeak.net> Author: auc Date: Wed Aug 2 16:09:10 2006 New Revision: 30901 Modified: pypy/dist/pypy/objspace/cclp/scheduler.py Log: remove dead code, transl. fix Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Wed Aug 2 16:09:10 2006 @@ -55,13 +55,9 @@ assert self._head not in self._blocked_on assert self._head not in self._blocked_byneed except: - self.display_head() - w("BLOCKED", str(self._blocked)) - all = {} - all.update(self._blocked_on) - all.update(self._blocked_byneed) - w(str(all)) - raise + #XXX give sched_info maybe + raise OperationError(self.space.w_RuntimeError, + self.space.wrap("scheduler is in an incoherent state")) def _chain_insert(self, thread): assert thread._next is thread @@ -69,11 +65,6 @@ assert isinstance(thread, ClonableCoroutine) assert isinstance(thread._next, ClonableCoroutine) assert isinstance(thread._prev, ClonableCoroutine) -## if self._head is None: -## thread._next = thread -## thread._prev = thread -## self._set_head(thread) -## else: r = self._head l = r._prev l._next = thread From ale at codespeak.net Wed Aug 2 16:51:40 2006 From: ale at codespeak.net (ale at codespeak.net) Date: Wed, 2 Aug 2006 16:51:40 +0200 (CEST) Subject: [pypy-svn] r30909 - pypy/dist/pypy/lib/pyontology Message-ID: <20060802145140.79D3810082@code0.codespeak.net> Author: ale Date: Wed Aug 2 16:51:39 2006 New Revision: 30909 Modified: pypy/dist/pypy/lib/pyontology/pyontology.py Log: more fidling with Literals Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Wed Aug 2 16:51:39 2006 @@ -7,7 +7,7 @@ import sys, py import time log = py.log.Producer("Pyontology") -#from pypy.tool.ansi_print import ansi_log +from pypy.tool.ansi_print import ansi_log py.log.setconsumer("Pyontology", None) @@ -116,6 +116,7 @@ if ('owl_Thing' in variables.keys() and isinstance(self, ClassDomain) and self.getValues() == []): variables[self.name].setValues(variables['owl_Thing'].getValues()) + log.finish("setting the domain %s to all individuals %r"%(self.name,variables[self.name])) variables.update(self.domains) glob_constraints.extend(self.in_constraint) assert len([x for x in glob_constraints if type(x)==list])==0 @@ -435,9 +436,13 @@ # Set the values of the property p to o self.type(s, Thing_uri) sub = self.make_var(Thing, s) - obj = self.make_var(Thing, o) + if type(o) == URIRef: + obj = self.make_var(Thing, o) + val = Individual(obj,o) + else: + val = o propdom = self.variables[avar] - res = propdom.addValue(Individual(sub,s),Individual(obj,o)) + res = propdom.addValue(Individual(sub,s), val) def resolve_item(self, item): item_as_subject = self.graph.triples((item, None, None)) @@ -612,15 +617,19 @@ def oneOf(self, s, var): # Oneof is used to generate a fixed class. The elements of the class # are exactly the ones in the list. - # Can be used to define an enumerated datatype as well + # Can be used to define an enumerated datatype as well. + # The memebers of the list can be Urirefs (Individuals) or Literals var = self.flatten_rdf_list(var) svar = self.make_var(FixedClassDomain, s) res = self.variables[var].getValues() - for ind in res: - self.make_var(Thing, ind) - self.variables[svar].setValues([ - Individual(self.make_var(None,x),x) for x in res]) - + if type(res[0]) == URIRef: + self.variables[svar].setValues([ + Individual(self.make_var(Thing, x), x) for x in res]) + for i in res: + self.type(i, Thing_uri) + else: + self.variables[svar].setValues(res) + def unionOf(self,s, var): var = self.flatten_rdf_list(var) vals = self.variables[var].getValues() @@ -744,7 +753,7 @@ def Hasvalue(cls ,prop, val): var = "%s_%s_hasvalue" %(cls, prop.name) dom = {var : fd(prop.getValues( ))} - cons = Expression([cls, var], " %s[1].cmp(%s) and %s.cmp( %s[0])" %( var, val, cls, var)) + cons = Expression([cls, var], " %s[1] == %s and %s.cmp( %s[0])" %( var, val, cls, var)) log("HASVALUE %r %r"%(prop.getValues(),dom)) return dom, [cons] From fijal at codespeak.net Wed Aug 2 17:16:05 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Aug 2006 17:16:05 +0200 (CEST) Subject: [pypy-svn] r30911 - pypy/dist/pypy/rpython/ootypesystem Message-ID: <20060802151605.E5BDE10082@code0.codespeak.net> Author: fijal Date: Wed Aug 2 17:16:04 2006 New Revision: 30911 Modified: pypy/dist/pypy/rpython/ootypesystem/bltregistry.py Log: Fixed decorator. Modified: pypy/dist/pypy/rpython/ootypesystem/bltregistry.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/bltregistry.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/bltregistry.py Wed Aug 2 17:16:04 2006 @@ -63,19 +63,25 @@ def described(cls, retval=None, args={}): def decorator(func): code = func.func_code - assert(code.co_argcount < len(func.func_defaults) + len(args), "Not enough information for describing method") + if not func.func_defaults: + defs = [] + else: + defs = func.func_defaults - for arg in xrange(1, code.co_argcount - len(func.func_defaults)): + + assert(code.co_argcount < len(defs) + len(args), "Not enough information for describing method") + + for arg in xrange(1, code.co_argcount - len(defs)): assert code.co_varnames[arg] in args, "Don't have example for arg %s" % code.co_varnames[arg] arg_pass = [] - start_pos = code.co_argcount - len(func.func_defaults) + start_pos = code.co_argcount - len(defs) for arg in xrange(1, code.co_argcount): varname = code.co_varnames[arg] if varname in args: arg_pass.append((varname, args[varname])) else: - arg_pass.append((varname, func.func_defaults[arg - start_pos])) + arg_pass.append((varname, defs[arg - start_pos])) cls._methods[func.__name__] = MethodDesc(arg_pass, retval) return func return decorator From fijal at codespeak.net Wed Aug 2 20:40:43 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Aug 2006 20:40:43 +0200 (CEST) Subject: [pypy-svn] r30916 - pypy/dist/pypy/bin Message-ID: <20060802184043.5F2E410083@code0.codespeak.net> Author: fijal Date: Wed Aug 2 20:40:41 2006 New Revision: 30916 Modified: pypy/dist/pypy/bin/jscompile.py Log: It's way better now (debug messages + debugger + viewer), but still, some stuff gets constant folded, which should not be constant folded. Modified: pypy/dist/pypy/bin/jscompile.py ============================================================================== --- pypy/dist/pypy/bin/jscompile.py (original) +++ pypy/dist/pypy/bin/jscompile.py Wed Aug 2 20:40:41 2006 @@ -7,9 +7,10 @@ import sys from pypy.translator.js.test.runtest import compile_function -from pypy.translator.translator import TranslationContext +#from pypy.translator.translator import TranslationContext +from pypy.translator.driver import TranslationDriver from pypy.translator.js.js import JS -from pypy.tool.error import AnnotatorError, FlowingError +from pypy.tool.error import AnnotatorError, FlowingError, debug class FunctionNotFound(Exception): pass @@ -42,18 +43,13 @@ exec(source_ssf) in globals() #fn = compile_function([mod.__dict__[f_name] for f_name in function_names], [[] for i in function_names]) # now we gonna just cut off not needed function + driver = TranslationDriver() try: - t = TranslationContext() - t.buildannotator().build_types(some_strange_function_which_will_never_be_called, []) - t.buildrtyper(type_system="ootype").specialize() - j = JS(t, [mod.__dict__[f_name] for f_name in function_names], False) - j.write_source() - f = open("out.js", "w") - f.write(j.tmpfile.open().read()) - f.close() - except (AnnotatorError, FlowingError), e: + driver.setup(some_strange_function_which_will_never_be_called, []) + driver.proceed(["compile_js"]) + except Exception, e: # do something nice with it - raise + debug(driver) if __name__ == '__main__': _main(sys.argv) From bea at codespeak.net Thu Aug 3 11:13:31 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 3 Aug 2006 11:13:31 +0200 (CEST) Subject: [pypy-svn] r30923 - pypy/dist/pypy/doc Message-ID: <20060803091331.420351007C@code0.codespeak.net> Author: bea Date: Thu Aug 3 11:13:27 2006 New Revision: 30923 Modified: pypy/dist/pypy/doc/extradoc.txt Log: update with my talks as references for 14.3. As Holger said - all talks are to be updated to this file as soon as they are done - author/presenters are responsible for this (I will chase them) Modified: pypy/dist/pypy/doc/extradoc.txt ============================================================================== --- pypy/dist/pypy/doc/extradoc.txt (original) +++ pypy/dist/pypy/doc/extradoc.txt Thu Aug 3 11:13:27 2006 @@ -5,6 +5,17 @@ Talks and Presentations ---------------------------------- +* `Trouble in Paradise: the Open Source Project PyPy, + EU-funding and Agile Practices`_ talk, by Bea During at + Agile 2006 (experience report). + +* `Sprint Driven Development, Agile Methodologies in a + Distributed Open Source Project (PyPy)`_ talk, by Bea During + at XP 2006 (experience report). + +* `Kill -1: process refactoring in the PyPy project`_ talk, by Bea During + at the Agile track/Europython 2006. + * `PyPy intro`_ talk, given by Michael Hudson at ACCU 2006 (April), also stating the status of the project. @@ -14,6 +25,9 @@ * `PyPy - the new Python implementation on the block`_ talk, given by Carl Friedrich Bolz and Holger Krekel at the 22nd Chaos Communication Conference in Berlin, Dec. 2005. + +* `Open Source, EU-Funding and Agile Methods`_ talk, given by Holger Krekel + and Bea During at the 22nd Chaos Communication Conference in Berlin, Dec. 2005 * oscon2003-paper_ an early paper presented at Oscon 2003 describing what the PyPy project is about and why you should care. @@ -37,6 +51,11 @@ .. _`EU funding for FOSS`: http://codespeak.net/pypy/extradoc/talk/2004-21C3-pypy-EU-hpk.pdf .. _`py lib slides`: http://codespeak.net/pypy/extradoc/talk/2005-pycon-py.pdf .. _`PyCon 2005`: http://codespeak.net/pypy/extradoc/talk/pypy-talk-pycon2005/README.html +.. _ `Trouble in Paradise`: http://codespeak.net/pypy/extradoc/talk/during-oss-sprints_talk.pdf +.. _ `Sprint Driven Development`:http://codespeak.net/pypy/extradoc/talk/during-xp2006-sprints.pdf +.. _ `Kill -1`:http://codespeak.net/pypy/extradoc/talk/kill_1_agiletalk.pdf +.. _ `Open Source, EU-Funding and Agile Methods`:http://codespeak.net/pypy/extradoc/talk/agility.pdf + Related projects ---------------------------------- From cfbolz at codespeak.net Thu Aug 3 11:21:37 2006 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 3 Aug 2006 11:21:37 +0200 (CEST) Subject: [pypy-svn] r30924 - pypy/dist/pypy/doc Message-ID: <20060803092137.965DE1007C@code0.codespeak.net> Author: cfbolz Date: Thu Aug 3 11:21:36 2006 New Revision: 30924 Modified: pypy/dist/pypy/doc/extradoc.txt Log: fix bea's rest issues Modified: pypy/dist/pypy/doc/extradoc.txt ============================================================================== --- pypy/dist/pypy/doc/extradoc.txt (original) +++ pypy/dist/pypy/doc/extradoc.txt Thu Aug 3 11:21:36 2006 @@ -5,16 +5,16 @@ Talks and Presentations ---------------------------------- -* `Trouble in Paradise: the Open Source Project PyPy, - EU-funding and Agile Practices`_ talk, by Bea During at - Agile 2006 (experience report). - -* `Sprint Driven Development, Agile Methodologies in a - Distributed Open Source Project (PyPy)`_ talk, by Bea During - at XP 2006 (experience report). +* `Trouble in Paradise`_: the Open Source Project PyPy, + EU-funding and Agile Practices talk, by Bea During at + Agile 2006 (experience report). + +* `Sprint Driven Development`_, Agile Methodologies in a + Distributed Open Source Project (PyPy) talk, by Bea During + at XP 2006 (experience report). -* `Kill -1: process refactoring in the PyPy project`_ talk, by Bea During - at the Agile track/Europython 2006. +* `Kill -1`_: process refactoring in the PyPy project talk, by Bea During + at the Agile track/Europython 2006. * `PyPy intro`_ talk, given by Michael Hudson at ACCU 2006 (April), also stating the status of the project. @@ -27,7 +27,7 @@ 22nd Chaos Communication Conference in Berlin, Dec. 2005. * `Open Source, EU-Funding and Agile Methods`_ talk, given by Holger Krekel - and Bea During at the 22nd Chaos Communication Conference in Berlin, Dec. 2005 + and Bea During at the 22nd Chaos Communication Conference in Berlin, Dec. 2005 * oscon2003-paper_ an early paper presented at Oscon 2003 describing what the PyPy project is about and why you should care. @@ -51,10 +51,10 @@ .. _`EU funding for FOSS`: http://codespeak.net/pypy/extradoc/talk/2004-21C3-pypy-EU-hpk.pdf .. _`py lib slides`: http://codespeak.net/pypy/extradoc/talk/2005-pycon-py.pdf .. _`PyCon 2005`: http://codespeak.net/pypy/extradoc/talk/pypy-talk-pycon2005/README.html -.. _ `Trouble in Paradise`: http://codespeak.net/pypy/extradoc/talk/during-oss-sprints_talk.pdf -.. _ `Sprint Driven Development`:http://codespeak.net/pypy/extradoc/talk/during-xp2006-sprints.pdf -.. _ `Kill -1`:http://codespeak.net/pypy/extradoc/talk/kill_1_agiletalk.pdf -.. _ `Open Source, EU-Funding and Agile Methods`:http://codespeak.net/pypy/extradoc/talk/agility.pdf +.. _`Trouble in Paradise`: http://codespeak.net/pypy/extradoc/talk/during-oss-sprints_talk.pdf +.. _`Sprint Driven Development`: http://codespeak.net/pypy/extradoc/talk/during-xp2006-sprints.pdf +.. _`Kill -1`: http://codespeak.net/pypy/extradoc/talk/kill_1_agiletalk.pdf +.. _`Open Source, EU-Funding and Agile Methods`: http://codespeak.net/pypy/extradoc/talk/agility.pdf Related projects From hpk at codespeak.net Thu Aug 3 13:33:56 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 3 Aug 2006 13:33:56 +0200 (CEST) Subject: [pypy-svn] r30936 - pypy/extradoc/eu-report Message-ID: <20060803113356.7E97A10080@code0.codespeak.net> Author: hpk Date: Thu Aug 3 13:33:53 2006 New Revision: 30936 Added: pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-FINAL-2006-08-03.pdf - copied unchanged from r30935, pypy/eu-tracking/deliverable/D14_3_Report_About_Milestone_Phase_2/D14.3-deliverablereport.pdf Log: adding final report version From hpk at codespeak.net Thu Aug 3 13:39:40 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 3 Aug 2006 13:39:40 +0200 (CEST) Subject: [pypy-svn] r30938 - pypy/dist/pypy/doc Message-ID: <20060803113940.3AF0E1007C@code0.codespeak.net> Author: hpk Date: Thu Aug 3 13:39:39 2006 New Revision: 30938 Modified: pypy/dist/pypy/doc/index-report.txt Log: link to final D14.3 report Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Thu Aug 3 13:39:39 2006 @@ -12,6 +12,10 @@ Reports of 2006 =============== +`D14.3 Report about Milestone/Phase 2`_ is the final report about +the second phase of the EU project, summarizing and detailing technical, +research, dissemination and community aspects. Feedback is very welcome! + `Draft D07.1 Massive Parallelism and Translation Aspects`_ is an interim version of a report about PyPy's optimization efforts, garbage collectors and massive parallelism (stackless) features. The report is not complete @@ -23,6 +27,7 @@ as the work going on to tie semantic web technologies and PyPy together. The report is still a draft, all feedback for it is welcome. + Reports of 2005 =============== @@ -65,3 +70,4 @@ .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf .. _`Draft D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-INTERIM-2006-07-31.pdf .. _`Draft D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf +.. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-FINAL-2006-08-03.pdf From rhymes at codespeak.net Thu Aug 3 14:40:54 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 3 Aug 2006 14:40:54 +0200 (CEST) Subject: [pypy-svn] r30944 - in pypy/dist/pypy/module: _ssl/test fcntl/test mmap/test rctime/test Message-ID: <20060803124054.31F6A1007C@code0.codespeak.net> Author: rhymes Date: Thu Aug 3 14:40:49 2006 New Revision: 30944 Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py pypy/dist/pypy/module/fcntl/test/test_fcntl.py pypy/dist/pypy/module/mmap/test/test_mmap.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: Windows is not supported. Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/test/test_ssl.py (original) +++ pypy/dist/pypy/module/_ssl/test/test_ssl.py Thu Aug 3 14:40:49 2006 @@ -2,6 +2,9 @@ from pypy.conftest import gettestobjspace import os +if os.name == "nt": + skip("Windows is not supported") + class AppTestSSL: def setup_class(cls): space = gettestobjspace(usemodules=('_ssl',)) @@ -132,4 +135,4 @@ data = ss.read(10) assert isinstance(data, str) assert len(data) == 10 - s.close() \ No newline at end of file + s.close() Modified: pypy/dist/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/test/test_fcntl.py Thu Aug 3 14:40:49 2006 @@ -2,6 +2,9 @@ from pypy.conftest import gettestobjspace import os +if os.name == "nt": + skip("fcntl module is not available on Windows") + def teardown_module(mod): for i in "abcde": if os.path.exists(i): Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Thu Aug 3 14:40:49 2006 @@ -2,6 +2,9 @@ from pypy.conftest import gettestobjspace import os +if os.name == "nt": + skip("Windows is not supported") + def teardown_module(mod): if os.path.exists("foo"): os.unlink("foo") Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Thu Aug 3 14:40:49 2006 @@ -1,5 +1,9 @@ from py.test import raises, skip from pypy.conftest import gettestobjspace +import os + +if os.name == "nt": + skip("Windows is not supported") class AppTestRCTime: def setup_class(cls): From antocuni at codespeak.net Thu Aug 3 14:55:18 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 3 Aug 2006 14:55:18 +0200 (CEST) Subject: [pypy-svn] r30945 - in pypy/dist/pypy/translator/cli: src test Message-ID: <20060803125518.802F01006F@code0.codespeak.net> Author: antocuni Date: Thu Aug 3 14:55:11 2006 New Revision: 30945 Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/test/test_builtin.py Log: This should fix the support for binary/text files both on unix and windows. There were many problems: - the .NET framework seems not to have any support for CRLF terminated text files, so we need to handle them explicitly; - we can't use StreamReader and StreamWriter because the are designed to handle unicode strings: the default encoding is UTF8, which breaks things when writing chars > 0x7F. The ASCII encoding doesn't handle chars > 0x7F too, so we can't use it. The solution is to directly read bytes and casting them to chars. Moreover, we need to pay attention on Windows when opening files in text mode, because we need the CRLF conversion. We will probably need a similar thing for MAC < OS9 in the future. Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_os.cs Thu Aug 3 14:55:11 2006 @@ -13,6 +13,7 @@ string Read(int count); } + // this class is used only for stdin/stdout/stderr class TextFile: IFile { private FileStream stream; @@ -46,41 +47,45 @@ } } - class CRLFFile: IFile + class CRLFTextFile: IFile { private FileStream stream; - private TextWriter writer; - private TextReader reader; - public CRLFFile(FileStream stream, TextReader reader, TextWriter writer) + public CRLFTextFile(FileStream stream) { this.stream = stream; - this.writer = writer; - this.reader = reader; } - + public FileStream GetStream() { return stream; } - + public void Write(string buffer) { - Debug.Assert(writer != null); // XXX: raise OSError? - writer.Write(buffer); - writer.Flush(); + foreach(char ch in buffer) { + if (ch == '\n') + stream.WriteByte((byte)'\r'); + stream.WriteByte((byte)ch); + } + stream.Flush(); // XXX: should we flush every time or not? } - + public string Read(int count) { - Debug.Assert(reader != null); // XXX: raise OSError? System.Text.StringBuilder builder = new System.Text.StringBuilder(count); + bool pending_CR = false; while (count-- > 0) { - int ch = reader.Read(); + int ch = stream.ReadByte(); if (ch == -1) break; - if (ch == '\r' && reader.Peek() == '\n') - ch = reader.Read(); - builder.Append((char)ch); + else if (ch == '\r') + pending_CR = true; + else { + if (pending_CR && ch != '\n') + builder.Append('\r'); + builder.Append((char)ch); + pending_CR = false; + } } return builder.ToString(); } @@ -136,6 +141,7 @@ static ll_os() { FileDescriptors = new Dictionary(); + // XXX: what about CRLF conversion for stdin, stdout and stderr? FileDescriptors[0] = new TextFile(null, Console.In, null); FileDescriptors[1] = new TextFile(null, null, Console.Out); FileDescriptors[2] = new TextFile(null, null, Console.Error); @@ -175,30 +181,13 @@ FileStream stream = new FileStream(name, f_mode, f_access); IFile f; - if ((flags & O_BINARY) != 0) + // - on Unix there is no difference between text and binary modes + // - on Windows text mode means that we should convert '\n' from and to '\r\n' + // - on Mac < OS9 text mode means that we should convert '\n' from and to '\r' -- XXX: TODO! + if ((flags & O_BINARY) == 0 && System.Environment.NewLine == "\r\n") + f = new CRLFTextFile(stream); + else f = new BinaryFile(stream); - else { - StreamWriter writer = null; - StreamReader reader = null; - if (f_access == FileAccess.Read || f_access == FileAccess.ReadWrite) - reader = new StreamReader(stream); - if (f_access == FileAccess.Write || f_access == FileAccess.ReadWrite) - { - Console.Error.WriteLine("opening {0} for writing", name); - writer = new StreamWriter(stream); - } - - if (System.Environment.NewLine == "\r\n") - { - Console.Error.WriteLine("opening {0} for reading (with CRLF conversion)", name); - f = new CRLFFile(stream, reader, writer); - } - else - { - Console.Error.WriteLine("opening {0} for reading (without CRLF conversion)", name); - f = new TextFile(stream, reader, writer); - } - } fdcount++; FileDescriptors[fdcount] = f; @@ -214,21 +203,22 @@ public static int ll_os_write(int fd, string buffer) { - PrintString("ll_os_write", buffer); getfd(fd).Write(buffer); return buffer.Length; } + /* private static void PrintString(string source, string s) { - Console.WriteLine(source); - Console.WriteLine(s); - Console.WriteLine("Length: {0}", s.Length); + Console.Error.WriteLine(source); + Console.Error.WriteLine(s); + Console.Error.WriteLine("Length: {0}", s.Length); for (int i=0; i>= 8 + b = x & 0xff + x >>= 8 + c = x & 0xff + x >>= 8 + d = x & 0xff + return chr(a) + chr(b) + chr(c) + chr(d) + def fn(magic): + fd = os.open(tmpfile, BIN_WRITEMASK, 0777) + os.write(fd, long2str(magic)) + os.close(fd) + self.interpret(fn, [MAGIC]) + contents = file(tmpfile).read() + assert contents == long2str(MAGIC) + def test_os_stat(self): def fn(): return os.stat('.')[0] From auc at codespeak.net Thu Aug 3 16:34:47 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 3 Aug 2006 16:34:47 +0200 (CEST) Subject: [pypy-svn] r30958 - pypy/dist/pypy/module/_stackless Message-ID: <20060803143447.2C11A1006C@code0.codespeak.net> Author: auc Date: Thu Aug 3 16:34:45 2006 New Revision: 30958 Modified: pypy/dist/pypy/module/_stackless/clonable.py Log: makes coro cloning work at application level :) Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Thu Aug 3 16:34:45 2006 @@ -18,6 +18,7 @@ def __init__(self, space, is_main=False): self.space = space + self.is_main = is_main state = self._get_state(space) Coroutine.__init__(self, state) self.flags = 0 @@ -41,9 +42,23 @@ ec = self.space.getexecutioncontext() ec.subcontext_leave(self) + def from_interp(self, interp_coro): + assert isinstance(interp_coro, InterpClonableCoroutine) + # get parent, frame, local_pool + new = ClonableCoroutine(self.space, self.is_main) + new.parent = interp_coro.parent + new.frame = interp_coro.frame + new.local_pool = interp_coro.local_pool + return new + def w_clone(self): + if self.is_main: + raise OperationError(self.space.w_NotImplementedError, + self.space.wrap("The main coroutine can't be cloned")) try: - return InterpClonableCoroutine.clone(self) + interp_coro = InterpClonableCoroutine.clone(self) + app_coro = self.from_interp(interp_coro) + return app_coro except NotImplementedError: raise OperationError(self.space.w_NotImplementedError, self.space.wrap("clone() is only available in translated PyPy")) From antocuni at codespeak.net Thu Aug 3 17:43:56 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 3 Aug 2006 17:43:56 +0200 (CEST) Subject: [pypy-svn] r30960 - in pypy/dist/pypy/translator: . cli cli/src cli/test Message-ID: <20060803154356.B521C1005A@code0.codespeak.net> Author: antocuni Date: Thu Aug 3 17:43:33 2006 New Revision: 30960 Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs pypy/dist/pypy/translator/cli/support.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/driver.py Log: some of the flags in the os module such as os.O_CREAT & co. are simply different depending on the platform: this means that the same rpython program can be different depending on the platform it is *compiled* on. Moreover, these values are passed to external functions implemented in pypylib.dll, which has no chance to know the original meaning of the flags. We really don't want this because .NET executables should be portable between windows and Mono: to solve, we patch some of the attributes of the os module to have the same value as in Windows just before the annotation, then we restore the original values when the compilation has been completed. This could be dangerous if "real" python code (instead of rpython code) tries to access such attributes when they are patched, but as far as now tests are still passing, so I hope not have broken anything. This is only a short-term hack, the long term solution will be to refactor the I/O model of pypy. Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/dist/pypy/translator/cli/src/ll_os.cs Thu Aug 3 17:43:33 2006 @@ -126,13 +126,19 @@ { private static Dictionary FileDescriptors; private static int fdcount; - private const int O_RDONLY = 0; - private const int O_WRONLY = 1; - private const int O_RDWR = 2; - private const int O_CREAT = 64; - private const int O_TRUNC = 512; - private const int O_APPEND = 1024; - private const int O_BINARY = 32768; + + // NB: these values are those used by Windows and they differs + // from the Unix ones; the os module is patched with these + // values before flowgraphing to make sure we get the very + // same values on each platform we do the compilation. + private const int O_RDONLY = 0x0000; + private const int O_WRONLY = 0x0001; + private const int O_RDWR = 0x0002; + private const int O_APPEND = 0x0008; + private const int O_CREAT = 0x0100; + private const int O_TRUNC = 0x0200; + private const int O_TEXT = 0x4000; + private const int O_BINARY = 0x8000; private const int S_IFMT = 61440; private const int S_IFDIR = 16384; Modified: pypy/dist/pypy/translator/cli/support.py ============================================================================== --- pypy/dist/pypy/translator/cli/support.py (original) +++ pypy/dist/pypy/translator/cli/support.py Thu Aug 3 17:43:33 2006 @@ -46,3 +46,34 @@ label = ', '.join([str(item) for item in key]) f.write('%s: %d\n' % (label, self.counters[key])) f.close() + +NT_OS = dict( + O_RDONLY = 0x0000, + O_WRONLY = 0x0001, + O_RDWR = 0x0002, + O_APPEND = 0x0008, + O_CREAT = 0x0100, + O_TRUNC = 0x0200, + O_TEXT = 0x4000, + O_BINARY = 0x8000 + ) + +def patch_os(defs=None): + """ + Modify the value of some attributes of the os module to be sure + they are the same on every platform pypy is compiled on. Returns a + dictionary containing the original values that can be passed to + patch_os to rollback to the original values. + """ + + import os + if defs is None: + defs = NT_OS + olddefs = {} + for name, value in defs.iteritems(): + try: + olddefs[name] = getattr(os, name) + except AttributeError: + pass + setattr(os, name, value) + return olddefs Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Thu Aug 3 17:43:33 2006 @@ -18,6 +18,7 @@ from pypy.translator.cli.database import LowLevelDatabase from pypy.translator.cli.sdk import SDK from pypy.translator.cli.entrypoint import BaseEntryPoint +from pypy.translator.cli.support import patch_os FLOAT_PRECISION = 8 @@ -112,9 +113,11 @@ def compile_function(func, annotation=[], graph=None): + olddefs = patch_os() # patch the values of some attributes of the os module gen = _build_gen(func, annotation, graph) gen.generate_source() exe_name = gen.build_exe() + patch_os(olddefs) # restore original values return CliFunctionWrapper(exe_name) def _build_gen(func, annotation, graph=None): Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Thu Aug 3 17:43:33 2006 @@ -17,15 +17,28 @@ test_os_path_exists = skip_os test_os_isdir = skip_os + def test_patch_os(self): + from pypy.translator.cli.support import patch_os, NT_OS + original_O_CREAT = os.O_CREAT + olddefs = patch_os() + assert os.O_CREAT == NT_OS['O_CREAT'] + patch_os(olddefs) + assert os.O_CREAT == original_O_CREAT + + def test_os_flags(self): + from pypy.translator.cli.support import NT_OS + def fn(): + return os.O_CREAT + assert self.interpret(fn, []) == NT_OS['O_CREAT'] + def test_os_read(self): BaseTestRbuiltin.test_os_read(self) def test_os_read_binary_crlf(self): tmpfile = str(udir.udir.join("os_read_test")) def fn(flag): - from pypy.module.__builtin__.importing import BIN_READMASK if flag: - fd = os.open(tmpfile, BIN_READMASK, 0666) + fd = os.open(tmpfile, os.O_RDONLY|os.O_BINARY, 0666) else: fd = os.open(tmpfile, os.O_RDONLY, 0666) res = os.read(fd, 4096) @@ -51,18 +64,15 @@ pass def test_os_open_write(self): - skip_win() tmpdir = str(udir.udir.join("os_write_test")) def fn(): - fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT, 0777) + fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) os.write(fd, "hello world") os.close(fd) self.interpret(fn, []) assert file(tmpdir).read() == 'hello world' def test_os_write_magic(self): - from pypy.module.__builtin__.importing import BIN_WRITEMASK - skip_win() MAGIC = 62061 | (ord('\r')<<16) | (ord('\n')<<24) tmpfile = str(udir.udir.join("os_write_test")) def long2str(x): @@ -75,11 +85,11 @@ d = x & 0xff return chr(a) + chr(b) + chr(c) + chr(d) def fn(magic): - fd = os.open(tmpfile, BIN_WRITEMASK, 0777) + fd = os.open(tmpfile, os.O_BINARY|os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0777) os.write(fd, long2str(magic)) os.close(fd) self.interpret(fn, [MAGIC]) - contents = file(tmpfile).read() + contents = file(tmpfile, 'rb').read() assert contents == long2str(MAGIC) def test_os_stat(self): Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Thu Aug 3 17:43:33 2006 @@ -199,7 +199,19 @@ self.done[goal] = True return res - def task_annotate(self): + def task_annotate(self): + # XXX: this should be a separate task, but I don't know how to + # specify a task to be executed before annotation only if the + # backend is 'cli' + + # patch some attributes of the os module to make sure they + # have the same value on every platform. + backend, ts = self.get_backend_and_type_system() + if backend == 'cli': + from pypy.translator.cli.support import patch_os + self.old_os_defs = patch_os() + # end of XXX + # includes annotation and annotatation simplifications translator = self.translator policy = self.policy @@ -482,9 +494,13 @@ 'Generating CLI source') def task_compile_cli(self): + from pypy.translator.cli.support import patch_os from pypy.translator.cli.test.runtest import CliFunctionWrapper filename = self.gen.build_exe() self.c_entryp = CliFunctionWrapper(filename) + # restore original os values + patch_os(self.old_os_defs) + self.log.info("Compiled %s" % filename) task_compile_cli = taskdef(task_compile_cli, ['source_cli'], 'Compiling CLI source') From fijal at codespeak.net Thu Aug 3 19:19:39 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Aug 2006 19:19:39 +0200 (CEST) Subject: [pypy-svn] r30962 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20060803171939.CB58110069@code0.codespeak.net> Author: fijal Date: Thu Aug 3 19:19:14 2006 New Revision: 30962 Added: pypy/dist/pypy/rpython/nonconst.py (contents, props changed) pypy/dist/pypy/rpython/test/test_nonconst.py (contents, props changed) Modified: pypy/dist/pypy/annotation/annrpython.py Log: Added non-constant constant. It means that when you write down NonConst(42) it gets annotated as SomeInteger(knowntype=int). Cool for testing without one() magic. Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Thu Aug 3 19:19:14 2006 @@ -23,6 +23,7 @@ def __init__(self, translator=None, policy=None, bookkeeper=None): import pypy.rpython.ootypesystem.ooregistry # has side effects import pypy.rpython.ootypesystem.bltregistry # has side effects + import pypy.rpython.nonconst # has side effects if translator is None: # interface for tests Added: pypy/dist/pypy/rpython/nonconst.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/nonconst.py Thu Aug 3 19:19:14 2006 @@ -0,0 +1,16 @@ + +""" simple non-constant constant. Ie constant which does not get annotated as constant +""" + +from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.annotation.bookkeeper import getbookkeeper + +class NonConstant(object): + def __init__(self, _constant): + self.constant = _constant + +class EntryNonConstant(ExtRegistryEntry): + _about_ = NonConstant + + def compute_result_annotation(self, arg): + return getbookkeeper().annotation_from_example(arg.const) Added: pypy/dist/pypy/rpython/test/test_nonconst.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/test/test_nonconst.py Thu Aug 3 19:19:14 2006 @@ -0,0 +1,18 @@ + +""" Test of non-constant constant. +""" + +from pypy.rpython.nonconst import NonConstant + +from pypy.objspace.flow import FlowObjSpace +from pypy.annotation.annrpython import RPythonAnnotator + +def test_nonconst(): + def nonconst_f(): + a = NonConstant(3) + return a + + a = RPythonAnnotator() + s = a.build_types(nonconst_f, []) + assert s.knowntype is int + assert not hasattr(s, 'const') From fijal at codespeak.net Thu Aug 3 19:32:52 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Aug 2006 19:32:52 +0200 (CEST) Subject: [pypy-svn] r30963 - pypy/dist/pypy/bin Message-ID: <20060803173252.D1FAB1006C@code0.codespeak.net> Author: fijal Date: Thu Aug 3 19:32:39 2006 New Revision: 30963 Modified: pypy/dist/pypy/bin/jscompile.py Log: Changed policy to not allow someobjects. Added non-constant generation. Modified: pypy/dist/pypy/bin/jscompile.py ============================================================================== --- pypy/dist/pypy/bin/jscompile.py (original) +++ pypy/dist/pypy/bin/jscompile.py Thu Aug 3 19:32:39 2006 @@ -11,6 +11,8 @@ from pypy.translator.driver import TranslationDriver from pypy.translator.js.js import JS from pypy.tool.error import AnnotatorError, FlowingError, debug +from pypy.rpython.nonconst import NonConstant +from pypy.annotation.policy import AnnotatorPolicy class FunctionNotFound(Exception): pass @@ -18,10 +20,13 @@ class BadSignature(Exception): pass +class JsPolicy(AnnotatorPolicy): + allow_someobjects = False + def get_args(func_data): l = [] for i in xrange(func_data.func_code.co_argcount): - l.append(repr(func_data.func_defaults[i])) + l.append("NonConstant(%s)" % repr(func_data.func_defaults[i])) return "(%s)" % ",".join(l) def _main(argv): @@ -45,7 +50,7 @@ # now we gonna just cut off not needed function driver = TranslationDriver() try: - driver.setup(some_strange_function_which_will_never_be_called, []) + driver.setup(some_strange_function_which_will_never_be_called, [], policy = JsPolicy()) driver.proceed(["compile_js"]) except Exception, e: # do something nice with it From fijal at codespeak.net Thu Aug 3 19:48:46 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Aug 2006 19:48:46 +0200 (CEST) Subject: [pypy-svn] r30964 - pypy/dist/pypy/rpython Message-ID: <20060803174846.A557110069@code0.codespeak.net> Author: fijal Date: Thu Aug 3 19:48:35 2006 New Revision: 30964 Modified: pypy/dist/pypy/rpython/nonconst.py Log: Fixed specialize_call Modified: pypy/dist/pypy/rpython/nonconst.py ============================================================================== --- pypy/dist/pypy/rpython/nonconst.py (original) +++ pypy/dist/pypy/rpython/nonconst.py Thu Aug 3 19:48:35 2006 @@ -4,6 +4,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation.bookkeeper import getbookkeeper +from pypy.objspace.flow.model import Variable, Constant class NonConstant(object): def __init__(self, _constant): @@ -14,3 +15,8 @@ def compute_result_annotation(self, arg): return getbookkeeper().annotation_from_example(arg.const) + + def specialize_call(self, hop): + v = Variable() + v.concretetype = hop.r_result.lowleveltype + return v From hpk at codespeak.net Fri Aug 4 07:45:09 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 4 Aug 2006 07:45:09 +0200 (CEST) Subject: [pypy-svn] r30972 - pypy/extradoc/eu-report Message-ID: <20060804054509.38E6710074@code0.codespeak.net> Author: hpk Date: Fri Aug 4 07:45:05 2006 New Revision: 30972 Added: pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-interim-2006-07-25.pdf - copied unchanged from r30969, pypy/eu-tracking/deliverable/D07_1_Massive_Parallelism_and_Translation_Aspects/D07.1-deliverablereport.pdf pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2006-07-28.pdf - copied unchanged from r30970, pypy/eu-tracking/deliverable/D09_1_Constraint_Solving_and_Semantic_Web/D09.1-deliverablereport.pdf pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf - copied unchanged from r30971, pypy/eu-tracking/deliverable/D14_3_Report_About_Milestone_Phase_2/D14.3-deliverablereport.pdf Removed: pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-INTERIM-2006-07-31.pdf pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-FINAL-2006-08-03.pdf Log: replace versions with ones with more exact revisions From hpk at codespeak.net Fri Aug 4 07:46:39 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 4 Aug 2006 07:46:39 +0200 (CEST) Subject: [pypy-svn] r30973 - pypy/dist/pypy/doc Message-ID: <20060804054639.EAB3F1006C@code0.codespeak.net> Author: hpk Date: Fri Aug 4 07:46:38 2006 New Revision: 30973 Modified: pypy/dist/pypy/doc/index-report.txt Log: reference exact revisions of reports Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Fri Aug 4 07:46:38 2006 @@ -68,6 +68,6 @@ .. _`D05.3 Implementation with Translation Aspects`: http://codespeak.net/svn/pypy/extradoc/eu-report/D05.3_Publish_on_implementation_with_translation_aspects.pdf .. _`D05.4 Encapsulating Low Level Aspects`: http://codespeak.net/svn/pypy/extradoc/eu-report/D05.4_Publish_on_encapsulating_low_level_language_aspects.pdf .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf -.. _`Draft D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-INTERIM-2006-07-31.pdf -.. _`Draft D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-INTERIM-2006-07-31.pdf -.. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-FINAL-2006-08-03.pdf +.. _`Draft D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-interim-2006-07-25.pdf +.. _`Draft D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2006-07-28.pdf +.. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf From hpk at codespeak.net Fri Aug 4 07:49:23 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 4 Aug 2006 07:49:23 +0200 (CEST) Subject: [pypy-svn] r30975 - pypy/extradoc/eu-report Message-ID: <20060804054923.7E6621006E@code0.codespeak.net> Author: hpk Date: Fri Aug 4 07:49:20 2006 New Revision: 30975 Removed: pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf Log: replace1 From hpk at codespeak.net Fri Aug 4 07:50:03 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 4 Aug 2006 07:50:03 +0200 (CEST) Subject: [pypy-svn] r30976 - pypy/extradoc/eu-report Message-ID: <20060804055003.4106210075@code0.codespeak.net> Author: hpk Date: Fri Aug 4 07:49:59 2006 New Revision: 30976 Added: pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf - copied unchanged from r30974, pypy/eu-tracking/deliverable/D14_3_Report_About_Milestone_Phase_2/D14.3-deliverablereport.pdf Log: replace2 From antocuni at codespeak.net Fri Aug 4 09:58:05 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 4 Aug 2006 09:58:05 +0200 (CEST) Subject: [pypy-svn] r30978 - pypy/dist/pypy/translator Message-ID: <20060804075805.A9F7F1006C@code0.codespeak.net> Author: antocuni Date: Fri Aug 4 09:58:03 2006 New Revision: 30978 Modified: pypy/dist/pypy/translator/driver.py Log: module/__builtin__/importing.py is imported at target setup time at is uses os.O_* flags at top level, so we need to patch os before that to let it see the "right" values. Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Fri Aug 4 09:58:03 2006 @@ -200,18 +200,6 @@ return res def task_annotate(self): - # XXX: this should be a separate task, but I don't know how to - # specify a task to be executed before annotation only if the - # backend is 'cli' - - # patch some attributes of the os module to make sure they - # have the same value on every platform. - backend, ts = self.get_backend_and_type_system() - if backend == 'cli': - from pypy.translator.cli.support import patch_os - self.old_os_defs = patch_os() - # end of XXX - # includes annotation and annotatation simplifications translator = self.translator policy = self.policy @@ -532,6 +520,13 @@ options = _default_options driver = TranslationDriver(options, default_goal, disable) + # patch some attributes of the os module to make sure they + # have the same value on every platform. + backend, ts = driver.get_backend_and_type_system() + if backend == 'cli': + from pypy.translator.cli.support import patch_os + driver.old_os_defs = patch_os() + target = targetspec_dic['target'] spec = target(driver, args) From bea at codespeak.net Fri Aug 4 12:39:22 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Fri, 4 Aug 2006 12:39:22 +0200 (CEST) Subject: [pypy-svn] r30983 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060804103922.71E271006F@code0.codespeak.net> Author: bea Date: Fri Aug 4 12:39:20 2006 New Revision: 30983 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: my sprint dates Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Fri Aug 4 12:39:20 2006 @@ -13,7 +13,8 @@ Michael Hudson 20-27 no idea Armin Rigo ? ? Holger Krekel 18-28 v/dublin on-campus (faxed form) -Maciej Fijalkowski 21-?/8 no idea +Maciej Fijalkowski 21-?/8 no idea +Bea During 20-25/8 ? ==================== ============== ======================== People on the following list were present at previous sprints: From mwh at codespeak.net Fri Aug 4 13:47:18 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 4 Aug 2006 13:47:18 +0200 (CEST) Subject: [pypy-svn] r30987 - pypy/dist/pypy/translator/stackless Message-ID: <20060804114718.7BA7E1006E@code0.codespeak.net> Author: mwh Date: Fri Aug 4 13:47:17 2006 New Revision: 30987 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: one more try to reuse some of the resume block code. reduces the size of pypy-c-stackess by another 100k or so -- not sure that's worth all the effort really, but the code is a bit cleaner now too. Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Fri Aug 4 13:47:17 2006 @@ -4,7 +4,7 @@ from pypy.translator.backendopt import support from pypy.objspace.flow import model from pypy.rpython.memory.gctransform import varoftype -from pypy.translator import unsimplify +from pypy.translator import unsimplify, simplify from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.translator.stackless import code, frame @@ -22,6 +22,28 @@ SAVE_STATISTICS = True +# we do this _a lot_: +def copyvar(var): + if isinstance(var, model.Variable): + return unsimplify.copyvar(None, var) + else: + return varoftype(var.concretetype) + +def copy_link_with_varmap(link, varmap): + varmap = varmap.copy() + def rename(arg): + if isinstance(arg, model.Variable): + if arg in varmap: + return varmap[arg] + else: + assert arg in [link.last_exception, link.last_exc_value] + r = copyvar(arg) + varmap[arg] = r + return r + else: + return arg + return link.copy(rename) + if SAVE_STATISTICS: import cStringIO @@ -36,6 +58,9 @@ self.total_pot_exact_saves = 0 self.pot_erased_saves = {} self.total_pot_erased_saves = 0 + self.saved_retrieval_ops = 0 + self.saved_cast_ops = 0 + self.saved_return_ops = 0 def __repr__(self): s = cStringIO.StringIO() print >> s, self.__class__.__name__ @@ -101,6 +126,32 @@ # complete. return self.value +# the strategy for sharing parts of the resume code: +# +# when a function is being resumed, there are three things that need to +# be done: the values (as erased types) need to be read out of the +# frame state structure, the return value needs to be read out of +# the global_state (this is also when the check is made if we are +# resuming with an exception) and the return value might need to be +# cast to an exact type. our strategy is to do each of these things +# in separate blocks, reusing blocks where we can. the retrieval and +# cast blocks will (conceptually at least) have a switch on the +# restate subcase at the end of it. +# +# conditions for reuse: +# +# 1. retrieval block: the erased types being read out +# +# 2. the return value blocks: the erased the erased return type and the exception +# and target of any exception links +# +# 3. the cast blocks: the exact types required. +# +# note that we order types by the index of the erased type in +# STORAGE_TYPES, to increase the chance that we can resuse the types. + + + class FrameTyper: # this class only exists independently to ease testing def __init__(self, stackless_gc=False, transformer=None): @@ -449,10 +500,13 @@ assert self.curr_graph is None self.curr_graph = graph self.curr_graph_save_blocks = {} + self.curr_graph_resume_retrieval_blocks = {} + self.curr_graph_resume_return_blocks = {} + self.curr_graph_resume_cast_blocks = {} + if SAVE_STATISTICS: self.stats.cur_rp_exact_types = {} self.stats.cur_rp_erased_types = {} - for block in list(graph.iterblocks()): assert block not in self.seen_blocks @@ -520,7 +574,7 @@ resuming_links = [] for resume_index, resume_block in enumerate(self.resume_blocks): resuming_links.append( - model.Link([], resume_block, resume_index)) + model.Link([var_resume_state], resume_block, resume_index)) resuming_links[-1].llexitcase = resume_index new_start_block.exitswitch = var_resume_state @@ -530,6 +584,14 @@ new_start_block.isstartblock = True graph.startblock = new_start_block + for block in graph.iterblocks(): + if len(block.exits) == 1 and block.exitswitch is not None: + block.exitswitch = None + block.exits[0].exitcase = block.exits[0].llexitcase = None + simplify.simplify_graph(graph, [simplify.eliminate_empty_blocks, + simplify.join_blocks, + simplify.transform_dead_op_vars]) + def insert_return_conversion(self, link, targettype, retvar): llops = LowLevelOpList() newvar = gen_cast(llops, targettype, retvar) @@ -584,10 +646,9 @@ else: self.explicit_resume_point_data[label] = frame_type - self.resume_blocks.append( - self._generate_resume_block(varsforcall, frame_type, res, block.exits)) + self._make_resume_handling(frame_type, varsforcall, res, block.exits) - restart_number = len(self.masterarray1) + len(self.resume_blocks)-1 + restart_number = len(self.masterarray1) + len(self.resume_blocks) - 1 if label in self.symbolic_restart_numbers: symb = self.symbolic_restart_numbers[label] @@ -745,11 +806,9 @@ args = vars_to_save(block) - save_block, resume_block, varsforcall = self.generate_save_and_resume_blocks( + save_block, varsforcall = self.generate_save_and_resume_blocks( args, var_unwind_exception, op.result, block.exits) - self.resume_blocks.append(resume_block) - newlink = model.Link(varsforcall + [var_unwind_exception], save_block, code.UnwindException) newlink.last_exception = model.Constant(code.UnwindException, @@ -843,9 +902,10 @@ varsforcall.insert(0, c_restart) varsforcall = [v for v in varsforcall if v.concretetype != lltype.Void] + self._make_resume_handling(frame_type, varsforcall0, + var_result, links_to_resumption) + return (self._generate_save_block(varsforcall, var_exception, saver), - self._generate_resume_block(varsforcall0, frame_type, - var_result, links_to_resumption), varsforcall) def _generate_save_block(self, varsforcall, var_unwind_exception, saver): @@ -960,6 +1020,182 @@ if SAVE_STATISTICS: self.stats.resumeops += len(newblock.operations) return newblock + + def _make_resume_handling(self, FRAME_TYPE, sorted_vars, v_retval, links_to_resumption): + resume_substate = len(self.resume_blocks) + + erased_types = [] + for v in sorted_vars: + if v.concretetype != lltype.Void: + erased_types.append(storage_type(v.concretetype)) + + retval_type = v_retval.concretetype + erased_retval_type = storage_type(retval_type) + + retrieve_block, output_args = self._get_resume_retrieval_block(FRAME_TYPE, erased_types) + + return_block, switch_block = self._get_resume_return_block( + erased_types, erased_retval_type, links_to_resumption[1:], sorted_vars) + + link = model.Link(output_args, return_block, resume_substate) + link.llexitcase = link.exitcase + retrieve_block.recloseblock(*(tuple(retrieve_block.exits) + (link,))) + + if erased_retval_type != lltype.Void: + erased_types.append(erased_retval_type) + cast_block, cast_args = self._get_resume_cast_block( + erased_types, + [v.concretetype for v in sorted_vars] + [retval_type]) + + link = model.Link(switch_block.inputargs, cast_block, resume_substate) + link.llexitcase = resume_substate + switch_block.recloseblock(*(tuple(switch_block.exits) + (link,))) + + varmap = dict([(v, cast_args[sorted_vars.index(v)]) for v in sorted_vars]) + for k, v in varmap.items(): + assert k.concretetype == v.concretetype + + varmap[v_retval] = cast_args[-1] + + link = copy_link_with_varmap(links_to_resumption[0], varmap) + link.exitcase = link.llexitcase = resume_substate + cast_block.recloseblock(*(tuple(cast_block.exits) + (link,))) + + self.resume_blocks.append(retrieve_block) + + def _make_resume_retrieval_block(self, FRAME_TYPE, erased_types): + retrieve_block = model.Block([varoftype(lltype.Signed)]) + retrieve_block.exitswitch = retrieve_block.inputargs[0] + + llops = LowLevelOpList() + llops.genop("setfield", + [self.ll_global_state, self.c_restart_substate_name, self.c_minus_one]) + v_state_hdr = llops.genop("getfield", + [self.ll_global_state, self.c_inst_top_name], + resulttype=lltype.Ptr(STATE_HEADER)) + v_state = gen_cast(llops, lltype.Ptr(FRAME_TYPE), v_state_hdr) + llops.genop("setfield", + [self.ll_global_state, self.c_inst_top_name, self.c_null_state]) + output_args = [retrieve_block.inputargs[0]] + assert len(FRAME_TYPE._names[1:]) == len(erased_types) + for fieldname, typ in zip(FRAME_TYPE._names[1:], erased_types): + assert FRAME_TYPE._flds[fieldname] == typ + output_args.append(llops.genop("getfield", + [v_state, model.Constant(fieldname, lltype.Void)], + resulttype=typ)) + retrieve_block.operations = llops + return retrieve_block, output_args + + def _get_resume_retrieval_block(self, FRAME_TYPE, erased_types): + key = tuple(erased_types) + if key in self.curr_graph_resume_retrieval_blocks: + retrieve_block, output_args = self.curr_graph_resume_retrieval_blocks[key] + if SAVE_STATISTICS: + self.stats.saved_retrieval_ops += len(retrieve_block.operations) + return retrieve_block, output_args + else: + retrieve_block, output_args = self._make_resume_retrieval_block( + FRAME_TYPE, erased_types) + self.curr_graph_resume_retrieval_blocks[key] = retrieve_block, output_args + return retrieve_block, output_args + + def _make_resume_return_block(self, erased_types, erased_retval_type, except_links, sorted_vars): + inputargs = [varoftype(lltype.Signed)] + [varoftype(t) for t in erased_types] + return_block = model.Block(inputargs) + return_block.exitswitch = model.c_last_exception + llops = LowLevelOpList() + + getretval = self.fetch_retvals[erased_retval_type] + v_retval = llops.genop("direct_call", [getretval], + resulttype=erased_retval_type) + + switch_block = model.Block([copyvar(v) for v in inputargs]) + switch_block.exitswitch = switch_block.inputargs[0] + + retlink = model.Link(inputargs, switch_block, None) + + if erased_retval_type != lltype.Void: + retlink.args.append(v_retval) + switch_block.inputargs.append(copyvar(v_retval)) + + links = [retlink] + + for except_link in except_links: + cast_block, cast_args = self._make_cast_block( + erased_types, [v.concretetype for v in sorted_vars]) + varmap = dict([(v, cast_args[sorted_vars.index(v)]) for v in sorted_vars]) + + link = model.Link(inputargs[1:], cast_block, except_link.exitcase) + link.llexitcase = except_link.llexitcase + for attr in "last_exception", "last_exc_value": + old = getattr(except_link, attr) + new = copyvar(old) + setattr(link, attr, new) + link.args.append(new) + newnew = copyvar(new) + cast_block.inputargs.append(newnew) + varmap[old] = newnew + links.append(link) + + link = copy_link_with_varmap(except_link, varmap) + link.exitcase = link.llexitcase = None + link.last_exception = link.last_exc_value = None + cast_block.closeblock(link) + + return_block.operations = llops + return_block.closeblock(*links) + + return return_block, switch_block + + def _get_resume_return_block(self, erased_types, erased_retval_type, except_links, sorted_vars): + key = (erased_retval_type,) + key += tuple(erased_types) + key += tuple([(elink.exitcase, elink.target) for elink in except_links]) + if except_links and max([len(elink.args) for elink in except_links]) > 2: + key = None + if key in self.curr_graph_resume_return_blocks: + return_block, switch_block = self.curr_graph_resume_return_blocks[key] + if SAVE_STATISTICS: + self.stats.saved_return_ops += len(return_block.operations) + return return_block, switch_block + else: + return_block, switch_block = self._make_resume_return_block(erased_types, erased_retval_type, except_links, sorted_vars) + if key is not None: + self.curr_graph_resume_return_blocks[key] = return_block, switch_block + return return_block, switch_block + + def _make_cast_block(self, erased_types, exact_types): + inputargs = [varoftype(t) for t in erased_types] + cast_block = model.Block(inputargs) + cast_block.operations = LowLevelOpList() + output_args = [] + assert len(inputargs) == len([typ for typ in exact_types if typ != lltype.Void]) + i_arg = 0 + for typ in exact_types: + if typ == lltype.Void: + output_args.append(model.Constant(None, lltype.Void)) + else: + arg = inputargs[i_arg] + i_arg += 1 + output_args.append(gen_cast(cast_block.operations, typ, arg)) + assert i_arg == len(inputargs) + return cast_block, output_args + + def _get_resume_cast_block(self, erased_vars, exact_types): + # returns something that you should add a link to (with + # recloseblock), and the output_args to use in that link. + key = tuple(exact_types) + if key in self.curr_graph_resume_cast_blocks: + cast_block, output_args = self.curr_graph_resume_cast_blocks[key] + if SAVE_STATISTICS: + self.stats.saved_cast_ops += len(cast_block.operations) + return cast_block, output_args + else: + cast_block, output_args = self._make_cast_block(erased_vars, exact_types) + cast_block.inputargs.insert(0, varoftype(lltype.Signed)) + cast_block.exitswitch = cast_block.inputargs[0] + self.curr_graph_resume_cast_blocks[key] = cast_block, output_args + return cast_block, output_args def generate_restart_infos(self, graph): restartinfo = frame.RestartInfo(graph, len(self.resume_blocks)) From fijal at codespeak.net Fri Aug 4 13:58:53 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Aug 2006 13:58:53 +0200 (CEST) Subject: [pypy-svn] r30988 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060804115853.208E11006E@code0.codespeak.net> Author: fijal Date: Fri Aug 4 13:58:51 2006 New Revision: 30988 Modified: pypy/extradoc/sprintinfo/ireland-2006/announce.txt pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: Added my and mwh's booking Added possible accomodation. Modified: pypy/extradoc/sprintinfo/ireland-2006/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/announce.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/announce.txt Fri Aug 4 13:58:51 2006 @@ -108,5 +108,10 @@ Dublin Road (15 to 20 mins walk to UL) Tel: +353 61 338385 / +353 61 331167 +- In Castletroy Lodge they've got no rooms for 23,24,25 [just called, fijal] + +Brookfield Hall, about 10 minutes walk from UNI. +Could be booked at travellerspoint.com + .. _`pypy-sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint .. _`people`: people.html Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Fri Aug 4 13:58:51 2006 @@ -10,11 +10,11 @@ ==================== ============== ======================== Anders Chrigstroem 20-28/8 ? Samuele Pedroni 20-28/8 ? -Michael Hudson 20-27 no idea +Michael Hudson 20-27 Brookfield Hall, Castletroy Armin Rigo ? ? Holger Krekel 18-28 v/dublin on-campus (faxed form) -Maciej Fijalkowski 21-?/8 no idea -Bea During 20-25/8 ? +Maciej Fijalkowski 21-28?/8 Brookfield Hall, Castletroy +Bea During 20-25/8 ? ==================== ============== ======================== People on the following list were present at previous sprints: From antocuni at codespeak.net Fri Aug 4 14:51:52 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 4 Aug 2006 14:51:52 +0200 (CEST) Subject: [pypy-svn] r30993 - pypy/dist/pypy/translator/cli Message-ID: <20060804125152.3A96A10072@code0.codespeak.net> Author: antocuni Date: Fri Aug 4 14:51:48 2006 New Revision: 30993 Modified: pypy/dist/pypy/translator/cli/rte.py Log: Don't build pypylib-unix.dll when used as a script. Modified: pypy/dist/pypy/translator/cli/rte.py ============================================================================== --- pypy/dist/pypy/translator/cli/rte.py (original) +++ pypy/dist/pypy/translator/cli/rte.py Fri Aug 4 14:51:48 2006 @@ -91,6 +91,4 @@ return DLL.get() if __name__ == '__main__': - dlls = [FrameworkDLL.get()] - if platform.system() != 'Windows': - dlls.append(UnixDLL.get()) + FrameworkDLL.get() From antocuni at codespeak.net Fri Aug 4 14:53:11 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 4 Aug 2006 14:53:11 +0200 (CEST) Subject: [pypy-svn] r30994 - in pypy/dist/pypy/translator/cli: . src Message-ID: <20060804125311.3361E10072@code0.codespeak.net> Author: antocuni Date: Fri Aug 4 14:53:06 2006 New Revision: 30994 Modified: pypy/dist/pypy/translator/cli/database.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: Preallocate constant Lists with the exact size when doing constant initialization. The pypy startup time on snake has passed from 17.7 to 11.8 secs (!!). Modified: pypy/dist/pypy/translator/cli/database.py ============================================================================== --- pypy/dist/pypy/translator/cli/database.py (original) +++ pypy/dist/pypy/translator/cli/database.py Fri Aug 4 14:53:06 2006 @@ -470,7 +470,8 @@ def instantiate(self, ilasm): assert not self.is_null() class_name = self.get_type(False) - ilasm.new('instance void class %s::.ctor()' % class_name) + AbstractConst.load(self.db, ootype.Signed, len(self.value._list), ilasm) + ilasm.new('instance void class %s::.ctor(int32)' % class_name) self.db.const_count.inc('List') self.db.const_count.inc('List', self.value._TYPE._ITEMTYPE) self.db.const_count.inc('List', len(self.value._list)) Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Aug 4 14:53:06 2006 @@ -282,6 +282,10 @@ { } + public List(int capacity): base(capacity) + { + } + public override string ToString() { // TODO: use StringBuilder instead @@ -341,6 +345,9 @@ { int Count = 0; + public ListOfVoid() { } + public ListOfVoid(int capacity) { } + public override string ToString() { // TODO: use StringBuilder instead From rhymes at codespeak.net Fri Aug 4 16:10:48 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 16:10:48 +0200 (CEST) Subject: [pypy-svn] r30999 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060804141048.317C71006E@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 16:10:40 2006 New Revision: 30999 Added: pypy/dist/pypy/module/bz2/ pypy/dist/pypy/module/bz2/__init__.py pypy/dist/pypy/module/bz2/app_bz2.py pypy/dist/pypy/module/bz2/bz2module.c pypy/dist/pypy/module/bz2/bzlib.py pypy/dist/pypy/module/bz2/fileobject.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/ pypy/dist/pypy/module/bz2/test/test_bz2.py Log: bz2 module initial import. Creation of BZ2File object works. Added: pypy/dist/pypy/module/bz2/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/__init__.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,10 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + interpleveldefs = { + 'BZ2File': 'interp_bz2.BZ2File', + } + + appleveldefs = { + '__doc__': 'app_bz2.__doc__' + } Added: pypy/dist/pypy/module/bz2/app_bz2.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/app_bz2.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,4 @@ +__doc__ = """The python bz2 module provides a comprehensive interface for +the bz2 compression library. It implements a complete file +interface, one shot (de)compression functions, and types for +sequential (de)compression.""" Added: pypy/dist/pypy/module/bz2/bz2module.c ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/bz2module.c Fri Aug 4 16:10:40 2006 @@ -0,0 +1,2198 @@ +/* + +python-bz2 - python bz2 library interface + +Copyright (c) 2002 Gustavo Niemeyer +Copyright (c) 2002 Python Software Foundation; All Rights Reserved + +*/ + +#include "Python.h" +#include +#include +#include "structmember.h" + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +static char __author__[] = +"The bz2 python module was written by:\n\ +\n\ + Gustavo Niemeyer \n\ +"; + +/* Our very own off_t-like type, 64-bit if possible */ +/* copied from Objects/fileobject.c */ +#if !defined(HAVE_LARGEFILE_SUPPORT) +typedef off_t Py_off_t; +#elif SIZEOF_OFF_T >= 8 +typedef off_t Py_off_t; +#elif SIZEOF_FPOS_T >= 8 +typedef fpos_t Py_off_t; +#else +#error "Large file support, but neither off_t nor fpos_t is large enough." +#endif + +#define BUF(v) PyString_AS_STRING((PyStringObject *)v) + +#define MODE_CLOSED 0 +#define MODE_READ 1 +#define MODE_READ_EOF 2 +#define MODE_WRITE 3 + +#define BZ2FileObject_Check(v) ((v)->ob_type == &BZ2File_Type) + + +#ifdef BZ_CONFIG_ERROR + +#if SIZEOF_LONG >= 8 +#define BZS_TOTAL_OUT(bzs) \ + (((long)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) +#elif SIZEOF_LONG_LONG >= 8 +#define BZS_TOTAL_OUT(bzs) \ + (((PY_LONG_LONG)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) +#else +#define BZS_TOTAL_OUT(bzs) \ + bzs->total_out_lo32 +#endif + +#else /* ! BZ_CONFIG_ERROR */ + +#define BZ2_bzRead bzRead +#define BZ2_bzReadOpen bzReadOpen +#define BZ2_bzReadClose bzReadClose +#define BZ2_bzWrite bzWrite +#define BZ2_bzWriteOpen bzWriteOpen +#define BZ2_bzWriteClose bzWriteClose +#define BZ2_bzCompress bzCompress +#define BZ2_bzCompressInit bzCompressInit +#define BZ2_bzCompressEnd bzCompressEnd +#define BZ2_bzDecompress bzDecompress +#define BZ2_bzDecompressInit bzDecompressInit +#define BZ2_bzDecompressEnd bzDecompressEnd + +#define BZS_TOTAL_OUT(bzs) bzs->total_out + +#endif /* ! BZ_CONFIG_ERROR */ + + +#ifdef WITH_THREAD +#define ACQUIRE_LOCK(obj) PyThread_acquire_lock(obj->lock, 1) +#define RELEASE_LOCK(obj) PyThread_release_lock(obj->lock) +#else +#define ACQUIRE_LOCK(obj) +#define RELEASE_LOCK(obj) +#endif + +/* Bits in f_newlinetypes */ +#define NEWLINE_UNKNOWN 0 /* No newline seen, yet */ +#define NEWLINE_CR 1 /* \r newline seen */ +#define NEWLINE_LF 2 /* \n newline seen */ +#define NEWLINE_CRLF 4 /* \r\n newline seen */ + +/* ===================================================================== */ +/* Structure definitions. */ + +typedef struct { + PyObject_HEAD + PyObject *file; + + char* f_buf; /* Allocated readahead buffer */ + char* f_bufend; /* Points after last occupied position */ + char* f_bufptr; /* Current buffer position */ + + int f_softspace; /* Flag used by 'print' command */ + + int f_univ_newline; /* Handle any newline convention */ + int f_newlinetypes; /* Types of newlines seen */ + int f_skipnextlf; /* Skip next \n */ + + BZFILE *fp; + int mode; + Py_off_t pos; + Py_off_t size; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2FileObject; + +typedef struct { + PyObject_HEAD + bz_stream bzs; + int running; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2CompObject; + +typedef struct { + PyObject_HEAD + bz_stream bzs; + int running; + PyObject *unused_data; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2DecompObject; + +/* ===================================================================== */ +/* Utility functions. */ + +static int +Util_CatchBZ2Error(int bzerror) +{ + int ret = 0; + switch(bzerror) { + case BZ_OK: + case BZ_STREAM_END: + break; + +#ifdef BZ_CONFIG_ERROR + case BZ_CONFIG_ERROR: + PyErr_SetString(PyExc_SystemError, + "the bz2 library was not compiled " + "correctly"); + ret = 1; + break; +#endif + + case BZ_PARAM_ERROR: + PyErr_SetString(PyExc_ValueError, + "the bz2 library has received wrong " + "parameters"); + ret = 1; + break; + + case BZ_MEM_ERROR: + PyErr_NoMemory(); + ret = 1; + break; + + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + PyErr_SetString(PyExc_IOError, "invalid data stream"); + ret = 1; + break; + + case BZ_IO_ERROR: + PyErr_SetString(PyExc_IOError, "unknown IO error"); + ret = 1; + break; + + case BZ_UNEXPECTED_EOF: + PyErr_SetString(PyExc_EOFError, + "compressed file ended before the " + "logical end-of-stream was detected"); + ret = 1; + break; + + case BZ_SEQUENCE_ERROR: + PyErr_SetString(PyExc_RuntimeError, + "wrong sequence of bz2 library " + "commands used"); + ret = 1; + break; + } + return ret; +} + +#if BUFSIZ < 8192 +#define SMALLCHUNK 8192 +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + +/* This is a hacked version of Python's fileobject.c:new_buffersize(). */ +static size_t +Util_NewBufferSize(size_t currentsize) +{ + if (currentsize > SMALLCHUNK) { + /* Keep doubling until we reach BIGCHUNK; + then keep adding BIGCHUNK. */ + if (currentsize <= BIGCHUNK) + return currentsize + currentsize; + else + return currentsize + BIGCHUNK; + } + return currentsize + SMALLCHUNK; +} + +/* This is a hacked version of Python's fileobject.c:get_line(). */ +static PyObject * +Util_GetLine(BZ2FileObject *f, int n) +{ + char c; + char *buf, *end; + size_t total_v_size; /* total # of slots in buffer */ + size_t used_v_size; /* # used slots in buffer */ + size_t increment; /* amount to increment the buffer */ + PyObject *v; + int bzerror; + int newlinetypes = f->f_newlinetypes; + int skipnextlf = f->f_skipnextlf; + int univ_newline = f->f_univ_newline; + + total_v_size = n > 0 ? n : 100; + v = PyString_FromStringAndSize((char *)NULL, total_v_size); + if (v == NULL) + return NULL; + + buf = BUF(v); + end = buf + total_v_size; + + for (;;) { + Py_BEGIN_ALLOW_THREADS + if (univ_newline) { + while (1) { + BZ2_bzRead(&bzerror, f->fp, &c, 1); + f->pos++; + if (bzerror != BZ_OK || buf == end) + break; + if (skipnextlf) { + skipnextlf = 0; + if (c == '\n') { + /* Seeing a \n here with + * skipnextlf true means we + * saw a \r before. + */ + newlinetypes |= NEWLINE_CRLF; + BZ2_bzRead(&bzerror, f->fp, + &c, 1); + if (bzerror != BZ_OK) + break; + } else { + newlinetypes |= NEWLINE_CR; + } + } + if (c == '\r') { + skipnextlf = 1; + c = '\n'; + } else if ( c == '\n') + newlinetypes |= NEWLINE_LF; + *buf++ = c; + if (c == '\n') break; + } + if (bzerror == BZ_STREAM_END && skipnextlf) + newlinetypes |= NEWLINE_CR; + } else /* If not universal newlines use the normal loop */ + do { + BZ2_bzRead(&bzerror, f->fp, &c, 1); + f->pos++; + *buf++ = c; + } while (bzerror == BZ_OK && c != '\n' && buf != end); + Py_END_ALLOW_THREADS + f->f_newlinetypes = newlinetypes; + f->f_skipnextlf = skipnextlf; + if (bzerror == BZ_STREAM_END) { + f->size = f->pos; + f->mode = MODE_READ_EOF; + break; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Py_DECREF(v); + return NULL; + } + if (c == '\n') + break; + /* Must be because buf == end */ + if (n > 0) + break; + used_v_size = total_v_size; + increment = total_v_size >> 2; /* mild exponential growth */ + total_v_size += increment; + if (total_v_size > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "line is longer than a Python string can hold"); + Py_DECREF(v); + return NULL; + } + if (_PyString_Resize(&v, total_v_size) < 0) + return NULL; + buf = BUF(v) + used_v_size; + end = BUF(v) + total_v_size; + } + + used_v_size = buf - BUF(v); + if (used_v_size != total_v_size) + _PyString_Resize(&v, used_v_size); + return v; +} + +/* This is a hacked version of Python's + * fileobject.c:Py_UniversalNewlineFread(). */ +size_t +Util_UnivNewlineRead(int *bzerror, BZFILE *stream, + char* buf, size_t n, BZ2FileObject *f) +{ + char *dst = buf; + int newlinetypes, skipnextlf; + + assert(buf != NULL); + assert(stream != NULL); + + if (!f->f_univ_newline) + return BZ2_bzRead(bzerror, stream, buf, n); + + newlinetypes = f->f_newlinetypes; + skipnextlf = f->f_skipnextlf; + + /* Invariant: n is the number of bytes remaining to be filled + * in the buffer. + */ + while (n) { + size_t nread; + int shortread; + char *src = dst; + + nread = BZ2_bzRead(bzerror, stream, dst, n); + assert(nread <= n); + n -= nread; /* assuming 1 byte out for each in; will adjust */ + shortread = n != 0; /* true iff EOF or error */ + while (nread--) { + char c = *src++; + if (c == '\r') { + /* Save as LF and set flag to skip next LF. */ + *dst++ = '\n'; + skipnextlf = 1; + } + else if (skipnextlf && c == '\n') { + /* Skip LF, and remember we saw CR LF. */ + skipnextlf = 0; + newlinetypes |= NEWLINE_CRLF; + ++n; + } + else { + /* Normal char to be stored in buffer. Also + * update the newlinetypes flag if either this + * is an LF or the previous char was a CR. + */ + if (c == '\n') + newlinetypes |= NEWLINE_LF; + else if (skipnextlf) + newlinetypes |= NEWLINE_CR; + *dst++ = c; + skipnextlf = 0; + } + } + if (shortread) { + /* If this is EOF, update type flags. */ + if (skipnextlf && *bzerror == BZ_STREAM_END) + newlinetypes |= NEWLINE_CR; + break; + } + } + f->f_newlinetypes = newlinetypes; + f->f_skipnextlf = skipnextlf; + return dst - buf; +} + +/* This is a hacked version of Python's fileobject.c:drop_readahead(). */ +static void +Util_DropReadAhead(BZ2FileObject *f) +{ + if (f->f_buf != NULL) { + PyMem_Free(f->f_buf); + f->f_buf = NULL; + } +} + +/* This is a hacked version of Python's fileobject.c:readahead(). */ +static int +Util_ReadAhead(BZ2FileObject *f, int bufsize) +{ + int chunksize; + int bzerror; + + if (f->f_buf != NULL) { + if((f->f_bufend - f->f_bufptr) >= 1) + return 0; + else + Util_DropReadAhead(f); + } + if (f->mode == MODE_READ_EOF) { + f->f_bufptr = f->f_buf; + f->f_bufend = f->f_buf; + return 0; + } + if ((f->f_buf = PyMem_Malloc(bufsize)) == NULL) { + return -1; + } + Py_BEGIN_ALLOW_THREADS + chunksize = Util_UnivNewlineRead(&bzerror, f->fp, f->f_buf, + bufsize, f); + Py_END_ALLOW_THREADS + f->pos += chunksize; + if (bzerror == BZ_STREAM_END) { + f->size = f->pos; + f->mode = MODE_READ_EOF; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Util_DropReadAhead(f); + return -1; + } + f->f_bufptr = f->f_buf; + f->f_bufend = f->f_buf + chunksize; + return 0; +} + +/* This is a hacked version of Python's + * fileobject.c:readahead_get_line_skip(). */ +static PyStringObject * +Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize) +{ + PyStringObject* s; + char *bufptr; + char *buf; + int len; + + if (f->f_buf == NULL) + if (Util_ReadAhead(f, bufsize) < 0) + return NULL; + + len = f->f_bufend - f->f_bufptr; + if (len == 0) + return (PyStringObject *) + PyString_FromStringAndSize(NULL, skip); + bufptr = memchr(f->f_bufptr, '\n', len); + if (bufptr != NULL) { + bufptr++; /* Count the '\n' */ + len = bufptr - f->f_bufptr; + s = (PyStringObject *) + PyString_FromStringAndSize(NULL, skip+len); + if (s == NULL) + return NULL; + memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len); + f->f_bufptr = bufptr; + if (bufptr == f->f_bufend) + Util_DropReadAhead(f); + } else { + bufptr = f->f_bufptr; + buf = f->f_buf; + f->f_buf = NULL; /* Force new readahead buffer */ + s = Util_ReadAheadGetLineSkip(f, skip+len, + bufsize + (bufsize>>2)); + if (s == NULL) { + PyMem_Free(buf); + return NULL; + } + memcpy(PyString_AS_STRING(s)+skip, bufptr, len); + PyMem_Free(buf); + } + return s; +} + +/* ===================================================================== */ +/* Methods of BZ2File. */ + +PyDoc_STRVAR(BZ2File_read__doc__, +"read([size]) -> string\n\ +\n\ +Read at most size uncompressed bytes, returned as a string. If the size\n\ +argument is negative or omitted, read until EOF is reached.\n\ +"); + +/* This is a hacked version of Python's fileobject.c:file_read(). */ +static PyObject * +BZ2File_read(BZ2FileObject *self, PyObject *args) +{ + long bytesrequested = -1; + size_t bytesread, buffersize, chunksize; + int bzerror; + PyObject *ret = NULL; + + if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested)) + return NULL; + + ACQUIRE_LOCK(self); + switch (self->mode) { + case MODE_READ: + break; + case MODE_READ_EOF: + ret = PyString_FromString(""); + goto cleanup; + case MODE_CLOSED: + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup; + default: + PyErr_SetString(PyExc_IOError, + "file is not ready for reading"); + goto cleanup; + } + + if (bytesrequested < 0) + buffersize = Util_NewBufferSize((size_t)0); + else + buffersize = bytesrequested; + if (buffersize > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "requested number of bytes is " + "more than a Python string can hold"); + goto cleanup; + } + ret = PyString_FromStringAndSize((char *)NULL, buffersize); + if (ret == NULL) + goto cleanup; + bytesread = 0; + + for (;;) { + Py_BEGIN_ALLOW_THREADS + chunksize = Util_UnivNewlineRead(&bzerror, self->fp, + BUF(ret)+bytesread, + buffersize-bytesread, + self); + self->pos += chunksize; + Py_END_ALLOW_THREADS + bytesread += chunksize; + if (bzerror == BZ_STREAM_END) { + self->size = self->pos; + self->mode = MODE_READ_EOF; + break; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Py_DECREF(ret); + ret = NULL; + goto cleanup; + } + if (bytesrequested < 0) { + buffersize = Util_NewBufferSize(buffersize); + if (_PyString_Resize(&ret, buffersize) < 0) + goto cleanup; + } else { + break; + } + } + if (bytesread != buffersize) + _PyString_Resize(&ret, bytesread); + +cleanup: + RELEASE_LOCK(self); + return ret; +} + +PyDoc_STRVAR(BZ2File_readline__doc__, +"readline([size]) -> string\n\ +\n\ +Return the next line from the file, as a string, retaining newline.\n\ +A non-negative size argument will limit the maximum number of bytes to\n\ +return (an incomplete line may be returned then). Return an empty\n\ +string at EOF.\n\ +"); + +static PyObject * +BZ2File_readline(BZ2FileObject *self, PyObject *args) +{ + PyObject *ret = NULL; + int sizehint = -1; + + if (!PyArg_ParseTuple(args, "|i:readline", &sizehint)) + return NULL; + + ACQUIRE_LOCK(self); + switch (self->mode) { + case MODE_READ: + break; + case MODE_READ_EOF: + ret = PyString_FromString(""); + goto cleanup; + case MODE_CLOSED: + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup; + default: + PyErr_SetString(PyExc_IOError, + "file is not ready for reading"); + goto cleanup; + } + + if (sizehint == 0) + ret = PyString_FromString(""); + else + ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint); + +cleanup: + RELEASE_LOCK(self); + return ret; +} + +PyDoc_STRVAR(BZ2File_readlines__doc__, +"readlines([size]) -> list\n\ +\n\ +Call readline() repeatedly and return a list of lines read.\n\ +The optional size argument, if given, is an approximate bound on the\n\ +total number of bytes in the lines returned.\n\ +"); + +/* This is a hacked version of Python's fileobject.c:file_readlines(). */ +static PyObject * +BZ2File_readlines(BZ2FileObject *self, PyObject *args) +{ + long sizehint = 0; + PyObject *list = NULL; + PyObject *line; + char small_buffer[SMALLCHUNK]; + char *buffer = small_buffer; + size_t buffersize = SMALLCHUNK; + PyObject *big_buffer = NULL; + size_t nfilled = 0; + size_t nread; + size_t totalread = 0; + char *p, *q, *end; + int err; + int shortread = 0; + int bzerror; + + if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint)) + return NULL; + + ACQUIRE_LOCK(self); + switch (self->mode) { + case MODE_READ: + break; + case MODE_READ_EOF: + list = PyList_New(0); + goto cleanup; + case MODE_CLOSED: + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup; + default: + PyErr_SetString(PyExc_IOError, + "file is not ready for reading"); + goto cleanup; + } + + if ((list = PyList_New(0)) == NULL) + goto cleanup; + + for (;;) { + Py_BEGIN_ALLOW_THREADS + nread = Util_UnivNewlineRead(&bzerror, self->fp, + buffer+nfilled, + buffersize-nfilled, self); + self->pos += nread; + Py_END_ALLOW_THREADS + if (bzerror == BZ_STREAM_END) { + self->size = self->pos; + self->mode = MODE_READ_EOF; + if (nread == 0) { + sizehint = 0; + break; + } + shortread = 1; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + error: + Py_DECREF(list); + list = NULL; + goto cleanup; + } + totalread += nread; + p = memchr(buffer+nfilled, '\n', nread); + if (!shortread && p == NULL) { + /* Need a larger buffer to fit this line */ + nfilled += nread; + buffersize *= 2; + if (buffersize > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "line is longer than a Python string can hold"); + goto error; + } + if (big_buffer == NULL) { + /* Create the big buffer */ + big_buffer = PyString_FromStringAndSize( + NULL, buffersize); + if (big_buffer == NULL) + goto error; + buffer = PyString_AS_STRING(big_buffer); + memcpy(buffer, small_buffer, nfilled); + } + else { + /* Grow the big buffer */ + _PyString_Resize(&big_buffer, buffersize); + buffer = PyString_AS_STRING(big_buffer); + } + continue; + } + end = buffer+nfilled+nread; + q = buffer; + while (p != NULL) { + /* Process complete lines */ + p++; + line = PyString_FromStringAndSize(q, p-q); + if (line == NULL) + goto error; + err = PyList_Append(list, line); + Py_DECREF(line); + if (err != 0) + goto error; + q = p; + p = memchr(q, '\n', end-q); + } + /* Move the remaining incomplete line to the start */ + nfilled = end-q; + memmove(buffer, q, nfilled); + if (sizehint > 0) + if (totalread >= (size_t)sizehint) + break; + if (shortread) { + sizehint = 0; + break; + } + } + if (nfilled != 0) { + /* Partial last line */ + line = PyString_FromStringAndSize(buffer, nfilled); + if (line == NULL) + goto error; + if (sizehint > 0) { + /* Need to complete the last line */ + PyObject *rest = Util_GetLine(self, 0); + if (rest == NULL) { + Py_DECREF(line); + goto error; + } + PyString_Concat(&line, rest); + Py_DECREF(rest); + if (line == NULL) + goto error; + } + err = PyList_Append(list, line); + Py_DECREF(line); + if (err != 0) + goto error; + } + + cleanup: + RELEASE_LOCK(self); + if (big_buffer) { + Py_DECREF(big_buffer); + } + return list; +} + +PyDoc_STRVAR(BZ2File_xreadlines__doc__, +"xreadlines() -> self\n\ +\n\ +For backward compatibility. BZ2File objects now include the performance\n\ +optimizations previously implemented in the xreadlines module.\n\ +"); + +PyDoc_STRVAR(BZ2File_write__doc__, +"write(data) -> None\n\ +\n\ +Write the 'data' string to file. Note that due to buffering, close() may\n\ +be needed before the file on disk reflects the data written.\n\ +"); + +/* This is a hacked version of Python's fileobject.c:file_write(). */ +static PyObject * +BZ2File_write(BZ2FileObject *self, PyObject *args) +{ + PyObject *ret = NULL; + char *buf; + int len; + int bzerror; + + if (!PyArg_ParseTuple(args, "s#:write", &buf, &len)) + return NULL; + + ACQUIRE_LOCK(self); + switch (self->mode) { + case MODE_WRITE: + break; + + case MODE_CLOSED: + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup;; + + default: + PyErr_SetString(PyExc_IOError, + "file is not ready for writing"); + goto cleanup;; + } + + self->f_softspace = 0; + + Py_BEGIN_ALLOW_THREADS + BZ2_bzWrite (&bzerror, self->fp, buf, len); + self->pos += len; + Py_END_ALLOW_THREADS + + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto cleanup; + } + + Py_INCREF(Py_None); + ret = Py_None; + +cleanup: + RELEASE_LOCK(self); + return ret; +} + +PyDoc_STRVAR(BZ2File_writelines__doc__, +"writelines(sequence_of_strings) -> None\n\ +\n\ +Write the sequence of strings to the file. Note that newlines are not\n\ +added. The sequence can be any iterable object producing strings. This is\n\ +equivalent to calling write() for each string.\n\ +"); + +/* This is a hacked version of Python's fileobject.c:file_writelines(). */ +static PyObject * +BZ2File_writelines(BZ2FileObject *self, PyObject *seq) +{ +#define CHUNKSIZE 1000 + PyObject *list = NULL; + PyObject *iter = NULL; + PyObject *ret = NULL; + PyObject *line; + int i, j, index, len, islist; + int bzerror; + + ACQUIRE_LOCK(self); + islist = PyList_Check(seq); + if (!islist) { + iter = PyObject_GetIter(seq); + if (iter == NULL) { + PyErr_SetString(PyExc_TypeError, + "writelines() requires an iterable argument"); + goto error; + } + list = PyList_New(CHUNKSIZE); + if (list == NULL) + goto error; + } + + /* Strategy: slurp CHUNKSIZE lines into a private list, + checking that they are all strings, then write that list + without holding the interpreter lock, then come back for more. */ + for (index = 0; ; index += CHUNKSIZE) { + if (islist) { + Py_XDECREF(list); + list = PyList_GetSlice(seq, index, index+CHUNKSIZE); + if (list == NULL) + goto error; + j = PyList_GET_SIZE(list); + } + else { + for (j = 0; j < CHUNKSIZE; j++) { + line = PyIter_Next(iter); + if (line == NULL) { + if (PyErr_Occurred()) + goto error; + break; + } + PyList_SetItem(list, j, line); + } + } + if (j == 0) + break; + + /* Check that all entries are indeed strings. If not, + apply the same rules as for file.write() and + convert the rets to strings. This is slow, but + seems to be the only way since all conversion APIs + could potentially execute Python code. */ + for (i = 0; i < j; i++) { + PyObject *v = PyList_GET_ITEM(list, i); + if (!PyString_Check(v)) { + const char *buffer; + Py_ssize_t len; + if (PyObject_AsCharBuffer(v, &buffer, &len)) { + PyErr_SetString(PyExc_TypeError, + "writelines() " + "argument must be " + "a sequence of " + "strings"); + goto error; + } + line = PyString_FromStringAndSize(buffer, + len); + if (line == NULL) + goto error; + Py_DECREF(v); + PyList_SET_ITEM(list, i, line); + } + } + + self->f_softspace = 0; + + /* Since we are releasing the global lock, the + following code may *not* execute Python code. */ + Py_BEGIN_ALLOW_THREADS + for (i = 0; i < j; i++) { + line = PyList_GET_ITEM(list, i); + len = PyString_GET_SIZE(line); + BZ2_bzWrite (&bzerror, self->fp, + PyString_AS_STRING(line), len); + if (bzerror != BZ_OK) { + Py_BLOCK_THREADS + Util_CatchBZ2Error(bzerror); + goto error; + } + } + Py_END_ALLOW_THREADS + + if (j < CHUNKSIZE) + break; + } + + Py_INCREF(Py_None); + ret = Py_None; + + error: + RELEASE_LOCK(self); + Py_XDECREF(list); + Py_XDECREF(iter); + return ret; +#undef CHUNKSIZE +} + +PyDoc_STRVAR(BZ2File_seek__doc__, +"seek(offset [, whence]) -> None\n\ +\n\ +Move to new file position. Argument offset is a byte count. Optional\n\ +argument whence defaults to 0 (offset from start of file, offset\n\ +should be >= 0); other values are 1 (move relative to current position,\n\ +positive or negative), and 2 (move relative to end of file, usually\n\ +negative, although many platforms allow seeking beyond the end of a file).\n\ +\n\ +Note that seeking of bz2 files is emulated, and depending on the parameters\n\ +the operation may be extremely slow.\n\ +"); + +static PyObject * +BZ2File_seek(BZ2FileObject *self, PyObject *args) +{ + int where = 0; + PyObject *offobj; + Py_off_t offset; + char small_buffer[SMALLCHUNK]; + char *buffer = small_buffer; + size_t buffersize = SMALLCHUNK; + int bytesread = 0; + size_t readsize; + int chunksize; + int bzerror; + PyObject *ret = NULL; + + if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &where)) + return NULL; +#if !defined(HAVE_LARGEFILE_SUPPORT) + offset = PyInt_AsLong(offobj); +#else + offset = PyLong_Check(offobj) ? + PyLong_AsLongLong(offobj) : PyInt_AsLong(offobj); +#endif + if (PyErr_Occurred()) + return NULL; + + ACQUIRE_LOCK(self); + Util_DropReadAhead(self); + switch (self->mode) { + case MODE_READ: + case MODE_READ_EOF: + break; + + case MODE_CLOSED: + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup;; + + default: + PyErr_SetString(PyExc_IOError, + "seek works only while reading"); + goto cleanup;; + } + + if (where == 2) { + if (self->size == -1) { + assert(self->mode != MODE_READ_EOF); + for (;;) { + Py_BEGIN_ALLOW_THREADS + chunksize = Util_UnivNewlineRead( + &bzerror, self->fp, + buffer, buffersize, + self); + self->pos += chunksize; + Py_END_ALLOW_THREADS + + bytesread += chunksize; + if (bzerror == BZ_STREAM_END) { + break; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto cleanup; + } + } + self->mode = MODE_READ_EOF; + self->size = self->pos; + bytesread = 0; + } + offset = self->size + offset; + } else if (where == 1) { + offset = self->pos + offset; + } + + /* Before getting here, offset must be the absolute position the file + * pointer should be set to. */ + + if (offset >= self->pos) { + /* we can move forward */ + offset -= self->pos; + } else { + /* we cannot move back, so rewind the stream */ + BZ2_bzReadClose(&bzerror, self->fp); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto cleanup; + } + ret = PyObject_CallMethod(self->file, "seek", "(i)", 0); + if (!ret) + goto cleanup; + Py_DECREF(ret); + ret = NULL; + self->pos = 0; + self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file), + 0, 0, NULL, 0); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto cleanup; + } + self->mode = MODE_READ; + } + + if (offset <= 0 || self->mode == MODE_READ_EOF) + goto exit; + + /* Before getting here, offset must be set to the number of bytes + * to walk forward. */ + for (;;) { + if (offset-bytesread > buffersize) + readsize = buffersize; + else + /* offset might be wider that readsize, but the result + * of the subtraction is bound by buffersize (see the + * condition above). buffersize is 8192. */ + readsize = (size_t)(offset-bytesread); + Py_BEGIN_ALLOW_THREADS + chunksize = Util_UnivNewlineRead(&bzerror, self->fp, + buffer, readsize, self); + self->pos += chunksize; + Py_END_ALLOW_THREADS + bytesread += chunksize; + if (bzerror == BZ_STREAM_END) { + self->size = self->pos; + self->mode = MODE_READ_EOF; + break; + } else if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto cleanup; + } + if (bytesread == offset) + break; + } + +exit: + Py_INCREF(Py_None); + ret = Py_None; + +cleanup: + RELEASE_LOCK(self); + return ret; +} + +PyDoc_STRVAR(BZ2File_tell__doc__, +"tell() -> int\n\ +\n\ +Return the current file position, an integer (may be a long integer).\n\ +"); + +static PyObject * +BZ2File_tell(BZ2FileObject *self, PyObject *args) +{ + PyObject *ret = NULL; + + if (self->mode == MODE_CLOSED) { + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + goto cleanup; + } + +#if !defined(HAVE_LARGEFILE_SUPPORT) + ret = PyInt_FromLong(self->pos); +#else + ret = PyLong_FromLongLong(self->pos); +#endif + +cleanup: + return ret; +} + +PyDoc_STRVAR(BZ2File_close__doc__, +"close() -> None or (perhaps) an integer\n\ +\n\ +Close the file. Sets data attribute .closed to true. A closed file\n\ +cannot be used for further I/O operations. close() may be called more\n\ +than once without error.\n\ +"); + +static PyObject * +BZ2File_close(BZ2FileObject *self) +{ + PyObject *ret = NULL; + int bzerror = BZ_OK; + + ACQUIRE_LOCK(self); + switch (self->mode) { + case MODE_READ: + case MODE_READ_EOF: + BZ2_bzReadClose(&bzerror, self->fp); + break; + case MODE_WRITE: + BZ2_bzWriteClose(&bzerror, self->fp, + 0, NULL, NULL); + break; + } + self->mode = MODE_CLOSED; + ret = PyObject_CallMethod(self->file, "close", NULL); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Py_XDECREF(ret); + ret = NULL; + } + + RELEASE_LOCK(self); + return ret; +} + +static PyObject *BZ2File_getiter(BZ2FileObject *self); + +static PyMethodDef BZ2File_methods[] = { + {"read", (PyCFunction)BZ2File_read, METH_VARARGS, BZ2File_read__doc__}, + {"readline", (PyCFunction)BZ2File_readline, METH_VARARGS, BZ2File_readline__doc__}, + {"readlines", (PyCFunction)BZ2File_readlines, METH_VARARGS, BZ2File_readlines__doc__}, + {"xreadlines", (PyCFunction)BZ2File_getiter, METH_VARARGS, BZ2File_xreadlines__doc__}, + {"write", (PyCFunction)BZ2File_write, METH_VARARGS, BZ2File_write__doc__}, + {"writelines", (PyCFunction)BZ2File_writelines, METH_O, BZ2File_writelines__doc__}, + {"seek", (PyCFunction)BZ2File_seek, METH_VARARGS, BZ2File_seek__doc__}, + {"tell", (PyCFunction)BZ2File_tell, METH_NOARGS, BZ2File_tell__doc__}, + {"close", (PyCFunction)BZ2File_close, METH_NOARGS, BZ2File_close__doc__}, + {NULL, NULL} /* sentinel */ +}; + + +/* ===================================================================== */ +/* Getters and setters of BZ2File. */ + +/* This is a hacked version of Python's fileobject.c:get_newlines(). */ +static PyObject * +BZ2File_get_newlines(BZ2FileObject *self, void *closure) +{ + switch (self->f_newlinetypes) { + case NEWLINE_UNKNOWN: + Py_INCREF(Py_None); + return Py_None; + case NEWLINE_CR: + return PyString_FromString("\r"); + case NEWLINE_LF: + return PyString_FromString("\n"); + case NEWLINE_CR|NEWLINE_LF: + return Py_BuildValue("(ss)", "\r", "\n"); + case NEWLINE_CRLF: + return PyString_FromString("\r\n"); + case NEWLINE_CR|NEWLINE_CRLF: + return Py_BuildValue("(ss)", "\r", "\r\n"); + case NEWLINE_LF|NEWLINE_CRLF: + return Py_BuildValue("(ss)", "\n", "\r\n"); + case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF: + return Py_BuildValue("(sss)", "\r", "\n", "\r\n"); + default: + PyErr_Format(PyExc_SystemError, + "Unknown newlines value 0x%x\n", + self->f_newlinetypes); + return NULL; + } +} + +static PyObject * +BZ2File_get_closed(BZ2FileObject *self, void *closure) +{ + return PyInt_FromLong(self->mode == MODE_CLOSED); +} + +static PyObject * +BZ2File_get_mode(BZ2FileObject *self, void *closure) +{ + return PyObject_GetAttrString(self->file, "mode"); +} + +static PyObject * +BZ2File_get_name(BZ2FileObject *self, void *closure) +{ + return PyObject_GetAttrString(self->file, "name"); +} + +static PyGetSetDef BZ2File_getset[] = { + {"closed", (getter)BZ2File_get_closed, NULL, + "True if the file is closed"}, + {"newlines", (getter)BZ2File_get_newlines, NULL, + "end-of-line convention used in this file"}, + {"mode", (getter)BZ2File_get_mode, NULL, + "file mode ('r', 'w', or 'U')"}, + {"name", (getter)BZ2File_get_name, NULL, + "file name"}, + {NULL} /* Sentinel */ +}; + + +/* ===================================================================== */ +/* Members of BZ2File_Type. */ + +#undef OFF +#define OFF(x) offsetof(BZ2FileObject, x) + +static PyMemberDef BZ2File_members[] = { + {"softspace", T_INT, OFF(f_softspace), 0, + "flag indicating that a space needs to be printed; used by print"}, + {NULL} /* Sentinel */ +}; + +/* ===================================================================== */ +/* Slot definitions for BZ2File_Type. */ + +static int +BZ2File_init(BZ2FileObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"filename", "mode", "buffering", + "compresslevel", 0}; + PyObject *name; + char *mode = "r"; + int buffering = -1; + int compresslevel = 9; + int bzerror; + int mode_char = 0; + + self->size = -1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|sii:BZ2File", + kwlist, &name, &mode, &buffering, + &compresslevel)) + return -1; + + if (compresslevel < 1 || compresslevel > 9) { + PyErr_SetString(PyExc_ValueError, + "compresslevel must be between 1 and 9"); + return -1; + } + + for (;;) { + int error = 0; + switch (*mode) { + case 'r': + case 'w': + if (mode_char) + error = 1; + mode_char = *mode; + break; + + case 'b': + break; + + case 'U': + self->f_univ_newline = 1; + break; + + default: + error = 1; + break; + } + if (error) { + PyErr_Format(PyExc_ValueError, + "invalid mode char %c", *mode); + return -1; + } + mode++; + if (*mode == '\0') + break; + } + + if (mode_char == 0) { + mode_char = 'r'; + } + + mode = (mode_char == 'r') ? "rb" : "wb"; + + self->file = PyObject_CallFunction((PyObject*)&PyFile_Type, "(Osi)", + name, mode, buffering); + if (self->file == NULL) + return -1; + + /* From now on, we have stuff to dealloc, so jump to error label + * instead of returning */ + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (!self->lock) + goto error; +#endif + + if (mode_char == 'r') + self->fp = BZ2_bzReadOpen(&bzerror, + PyFile_AsFile(self->file), + 0, 0, NULL, 0); + else + self->fp = BZ2_bzWriteOpen(&bzerror, + PyFile_AsFile(self->file), + compresslevel, 0, 0); + + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + + self->mode = (mode_char == 'r') ? MODE_READ : MODE_WRITE; + + return 0; + +error: + Py_DECREF(self->file); +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + return -1; +} + +static void +BZ2File_dealloc(BZ2FileObject *self) +{ + int bzerror; +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + switch (self->mode) { + case MODE_READ: + case MODE_READ_EOF: + BZ2_bzReadClose(&bzerror, self->fp); + break; + case MODE_WRITE: + BZ2_bzWriteClose(&bzerror, self->fp, + 0, NULL, NULL); + break; + } + Util_DropReadAhead(self); + Py_XDECREF(self->file); + self->ob_type->tp_free((PyObject *)self); +} + +/* This is a hacked version of Python's fileobject.c:file_getiter(). */ +static PyObject * +BZ2File_getiter(BZ2FileObject *self) +{ + if (self->mode == MODE_CLOSED) { + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + return NULL; + } + Py_INCREF((PyObject*)self); + return (PyObject *)self; +} + +/* This is a hacked version of Python's fileobject.c:file_iternext(). */ +#define READAHEAD_BUFSIZE 8192 +static PyObject * +BZ2File_iternext(BZ2FileObject *self) +{ + PyStringObject* ret; + ACQUIRE_LOCK(self); + if (self->mode == MODE_CLOSED) { + PyErr_SetString(PyExc_ValueError, + "I/O operation on closed file"); + return NULL; + } + ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); + RELEASE_LOCK(self); + if (ret == NULL || PyString_GET_SIZE(ret) == 0) { + Py_XDECREF(ret); + return NULL; + } + return (PyObject *)ret; +} + +/* ===================================================================== */ +/* BZ2File_Type definition. */ + +PyDoc_VAR(BZ2File__doc__) = +PyDoc_STR( +"BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object\n\ +\n\ +Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or\n\ +writing. When opened for writing, the file will be created if it doesn't\n\ +exist, and truncated otherwise. If the buffering argument is given, 0 means\n\ +unbuffered, and larger numbers specify the buffer size. If compresslevel\n\ +is given, must be a number between 1 and 9.\n\ +") +PyDoc_STR( +"\n\ +Add a 'U' to mode to open the file for input with universal newline\n\ +support. Any line ending in the input file will be seen as a '\\n' in\n\ +Python. Also, a file so opened gains the attribute 'newlines'; the value\n\ +for this attribute is one of None (no newline read yet), '\\r', '\\n',\n\ +'\\r\\n' or a tuple containing all the newline types seen. Universal\n\ +newlines are available only when reading.\n\ +") +; + +static PyTypeObject BZ2File_Type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "bz2.BZ2File", /*tp_name*/ + sizeof(BZ2FileObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BZ2File_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + PyObject_GenericGetAttr,/*tp_getattro*/ + PyObject_GenericSetAttr,/*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + BZ2File__doc__, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + (getiterfunc)BZ2File_getiter, /*tp_iter*/ + (iternextfunc)BZ2File_iternext, /*tp_iternext*/ + BZ2File_methods, /*tp_methods*/ + BZ2File_members, /*tp_members*/ + BZ2File_getset, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)BZ2File_init, /*tp_init*/ + PyType_GenericAlloc, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + _PyObject_Del, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + + +/* ===================================================================== */ +/* Methods of BZ2Comp. */ + +PyDoc_STRVAR(BZ2Comp_compress__doc__, +"compress(data) -> string\n\ +\n\ +Provide more data to the compressor object. It will return chunks of\n\ +compressed data whenever possible. When you've finished providing data\n\ +to compress, call the flush() method to finish the compression process,\n\ +and return what is left in the internal buffers.\n\ +"); + +static PyObject * +BZ2Comp_compress(BZ2CompObject *self, PyObject *args) +{ + char *data; + int datasize; + int bufsize = SMALLCHUNK; + PY_LONG_LONG totalout; + PyObject *ret = NULL; + bz_stream *bzs = &self->bzs; + int bzerror; + + if (!PyArg_ParseTuple(args, "s#:compress", &data, &datasize)) + return NULL; + + if (datasize == 0) + return PyString_FromString(""); + + ACQUIRE_LOCK(self); + if (!self->running) { + PyErr_SetString(PyExc_ValueError, + "this object was already flushed"); + goto error; + } + + ret = PyString_FromStringAndSize(NULL, bufsize); + if (!ret) + goto error; + + bzs->next_in = data; + bzs->avail_in = datasize; + bzs->next_out = BUF(ret); + bzs->avail_out = bufsize; + + totalout = BZS_TOTAL_OUT(bzs); + + for (;;) { + Py_BEGIN_ALLOW_THREADS + bzerror = BZ2_bzCompress(bzs, BZ_RUN); + Py_END_ALLOW_THREADS + if (bzerror != BZ_RUN_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + if (bzs->avail_out == 0) { + bufsize = Util_NewBufferSize(bufsize); + if (_PyString_Resize(&ret, bufsize) < 0) { + BZ2_bzCompressEnd(bzs); + goto error; + } + bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) + - totalout); + bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); + } else if (bzs->avail_in == 0) { + break; + } + } + + _PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); + + RELEASE_LOCK(self); + return ret; + +error: + RELEASE_LOCK(self); + Py_XDECREF(ret); + return NULL; +} + +PyDoc_STRVAR(BZ2Comp_flush__doc__, +"flush() -> string\n\ +\n\ +Finish the compression process and return what is left in internal buffers.\n\ +You must not use the compressor object after calling this method.\n\ +"); + +static PyObject * +BZ2Comp_flush(BZ2CompObject *self) +{ + int bufsize = SMALLCHUNK; + PyObject *ret = NULL; + bz_stream *bzs = &self->bzs; + PY_LONG_LONG totalout; + int bzerror; + + ACQUIRE_LOCK(self); + if (!self->running) { + PyErr_SetString(PyExc_ValueError, "object was already " + "flushed"); + goto error; + } + self->running = 0; + + ret = PyString_FromStringAndSize(NULL, bufsize); + if (!ret) + goto error; + + bzs->next_out = BUF(ret); + bzs->avail_out = bufsize; + + totalout = BZS_TOTAL_OUT(bzs); + + for (;;) { + Py_BEGIN_ALLOW_THREADS + bzerror = BZ2_bzCompress(bzs, BZ_FINISH); + Py_END_ALLOW_THREADS + if (bzerror == BZ_STREAM_END) { + break; + } else if (bzerror != BZ_FINISH_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + if (bzs->avail_out == 0) { + bufsize = Util_NewBufferSize(bufsize); + if (_PyString_Resize(&ret, bufsize) < 0) + goto error; + bzs->next_out = BUF(ret); + bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) + - totalout); + bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); + } + } + + if (bzs->avail_out != 0) + _PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); + + RELEASE_LOCK(self); + return ret; + +error: + RELEASE_LOCK(self); + Py_XDECREF(ret); + return NULL; +} + +static PyMethodDef BZ2Comp_methods[] = { + {"compress", (PyCFunction)BZ2Comp_compress, METH_VARARGS, + BZ2Comp_compress__doc__}, + {"flush", (PyCFunction)BZ2Comp_flush, METH_NOARGS, + BZ2Comp_flush__doc__}, + {NULL, NULL} /* sentinel */ +}; + + +/* ===================================================================== */ +/* Slot definitions for BZ2Comp_Type. */ + +static int +BZ2Comp_init(BZ2CompObject *self, PyObject *args, PyObject *kwargs) +{ + int compresslevel = 9; + int bzerror; + static char *kwlist[] = {"compresslevel", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:BZ2Compressor", + kwlist, &compresslevel)) + return -1; + + if (compresslevel < 1 || compresslevel > 9) { + PyErr_SetString(PyExc_ValueError, + "compresslevel must be between 1 and 9"); + goto error; + } + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (!self->lock) + goto error; +#endif + + memset(&self->bzs, 0, sizeof(bz_stream)); + bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + + self->running = 1; + + return 0; +error: +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + return -1; +} + +static void +BZ2Comp_dealloc(BZ2CompObject *self) +{ +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + BZ2_bzCompressEnd(&self->bzs); + self->ob_type->tp_free((PyObject *)self); +} + + +/* ===================================================================== */ +/* BZ2Comp_Type definition. */ + +PyDoc_STRVAR(BZ2Comp__doc__, +"BZ2Compressor([compresslevel=9]) -> compressor object\n\ +\n\ +Create a new compressor object. This object may be used to compress\n\ +data sequentially. If you want to compress data in one shot, use the\n\ +compress() function instead. The compresslevel parameter, if given,\n\ +must be a number between 1 and 9.\n\ +"); + +static PyTypeObject BZ2Comp_Type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "bz2.BZ2Compressor", /*tp_name*/ + sizeof(BZ2CompObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BZ2Comp_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + PyObject_GenericGetAttr,/*tp_getattro*/ + PyObject_GenericSetAttr,/*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + BZ2Comp__doc__, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + BZ2Comp_methods, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)BZ2Comp_init, /*tp_init*/ + PyType_GenericAlloc, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + _PyObject_Del, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + + +/* ===================================================================== */ +/* Members of BZ2Decomp. */ + +#undef OFF +#define OFF(x) offsetof(BZ2DecompObject, x) + +static PyMemberDef BZ2Decomp_members[] = { + {"unused_data", T_OBJECT, OFF(unused_data), RO}, + {NULL} /* Sentinel */ +}; + + +/* ===================================================================== */ +/* Methods of BZ2Decomp. */ + +PyDoc_STRVAR(BZ2Decomp_decompress__doc__, +"decompress(data) -> string\n\ +\n\ +Provide more data to the decompressor object. It will return chunks\n\ +of decompressed data whenever possible. If you try to decompress data\n\ +after the end of stream is found, EOFError will be raised. If any data\n\ +was found after the end of stream, it'll be ignored and saved in\n\ +unused_data attribute.\n\ +"); + +static PyObject * +BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) +{ + char *data; + int datasize; + int bufsize = SMALLCHUNK; + PY_LONG_LONG totalout; + PyObject *ret = NULL; + bz_stream *bzs = &self->bzs; + int bzerror; + + if (!PyArg_ParseTuple(args, "s#:decompress", &data, &datasize)) + return NULL; + + ACQUIRE_LOCK(self); + if (!self->running) { + PyErr_SetString(PyExc_EOFError, "end of stream was " + "already found"); + goto error; + } + + ret = PyString_FromStringAndSize(NULL, bufsize); + if (!ret) + goto error; + + bzs->next_in = data; + bzs->avail_in = datasize; + bzs->next_out = BUF(ret); + bzs->avail_out = bufsize; + + totalout = BZS_TOTAL_OUT(bzs); + + for (;;) { + Py_BEGIN_ALLOW_THREADS + bzerror = BZ2_bzDecompress(bzs); + Py_END_ALLOW_THREADS + if (bzerror == BZ_STREAM_END) { + if (bzs->avail_in != 0) { + Py_DECREF(self->unused_data); + self->unused_data = + PyString_FromStringAndSize(bzs->next_in, + bzs->avail_in); + } + self->running = 0; + break; + } + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + if (bzs->avail_out == 0) { + bufsize = Util_NewBufferSize(bufsize); + if (_PyString_Resize(&ret, bufsize) < 0) { + BZ2_bzDecompressEnd(bzs); + goto error; + } + bzs->next_out = BUF(ret); + bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) + - totalout); + bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); + } else if (bzs->avail_in == 0) { + break; + } + } + + if (bzs->avail_out != 0) + _PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); + + RELEASE_LOCK(self); + return ret; + +error: + RELEASE_LOCK(self); + Py_XDECREF(ret); + return NULL; +} + +static PyMethodDef BZ2Decomp_methods[] = { + {"decompress", (PyCFunction)BZ2Decomp_decompress, METH_VARARGS, BZ2Decomp_decompress__doc__}, + {NULL, NULL} /* sentinel */ +}; + + +/* ===================================================================== */ +/* Slot definitions for BZ2Decomp_Type. */ + +static int +BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs) +{ + int bzerror; + + if (!PyArg_ParseTuple(args, ":BZ2Decompressor")) + return -1; + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (!self->lock) + goto error; +#endif + + self->unused_data = PyString_FromString(""); + if (!self->unused_data) + goto error; + + memset(&self->bzs, 0, sizeof(bz_stream)); + bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + goto error; + } + + self->running = 1; + + return 0; + +error: +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + Py_XDECREF(self->unused_data); + return -1; +} + +static void +BZ2Decomp_dealloc(BZ2DecompObject *self) +{ +#ifdef WITH_THREAD + if (self->lock) + PyThread_free_lock(self->lock); +#endif + Py_XDECREF(self->unused_data); + BZ2_bzDecompressEnd(&self->bzs); + self->ob_type->tp_free((PyObject *)self); +} + + +/* ===================================================================== */ +/* BZ2Decomp_Type definition. */ + +PyDoc_STRVAR(BZ2Decomp__doc__, +"BZ2Decompressor() -> decompressor object\n\ +\n\ +Create a new decompressor object. This object may be used to decompress\n\ +data sequentially. If you want to decompress data in one shot, use the\n\ +decompress() function instead.\n\ +"); + +static PyTypeObject BZ2Decomp_Type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "bz2.BZ2Decompressor", /*tp_name*/ + sizeof(BZ2DecompObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BZ2Decomp_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + PyObject_GenericGetAttr,/*tp_getattro*/ + PyObject_GenericSetAttr,/*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + BZ2Decomp__doc__, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + BZ2Decomp_methods, /*tp_methods*/ + BZ2Decomp_members, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)BZ2Decomp_init, /*tp_init*/ + PyType_GenericAlloc, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + _PyObject_Del, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + + +/* ===================================================================== */ +/* Module functions. */ + +PyDoc_STRVAR(bz2_compress__doc__, +"compress(data [, compresslevel=9]) -> string\n\ +\n\ +Compress data in one shot. If you want to compress data sequentially,\n\ +use an instance of BZ2Compressor instead. The compresslevel parameter, if\n\ +given, must be a number between 1 and 9.\n\ +"); + +static PyObject * +bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int compresslevel=9; + char *data; + int datasize; + int bufsize; + PyObject *ret = NULL; + bz_stream _bzs; + bz_stream *bzs = &_bzs; + int bzerror; + static char *kwlist[] = {"data", "compresslevel", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|i", + kwlist, &data, &datasize, + &compresslevel)) + return NULL; + + if (compresslevel < 1 || compresslevel > 9) { + PyErr_SetString(PyExc_ValueError, + "compresslevel must be between 1 and 9"); + return NULL; + } + + /* Conforming to bz2 manual, this is large enough to fit compressed + * data in one shot. We will check it later anyway. */ + bufsize = datasize + (datasize/100+1) + 600; + + ret = PyString_FromStringAndSize(NULL, bufsize); + if (!ret) + return NULL; + + memset(bzs, 0, sizeof(bz_stream)); + + bzs->next_in = data; + bzs->avail_in = datasize; + bzs->next_out = BUF(ret); + bzs->avail_out = bufsize; + + bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Py_DECREF(ret); + return NULL; + } + + for (;;) { + Py_BEGIN_ALLOW_THREADS + bzerror = BZ2_bzCompress(bzs, BZ_FINISH); + Py_END_ALLOW_THREADS + if (bzerror == BZ_STREAM_END) { + break; + } else if (bzerror != BZ_FINISH_OK) { + BZ2_bzCompressEnd(bzs); + Util_CatchBZ2Error(bzerror); + Py_DECREF(ret); + return NULL; + } + if (bzs->avail_out == 0) { + bufsize = Util_NewBufferSize(bufsize); + if (_PyString_Resize(&ret, bufsize) < 0) { + BZ2_bzCompressEnd(bzs); + Py_DECREF(ret); + return NULL; + } + bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); + bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); + } + } + + if (bzs->avail_out != 0) + _PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); + BZ2_bzCompressEnd(bzs); + + return ret; +} + +PyDoc_STRVAR(bz2_decompress__doc__, +"decompress(data) -> decompressed data\n\ +\n\ +Decompress data in one shot. If you want to decompress data sequentially,\n\ +use an instance of BZ2Decompressor instead.\n\ +"); + +static PyObject * +bz2_decompress(PyObject *self, PyObject *args) +{ + char *data; + int datasize; + int bufsize = SMALLCHUNK; + PyObject *ret; + bz_stream _bzs; + bz_stream *bzs = &_bzs; + int bzerror; + + if (!PyArg_ParseTuple(args, "s#:decompress", &data, &datasize)) + return NULL; + + if (datasize == 0) + return PyString_FromString(""); + + ret = PyString_FromStringAndSize(NULL, bufsize); + if (!ret) + return NULL; + + memset(bzs, 0, sizeof(bz_stream)); + + bzs->next_in = data; + bzs->avail_in = datasize; + bzs->next_out = BUF(ret); + bzs->avail_out = bufsize; + + bzerror = BZ2_bzDecompressInit(bzs, 0, 0); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + Py_DECREF(ret); + return NULL; + } + + for (;;) { + Py_BEGIN_ALLOW_THREADS + bzerror = BZ2_bzDecompress(bzs); + Py_END_ALLOW_THREADS + if (bzerror == BZ_STREAM_END) { + break; + } else if (bzerror != BZ_OK) { + BZ2_bzDecompressEnd(bzs); + Util_CatchBZ2Error(bzerror); + Py_DECREF(ret); + return NULL; + } + if (bzs->avail_out == 0) { + bufsize = Util_NewBufferSize(bufsize); + if (_PyString_Resize(&ret, bufsize) < 0) { + BZ2_bzDecompressEnd(bzs); + Py_DECREF(ret); + return NULL; + } + bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); + bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); + } else if (bzs->avail_in == 0) { + BZ2_bzDecompressEnd(bzs); + PyErr_SetString(PyExc_ValueError, + "couldn't find end of stream"); + Py_DECREF(ret); + return NULL; + } + } + + if (bzs->avail_out != 0) + _PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); + BZ2_bzDecompressEnd(bzs); + + return ret; +} + +static PyMethodDef bz2_methods[] = { + {"compress", (PyCFunction) bz2_compress, METH_VARARGS|METH_KEYWORDS, + bz2_compress__doc__}, + {"decompress", (PyCFunction) bz2_decompress, METH_VARARGS, + bz2_decompress__doc__}, + {NULL, NULL} /* sentinel */ +}; + +/* ===================================================================== */ +/* Initialization function. */ + +PyDoc_STRVAR(bz2__doc__, +"The python bz2 module provides a comprehensive interface for\n\ +the bz2 compression library. It implements a complete file\n\ +interface, one shot (de)compression functions, and types for\n\ +sequential (de)compression.\n\ +"); + +PyMODINIT_FUNC +initbz2(void) +{ + PyObject *m; + + BZ2File_Type.ob_type = &PyType_Type; + BZ2Comp_Type.ob_type = &PyType_Type; + BZ2Decomp_Type.ob_type = &PyType_Type; + + m = Py_InitModule3("bz2", bz2_methods, bz2__doc__); + if (m == NULL) + return; + + PyModule_AddObject(m, "__author__", PyString_FromString(__author__)); + + Py_INCREF(&BZ2File_Type); + PyModule_AddObject(m, "BZ2File", (PyObject *)&BZ2File_Type); + + Py_INCREF(&BZ2Comp_Type); + PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Comp_Type); + + Py_INCREF(&BZ2Decomp_Type); + PyModule_AddObject(m, "BZ2Decompressor", (PyObject *)&BZ2Decomp_Type); +} Added: pypy/dist/pypy/module/bz2/bzlib.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/bzlib.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,283 @@ +from ctypes import * +STRING = c_char_p + + +__darwin_nl_item = c_int +__darwin_wctrans_t = c_int +__darwin_wctype_t = c_ulong +class bz_stream(Structure): + pass +bz_stream._fields_ = [ + ('next_in', STRING), + ('avail_in', c_uint), + ('total_in_lo32', c_uint), + ('total_in_hi32', c_uint), + ('next_out', STRING), + ('avail_out', c_uint), + ('total_out_lo32', c_uint), + ('total_out_hi32', c_uint), + ('state', c_void_p), + ('bzalloc', CFUNCTYPE(c_void_p, c_void_p, c_int, c_int)), + ('bzfree', CFUNCTYPE(None, c_void_p, c_void_p)), + ('opaque', c_void_p), +] +assert sizeof(bz_stream) == 48, sizeof(bz_stream) +assert alignment(bz_stream) == 4, alignment(bz_stream) +BZFILE = None +__int8_t = c_byte +__uint8_t = c_ubyte +__int16_t = c_short +__uint16_t = c_ushort +__int32_t = c_int +__uint32_t = c_uint +__int64_t = c_longlong +__uint64_t = c_ulonglong +__darwin_intptr_t = c_long +__darwin_natural_t = c_uint +__darwin_ct_rune_t = c_int +class __mbstate_t(Union): + pass +__mbstate_t._pack_ = 4 +__mbstate_t._fields_ = [ + ('__mbstate8', c_char * 128), + ('_mbstateL', c_longlong), +] +assert sizeof(__mbstate_t) == 128, sizeof(__mbstate_t) +assert alignment(__mbstate_t) == 4, alignment(__mbstate_t) +__darwin_mbstate_t = __mbstate_t +__darwin_ptrdiff_t = c_int +__darwin_size_t = c_ulong +__darwin_va_list = STRING +__darwin_wchar_t = c_int +__darwin_rune_t = __darwin_wchar_t +__darwin_wint_t = c_int +__darwin_clock_t = c_ulong +__darwin_socklen_t = __uint32_t +__darwin_ssize_t = c_long +__darwin_time_t = c_long +va_list = __darwin_va_list +size_t = __darwin_size_t +__darwin_off_t = __int64_t +fpos_t = __darwin_off_t +class __sbuf(Structure): + pass +__sbuf._fields_ = [ + ('_base', POINTER(c_ubyte)), + ('_size', c_int), +] +assert sizeof(__sbuf) == 8, sizeof(__sbuf) +assert alignment(__sbuf) == 4, alignment(__sbuf) +class __sFILEX(Structure): + pass +class __sFILE(Structure): + pass +__sFILE._pack_ = 4 +__sFILE._fields_ = [ + ('_p', POINTER(c_ubyte)), + ('_r', c_int), + ('_w', c_int), + ('_flags', c_short), + ('_file', c_short), + ('_bf', __sbuf), + ('_lbfsize', c_int), + ('_cookie', c_void_p), + ('_close', CFUNCTYPE(c_int, c_void_p)), + ('_read', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), + ('_seek', CFUNCTYPE(fpos_t, c_void_p, c_longlong, c_int)), + ('_write', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), + ('_ub', __sbuf), + ('_extra', POINTER(__sFILEX)), + ('_ur', c_int), + ('_ubuf', c_ubyte * 3), + ('_nbuf', c_ubyte * 1), + ('_lb', __sbuf), + ('_blksize', c_int), + ('_offset', fpos_t), +] +assert sizeof(__sFILE) == 88, sizeof(__sFILE) +assert alignment(__sFILE) == 4, alignment(__sFILE) +FILE = __sFILE +class mcontext(Structure): + pass +class mcontext64(Structure): + pass +class __darwin_pthread_handler_rec(Structure): + pass +__darwin_pthread_handler_rec._fields_ = [ + ('__routine', CFUNCTYPE(None, c_void_p)), + ('__arg', c_void_p), + ('__next', POINTER(__darwin_pthread_handler_rec)), +] +assert sizeof(__darwin_pthread_handler_rec) == 12, sizeof(__darwin_pthread_handler_rec) +assert alignment(__darwin_pthread_handler_rec) == 4, alignment(__darwin_pthread_handler_rec) +class _opaque_pthread_attr_t(Structure): + pass +_opaque_pthread_attr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 36), +] +assert sizeof(_opaque_pthread_attr_t) == 40, sizeof(_opaque_pthread_attr_t) +assert alignment(_opaque_pthread_attr_t) == 4, alignment(_opaque_pthread_attr_t) +class _opaque_pthread_cond_t(Structure): + pass +_opaque_pthread_cond_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 24), +] +assert sizeof(_opaque_pthread_cond_t) == 28, sizeof(_opaque_pthread_cond_t) +assert alignment(_opaque_pthread_cond_t) == 4, alignment(_opaque_pthread_cond_t) +class _opaque_pthread_condattr_t(Structure): + pass +_opaque_pthread_condattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 4), +] +assert sizeof(_opaque_pthread_condattr_t) == 8, sizeof(_opaque_pthread_condattr_t) +assert alignment(_opaque_pthread_condattr_t) == 4, alignment(_opaque_pthread_condattr_t) +class _opaque_pthread_mutex_t(Structure): + pass +_opaque_pthread_mutex_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 40), +] +assert sizeof(_opaque_pthread_mutex_t) == 44, sizeof(_opaque_pthread_mutex_t) +assert alignment(_opaque_pthread_mutex_t) == 4, alignment(_opaque_pthread_mutex_t) +class _opaque_pthread_mutexattr_t(Structure): + pass +_opaque_pthread_mutexattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 8), +] +assert sizeof(_opaque_pthread_mutexattr_t) == 12, sizeof(_opaque_pthread_mutexattr_t) +assert alignment(_opaque_pthread_mutexattr_t) == 4, alignment(_opaque_pthread_mutexattr_t) +class _opaque_pthread_once_t(Structure): + pass +_opaque_pthread_once_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 4), +] +assert sizeof(_opaque_pthread_once_t) == 8, sizeof(_opaque_pthread_once_t) +assert alignment(_opaque_pthread_once_t) == 4, alignment(_opaque_pthread_once_t) +class _opaque_pthread_rwlock_t(Structure): + pass +_opaque_pthread_rwlock_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 124), +] +assert sizeof(_opaque_pthread_rwlock_t) == 128, sizeof(_opaque_pthread_rwlock_t) +assert alignment(_opaque_pthread_rwlock_t) == 4, alignment(_opaque_pthread_rwlock_t) +class _opaque_pthread_rwlockattr_t(Structure): + pass +_opaque_pthread_rwlockattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 12), +] +assert sizeof(_opaque_pthread_rwlockattr_t) == 16, sizeof(_opaque_pthread_rwlockattr_t) +assert alignment(_opaque_pthread_rwlockattr_t) == 4, alignment(_opaque_pthread_rwlockattr_t) +class _opaque_pthread_t(Structure): + pass +_opaque_pthread_t._fields_ = [ + ('__sig', c_long), + ('__cleanup_stack', POINTER(__darwin_pthread_handler_rec)), + ('__opaque', c_char * 596), +] +assert sizeof(_opaque_pthread_t) == 604, sizeof(_opaque_pthread_t) +assert alignment(_opaque_pthread_t) == 4, alignment(_opaque_pthread_t) +__darwin_blkcnt_t = __int64_t +__darwin_blksize_t = __int32_t +__darwin_dev_t = __int32_t +__darwin_fsblkcnt_t = c_uint +__darwin_fsfilcnt_t = c_uint +__darwin_gid_t = __uint32_t +__darwin_id_t = __uint32_t +__darwin_ino_t = __uint32_t +__darwin_mach_port_name_t = __darwin_natural_t +__darwin_mach_port_t = __darwin_mach_port_name_t +__darwin_mcontext_t = POINTER(mcontext) +__darwin_mcontext64_t = POINTER(mcontext64) +__darwin_mode_t = __uint16_t +__darwin_pid_t = __int32_t +__darwin_pthread_attr_t = _opaque_pthread_attr_t +__darwin_pthread_cond_t = _opaque_pthread_cond_t +__darwin_pthread_condattr_t = _opaque_pthread_condattr_t +__darwin_pthread_key_t = c_ulong +__darwin_pthread_mutex_t = _opaque_pthread_mutex_t +__darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t +__darwin_pthread_once_t = _opaque_pthread_once_t +__darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t +__darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t +__darwin_pthread_t = POINTER(_opaque_pthread_t) +__darwin_sigset_t = __uint32_t +__darwin_suseconds_t = __int32_t +__darwin_uid_t = __uint32_t +__darwin_useconds_t = __uint32_t +__darwin_uuid_t = c_ubyte * 16 +class sigaltstack(Structure): + pass +sigaltstack._fields_ = [ + ('ss_sp', c_void_p), + ('ss_size', __darwin_size_t), + ('ss_flags', c_int), +] +assert sizeof(sigaltstack) == 12, sizeof(sigaltstack) +assert alignment(sigaltstack) == 4, alignment(sigaltstack) +__darwin_stack_t = sigaltstack +class ucontext(Structure): + pass +ucontext._fields_ = [ + ('uc_onstack', c_int), + ('uc_sigmask', __darwin_sigset_t), + ('uc_stack', __darwin_stack_t), + ('uc_link', POINTER(ucontext)), + ('uc_mcsize', __darwin_size_t), + ('uc_mcontext', __darwin_mcontext_t), +] +assert sizeof(ucontext) == 32, sizeof(ucontext) +assert alignment(ucontext) == 4, alignment(ucontext) +__darwin_ucontext_t = ucontext +class ucontext64(Structure): + pass +ucontext64._fields_ = [ + ('uc_onstack', c_int), + ('uc_sigmask', __darwin_sigset_t), + ('uc_stack', __darwin_stack_t), + ('uc_link', POINTER(ucontext64)), + ('uc_mcsize', __darwin_size_t), + ('uc_mcontext64', __darwin_mcontext64_t), +] +assert sizeof(ucontext64) == 32, sizeof(ucontext64) +assert alignment(ucontext64) == 4, alignment(ucontext64) +__darwin_ucontext64_t = ucontext64 +__all__ = ['__uint16_t', '__darwin_off_t', '__uint64_t', '__int16_t', + '__darwin_pthread_condattr_t', '__darwin_pthread_key_t', + '__darwin_pthread_handler_rec', '__darwin_id_t', + '__darwin_ucontext64_t', '__darwin_pthread_mutex_t', + '__darwin_wctype_t', '_opaque_pthread_once_t', + '__darwin_time_t', '__darwin_nl_item', + '_opaque_pthread_condattr_t', '__darwin_pthread_rwlock_t', + 'FILE', 'size_t', '__darwin_pthread_rwlockattr_t', + '__darwin_fsblkcnt_t', '__darwin_rune_t', + '__darwin_intptr_t', 'ucontext64', '_opaque_pthread_t', + '__darwin_va_list', '__uint32_t', '__darwin_sigset_t', + '__sbuf', 'fpos_t', '_opaque_pthread_mutexattr_t', + '__darwin_socklen_t', '__darwin_suseconds_t', + '__darwin_gid_t', '__darwin_uuid_t', '__darwin_dev_t', + '__darwin_pthread_mutexattr_t', '__darwin_stack_t', + '__darwin_pthread_once_t', '__darwin_ucontext_t', + '__darwin_blksize_t', '__darwin_mode_t', '__sFILE', + '__darwin_wctrans_t', '__darwin_fsfilcnt_t', '__mbstate_t', + '__sFILEX', '_opaque_pthread_rwlockattr_t', + '__darwin_mach_port_t', '__uint8_t', '__darwin_uid_t', + '__int8_t', 'sigaltstack', '__darwin_wchar_t', + '_opaque_pthread_mutex_t', '__darwin_pthread_attr_t', + '__darwin_mach_port_name_t', 'BZFILE', + '__darwin_pthread_t', '_opaque_pthread_attr_t', 'ucontext', + '__darwin_wint_t', '__darwin_pthread_cond_t', + '__darwin_mcontext64_t', '__darwin_natural_t', + '_opaque_pthread_cond_t', '__darwin_blkcnt_t', 'mcontext', + '__darwin_ssize_t', 'mcontext64', '__darwin_mcontext_t', + '__darwin_size_t', '__darwin_pid_t', '__darwin_ptrdiff_t', + '_opaque_pthread_rwlock_t', '__darwin_ct_rune_t', + 'va_list', '__darwin_ino_t', '__int32_t', '__int64_t', + '__darwin_mbstate_t', '__darwin_useconds_t', 'bz_stream', + '__darwin_clock_t'] Added: pypy/dist/pypy/module/bz2/fileobject.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/fileobject.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,1008 @@ +from ctypes import * +STRING = c_char_p + + +P_ALL = 0 +OSUnknownByteOrder = 0 +OSBigEndian = 2 +_FP_NAN = 1 +_FP_INFINITE = 2 +P_PGID = 2 +_FP_NORMAL = 4 +OSLittleEndian = 1 +P_PID = 1 +_FP_SUBNORMAL = 5 +_FP_SUPERNORMAL = 6 +_FP_ZERO = 3 +__darwin_nl_item = c_int +__darwin_wctrans_t = c_int +__darwin_wctype_t = c_ulong +float_t = c_float +double_t = c_double + +# values for unnamed enumeration +__int8_t = c_byte +__uint8_t = c_ubyte +__int16_t = c_short +__uint16_t = c_ushort +__int32_t = c_int +__uint32_t = c_uint +__int64_t = c_longlong +__uint64_t = c_ulonglong +__darwin_intptr_t = c_long +__darwin_natural_t = c_uint +__darwin_ct_rune_t = c_int +class __mbstate_t(Union): + pass +__mbstate_t._pack_ = 4 +__mbstate_t._fields_ = [ + ('__mbstate8', c_char * 128), + ('_mbstateL', c_longlong), +] +assert sizeof(__mbstate_t) == 128, sizeof(__mbstate_t) +assert alignment(__mbstate_t) == 4, alignment(__mbstate_t) +__darwin_mbstate_t = __mbstate_t +__darwin_ptrdiff_t = c_int +__darwin_size_t = c_ulong +__darwin_va_list = STRING +__darwin_wchar_t = c_int +__darwin_rune_t = __darwin_wchar_t +__darwin_wint_t = c_int +__darwin_clock_t = c_ulong +__darwin_socklen_t = __uint32_t +__darwin_ssize_t = c_long +__darwin_time_t = c_long +sig_atomic_t = c_int +class sigcontext(Structure): + pass +sigcontext._fields_ = [ + ('sc_onstack', c_int), + ('sc_mask', c_int), + ('sc_eax', c_uint), + ('sc_ebx', c_uint), + ('sc_ecx', c_uint), + ('sc_edx', c_uint), + ('sc_edi', c_uint), + ('sc_esi', c_uint), + ('sc_ebp', c_uint), + ('sc_esp', c_uint), + ('sc_ss', c_uint), + ('sc_eflags', c_uint), + ('sc_eip', c_uint), + ('sc_cs', c_uint), + ('sc_ds', c_uint), + ('sc_es', c_uint), + ('sc_fs', c_uint), + ('sc_gs', c_uint), +] +assert sizeof(sigcontext) == 72, sizeof(sigcontext) +assert alignment(sigcontext) == 4, alignment(sigcontext) +int8_t = c_byte +u_int8_t = c_ubyte +int16_t = c_short +u_int16_t = c_ushort +int32_t = c_int +u_int32_t = c_uint +int64_t = c_longlong +u_int64_t = c_ulonglong +register_t = int32_t +intptr_t = __darwin_intptr_t +uintptr_t = c_ulong +user_addr_t = u_int64_t +user_size_t = u_int64_t +user_ssize_t = int64_t +user_long_t = int64_t +user_ulong_t = u_int64_t +user_time_t = int64_t +syscall_arg_t = u_int64_t + +# values for unnamed enumeration +va_list = __darwin_va_list +size_t = __darwin_size_t +__darwin_off_t = __int64_t +fpos_t = __darwin_off_t +class __sbuf(Structure): + pass +__sbuf._fields_ = [ + ('_base', POINTER(c_ubyte)), + ('_size', c_int), +] +assert sizeof(__sbuf) == 8, sizeof(__sbuf) +assert alignment(__sbuf) == 4, alignment(__sbuf) +class __sFILEX(Structure): + pass +class __sFILE(Structure): + pass +__sFILE._pack_ = 4 +__sFILE._fields_ = [ + ('_p', POINTER(c_ubyte)), + ('_r', c_int), + ('_w', c_int), + ('_flags', c_short), + ('_file', c_short), + ('_bf', __sbuf), + ('_lbfsize', c_int), + ('_cookie', c_void_p), + ('_close', CFUNCTYPE(c_int, c_void_p)), + ('_read', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), + ('_seek', CFUNCTYPE(fpos_t, c_void_p, c_longlong, c_int)), + ('_write', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), + ('_ub', __sbuf), + ('_extra', POINTER(__sFILEX)), + ('_ur', c_int), + ('_ubuf', c_ubyte * 3), + ('_nbuf', c_ubyte * 1), + ('_lb', __sbuf), + ('_blksize', c_int), + ('_offset', fpos_t), +] +assert sizeof(__sFILE) == 88, sizeof(__sFILE) +assert alignment(__sFILE) == 4, alignment(__sFILE) +FILE = __sFILE +ct_rune_t = __darwin_ct_rune_t +rune_t = __darwin_rune_t +class div_t(Structure): + pass +div_t._fields_ = [ + ('quot', c_int), + ('rem', c_int), +] +assert sizeof(div_t) == 8, sizeof(div_t) +assert alignment(div_t) == 4, alignment(div_t) +class ldiv_t(Structure): + pass +ldiv_t._fields_ = [ + ('quot', c_long), + ('rem', c_long), +] +assert sizeof(ldiv_t) == 8, sizeof(ldiv_t) +assert alignment(ldiv_t) == 4, alignment(ldiv_t) +class lldiv_t(Structure): + pass +lldiv_t._pack_ = 4 +lldiv_t._fields_ = [ + ('quot', c_longlong), + ('rem', c_longlong), +] +assert sizeof(lldiv_t) == 16, sizeof(lldiv_t) +assert alignment(lldiv_t) == 4, alignment(lldiv_t) +class mcontext(Structure): + pass +class mcontext64(Structure): + pass +class __darwin_pthread_handler_rec(Structure): + pass +__darwin_pthread_handler_rec._fields_ = [ + ('__routine', CFUNCTYPE(None, c_void_p)), + ('__arg', c_void_p), + ('__next', POINTER(__darwin_pthread_handler_rec)), +] +assert sizeof(__darwin_pthread_handler_rec) == 12, sizeof(__darwin_pthread_handler_rec) +assert alignment(__darwin_pthread_handler_rec) == 4, alignment(__darwin_pthread_handler_rec) +class _opaque_pthread_attr_t(Structure): + pass +_opaque_pthread_attr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 36), +] +assert sizeof(_opaque_pthread_attr_t) == 40, sizeof(_opaque_pthread_attr_t) +assert alignment(_opaque_pthread_attr_t) == 4, alignment(_opaque_pthread_attr_t) +class _opaque_pthread_cond_t(Structure): + pass +_opaque_pthread_cond_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 24), +] +assert sizeof(_opaque_pthread_cond_t) == 28, sizeof(_opaque_pthread_cond_t) +assert alignment(_opaque_pthread_cond_t) == 4, alignment(_opaque_pthread_cond_t) +class _opaque_pthread_condattr_t(Structure): + pass +_opaque_pthread_condattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 4), +] +assert sizeof(_opaque_pthread_condattr_t) == 8, sizeof(_opaque_pthread_condattr_t) +assert alignment(_opaque_pthread_condattr_t) == 4, alignment(_opaque_pthread_condattr_t) +class _opaque_pthread_mutex_t(Structure): + pass +_opaque_pthread_mutex_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 40), +] +assert sizeof(_opaque_pthread_mutex_t) == 44, sizeof(_opaque_pthread_mutex_t) +assert alignment(_opaque_pthread_mutex_t) == 4, alignment(_opaque_pthread_mutex_t) +class _opaque_pthread_mutexattr_t(Structure): + pass +_opaque_pthread_mutexattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 8), +] +assert sizeof(_opaque_pthread_mutexattr_t) == 12, sizeof(_opaque_pthread_mutexattr_t) +assert alignment(_opaque_pthread_mutexattr_t) == 4, alignment(_opaque_pthread_mutexattr_t) +class _opaque_pthread_once_t(Structure): + pass +_opaque_pthread_once_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 4), +] +assert sizeof(_opaque_pthread_once_t) == 8, sizeof(_opaque_pthread_once_t) +assert alignment(_opaque_pthread_once_t) == 4, alignment(_opaque_pthread_once_t) +class _opaque_pthread_rwlock_t(Structure): + pass +_opaque_pthread_rwlock_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 124), +] +assert sizeof(_opaque_pthread_rwlock_t) == 128, sizeof(_opaque_pthread_rwlock_t) +assert alignment(_opaque_pthread_rwlock_t) == 4, alignment(_opaque_pthread_rwlock_t) +class _opaque_pthread_rwlockattr_t(Structure): + pass +_opaque_pthread_rwlockattr_t._fields_ = [ + ('__sig', c_long), + ('__opaque', c_char * 12), +] +assert sizeof(_opaque_pthread_rwlockattr_t) == 16, sizeof(_opaque_pthread_rwlockattr_t) +assert alignment(_opaque_pthread_rwlockattr_t) == 4, alignment(_opaque_pthread_rwlockattr_t) +class _opaque_pthread_t(Structure): + pass +_opaque_pthread_t._fields_ = [ + ('__sig', c_long), + ('__cleanup_stack', POINTER(__darwin_pthread_handler_rec)), + ('__opaque', c_char * 596), +] +assert sizeof(_opaque_pthread_t) == 604, sizeof(_opaque_pthread_t) +assert alignment(_opaque_pthread_t) == 4, alignment(_opaque_pthread_t) +__darwin_blkcnt_t = __int64_t +__darwin_blksize_t = __int32_t +__darwin_dev_t = __int32_t +__darwin_fsblkcnt_t = c_uint +__darwin_fsfilcnt_t = c_uint +__darwin_gid_t = __uint32_t +__darwin_id_t = __uint32_t +__darwin_ino_t = __uint32_t +__darwin_mach_port_name_t = __darwin_natural_t +__darwin_mach_port_t = __darwin_mach_port_name_t +__darwin_mcontext_t = POINTER(mcontext) +__darwin_mcontext64_t = POINTER(mcontext64) +__darwin_mode_t = __uint16_t +__darwin_pid_t = __int32_t +__darwin_pthread_attr_t = _opaque_pthread_attr_t +__darwin_pthread_cond_t = _opaque_pthread_cond_t +__darwin_pthread_condattr_t = _opaque_pthread_condattr_t +__darwin_pthread_key_t = c_ulong +__darwin_pthread_mutex_t = _opaque_pthread_mutex_t +__darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t +__darwin_pthread_once_t = _opaque_pthread_once_t +__darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t +__darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t +__darwin_pthread_t = POINTER(_opaque_pthread_t) +__darwin_sigset_t = __uint32_t +__darwin_suseconds_t = __int32_t +__darwin_uid_t = __uint32_t +__darwin_useconds_t = __uint32_t +__darwin_uuid_t = c_ubyte * 16 +class sigaltstack(Structure): + pass +sigaltstack._fields_ = [ + ('ss_sp', c_void_p), + ('ss_size', __darwin_size_t), + ('ss_flags', c_int), +] +assert sizeof(sigaltstack) == 12, sizeof(sigaltstack) +assert alignment(sigaltstack) == 4, alignment(sigaltstack) +__darwin_stack_t = sigaltstack +class ucontext(Structure): + pass +ucontext._fields_ = [ + ('uc_onstack', c_int), + ('uc_sigmask', __darwin_sigset_t), + ('uc_stack', __darwin_stack_t), + ('uc_link', POINTER(ucontext)), + ('uc_mcsize', __darwin_size_t), + ('uc_mcontext', __darwin_mcontext_t), +] +assert sizeof(ucontext) == 32, sizeof(ucontext) +assert alignment(ucontext) == 4, alignment(ucontext) +__darwin_ucontext_t = ucontext +class ucontext64(Structure): + pass +ucontext64._fields_ = [ + ('uc_onstack', c_int), + ('uc_sigmask', __darwin_sigset_t), + ('uc_stack', __darwin_stack_t), + ('uc_link', POINTER(ucontext64)), + ('uc_mcsize', __darwin_size_t), + ('uc_mcontext64', __darwin_mcontext64_t), +] +assert sizeof(ucontext64) == 32, sizeof(ucontext64) +assert alignment(ucontext64) == 4, alignment(ucontext64) +__darwin_ucontext64_t = ucontext64 +class timeval(Structure): + pass +timeval._fields_ = [ + ('tv_sec', __darwin_time_t), + ('tv_usec', __darwin_suseconds_t), +] +assert sizeof(timeval) == 8, sizeof(timeval) +assert alignment(timeval) == 4, alignment(timeval) +rlim_t = __int64_t +class rusage(Structure): + pass +rusage._fields_ = [ + ('ru_utime', timeval), + ('ru_stime', timeval), + ('ru_maxrss', c_long), + ('ru_ixrss', c_long), + ('ru_idrss', c_long), + ('ru_isrss', c_long), + ('ru_minflt', c_long), + ('ru_majflt', c_long), + ('ru_nswap', c_long), + ('ru_inblock', c_long), + ('ru_oublock', c_long), + ('ru_msgsnd', c_long), + ('ru_msgrcv', c_long), + ('ru_nsignals', c_long), + ('ru_nvcsw', c_long), + ('ru_nivcsw', c_long), +] +assert sizeof(rusage) == 72, sizeof(rusage) +assert alignment(rusage) == 4, alignment(rusage) +class rlimit(Structure): + pass +rlimit._pack_ = 4 +rlimit._fields_ = [ + ('rlim_cur', rlim_t), + ('rlim_max', rlim_t), +] +assert sizeof(rlimit) == 16, sizeof(rlimit) +assert alignment(rlimit) == 4, alignment(rlimit) +mcontext_t = __darwin_mcontext_t +mcontext64_t = __darwin_mcontext64_t +sigset_t = __darwin_sigset_t +ucontext_t = __darwin_ucontext_t +ucontext64_t = __darwin_ucontext64_t +class sigval(Union): + pass +sigval._fields_ = [ + ('sival_int', c_int), + ('sival_ptr', c_void_p), +] +assert sizeof(sigval) == 4, sizeof(sigval) +assert alignment(sigval) == 4, alignment(sigval) +class sigevent(Structure): + pass +pthread_attr_t = __darwin_pthread_attr_t +sigevent._fields_ = [ + ('sigev_notify', c_int), + ('sigev_signo', c_int), + ('sigev_value', sigval), + ('sigev_notify_function', CFUNCTYPE(None, sigval)), + ('sigev_notify_attributes', POINTER(pthread_attr_t)), +] +assert sizeof(sigevent) == 20, sizeof(sigevent) +assert alignment(sigevent) == 4, alignment(sigevent) +class __siginfo(Structure): + pass +pid_t = __darwin_pid_t +uid_t = __darwin_uid_t +__siginfo._fields_ = [ + ('si_signo', c_int), + ('si_errno', c_int), + ('si_code', c_int), + ('si_pid', pid_t), + ('si_uid', uid_t), + ('si_status', c_int), + ('si_addr', c_void_p), + ('si_value', sigval), + ('si_band', c_long), + ('pad', c_ulong * 7), +] +assert sizeof(__siginfo) == 64, sizeof(__siginfo) +assert alignment(__siginfo) == 4, alignment(__siginfo) +siginfo_t = __siginfo +class __sigaction_u(Union): + pass +__sigaction_u._fields_ = [ + ('__sa_handler', CFUNCTYPE(None, c_int)), + ('__sa_sigaction', CFUNCTYPE(None, c_int, POINTER(__siginfo), c_void_p)), +] +assert sizeof(__sigaction_u) == 4, sizeof(__sigaction_u) +assert alignment(__sigaction_u) == 4, alignment(__sigaction_u) +class __sigaction(Structure): + pass +__sigaction._fields_ = [ + ('__sigaction_u', __sigaction_u), + ('sa_tramp', CFUNCTYPE(None, c_void_p, c_int, c_int, POINTER(siginfo_t), c_void_p)), + ('sa_mask', sigset_t), + ('sa_flags', c_int), +] +assert sizeof(__sigaction) == 16, sizeof(__sigaction) +assert alignment(__sigaction) == 4, alignment(__sigaction) +class sigaction(Structure): + pass +sigaction._fields_ = [ + ('__sigaction_u', __sigaction_u), + ('sa_mask', sigset_t), + ('sa_flags', c_int), +] +assert sizeof(sigaction) == 12, sizeof(sigaction) +assert alignment(sigaction) == 4, alignment(sigaction) +sig_t = CFUNCTYPE(None, c_int) +stack_t = __darwin_stack_t +class sigvec(Structure): + pass +sigvec._fields_ = [ + ('sv_handler', CFUNCTYPE(None, c_int)), + ('sv_mask', c_int), + ('sv_flags', c_int), +] +assert sizeof(sigvec) == 12, sizeof(sigvec) +assert alignment(sigvec) == 4, alignment(sigvec) +class sigstack(Structure): + pass +sigstack._fields_ = [ + ('ss_sp', STRING), + ('ss_onstack', c_int), +] +assert sizeof(sigstack) == 8, sizeof(sigstack) +assert alignment(sigstack) == 4, alignment(sigstack) +class ostat(Structure): + pass +ino_t = __darwin_ino_t +mode_t = __darwin_mode_t +nlink_t = __uint16_t +class timespec(Structure): + pass +time_t = __darwin_time_t +timespec._fields_ = [ + ('tv_sec', time_t), + ('tv_nsec', c_long), +] +assert sizeof(timespec) == 8, sizeof(timespec) +assert alignment(timespec) == 4, alignment(timespec) +ostat._fields_ = [ + ('st_dev', __uint16_t), + ('st_ino', ino_t), + ('st_mode', mode_t), + ('st_nlink', nlink_t), + ('st_uid', __uint16_t), + ('st_gid', __uint16_t), + ('st_rdev', __uint16_t), + ('st_size', __int32_t), + ('st_atimespec', timespec), + ('st_mtimespec', timespec), + ('st_ctimespec', timespec), + ('st_blksize', __int32_t), + ('st_blocks', __int32_t), + ('st_flags', __uint32_t), + ('st_gen', __uint32_t), +] +assert sizeof(ostat) == 64, sizeof(ostat) +assert alignment(ostat) == 4, alignment(ostat) +class stat(Structure): + pass +dev_t = __darwin_dev_t +gid_t = __darwin_gid_t +off_t = __darwin_off_t +blkcnt_t = __darwin_blkcnt_t +blksize_t = __darwin_blksize_t +stat._pack_ = 4 +stat._fields_ = [ + ('st_dev', dev_t), + ('st_ino', ino_t), + ('st_mode', mode_t), + ('st_nlink', nlink_t), + ('st_uid', uid_t), + ('st_gid', gid_t), + ('st_rdev', dev_t), + ('st_atimespec', timespec), + ('st_mtimespec', timespec), + ('st_ctimespec', timespec), + ('st_size', off_t), + ('st_blocks', blkcnt_t), + ('st_blksize', blksize_t), + ('st_flags', __uint32_t), + ('st_gen', __uint32_t), + ('st_lspare', __int32_t), + ('st_qspare', __int64_t * 2), +] +assert sizeof(stat) == 96, sizeof(stat) +assert alignment(stat) == 4, alignment(stat) +class _filesec(Structure): + pass +filesec_t = POINTER(_filesec) +tcflag_t = c_ulong +cc_t = c_ubyte +speed_t = c_long +class termios(Structure): + pass +termios._fields_ = [ + ('c_iflag', tcflag_t), + ('c_oflag', tcflag_t), + ('c_cflag', tcflag_t), + ('c_lflag', tcflag_t), + ('c_cc', cc_t * 20), + ('c_ispeed', speed_t), + ('c_ospeed', speed_t), +] +assert sizeof(termios) == 44, sizeof(termios) +assert alignment(termios) == 4, alignment(termios) +class itimerval(Structure): + pass +itimerval._fields_ = [ + ('it_interval', timeval), + ('it_value', timeval), +] +assert sizeof(itimerval) == 16, sizeof(itimerval) +assert alignment(itimerval) == 4, alignment(itimerval) +class timezone(Structure): + pass +timezone._fields_ = [ + ('tz_minuteswest', c_int), + ('tz_dsttime', c_int), +] +assert sizeof(timezone) == 8, sizeof(timezone) +assert alignment(timezone) == 4, alignment(timezone) +class clockinfo(Structure): + pass +clockinfo._fields_ = [ + ('hz', c_int), + ('tick', c_int), + ('tickadj', c_int), + ('stathz', c_int), + ('profhz', c_int), +] +assert sizeof(clockinfo) == 20, sizeof(clockinfo) +assert alignment(clockinfo) == 4, alignment(clockinfo) +class winsize(Structure): + pass +winsize._fields_ = [ + ('ws_row', c_ushort), + ('ws_col', c_ushort), + ('ws_xpixel', c_ushort), + ('ws_ypixel', c_ushort), +] +assert sizeof(winsize) == 8, sizeof(winsize) +assert alignment(winsize) == 2, alignment(winsize) +u_char = c_ubyte +u_short = c_ushort +u_int = c_uint +u_long = c_ulong +ushort = c_ushort +uint = c_uint +u_quad_t = u_int64_t +quad_t = int64_t +qaddr_t = POINTER(quad_t) +caddr_t = STRING +daddr_t = int32_t +fixpt_t = u_int32_t +in_addr_t = __uint32_t +in_port_t = __uint16_t +key_t = __int32_t +id_t = __darwin_id_t +segsz_t = int32_t +swblk_t = int32_t +clock_t = __darwin_clock_t +ssize_t = __darwin_ssize_t +useconds_t = __darwin_useconds_t +suseconds_t = __darwin_suseconds_t +fd_mask = __int32_t +class fd_set(Structure): + pass +fd_set._fields_ = [ + ('fds_bits', __int32_t * 32), +] +assert sizeof(fd_set) == 128, sizeof(fd_set) +assert alignment(fd_set) == 4, alignment(fd_set) +pthread_cond_t = __darwin_pthread_cond_t +pthread_condattr_t = __darwin_pthread_condattr_t +pthread_mutex_t = __darwin_pthread_mutex_t +pthread_mutexattr_t = __darwin_pthread_mutexattr_t +pthread_once_t = __darwin_pthread_once_t +pthread_rwlock_t = __darwin_pthread_rwlock_t +pthread_rwlockattr_t = __darwin_pthread_rwlockattr_t +pthread_t = __darwin_pthread_t +pthread_key_t = __darwin_pthread_key_t +fsblkcnt_t = __darwin_fsblkcnt_t +fsfilcnt_t = __darwin_fsfilcnt_t + +# values for enumeration 'idtype_t' +idtype_t = c_int # enum +class wait(Union): + pass +class N4wait3DOLLAR_3E(Structure): + pass +N4wait3DOLLAR_3E._fields_ = [ + ('w_Termsig', c_uint, 7), + ('w_Coredump', c_uint, 1), + ('w_Retcode', c_uint, 8), + ('w_Filler', c_uint, 16), +] +assert sizeof(N4wait3DOLLAR_3E) == 4, sizeof(N4wait3DOLLAR_3E) +assert alignment(N4wait3DOLLAR_3E) == 4, alignment(N4wait3DOLLAR_3E) +class N4wait3DOLLAR_4E(Structure): + pass +N4wait3DOLLAR_4E._fields_ = [ + ('w_Stopval', c_uint, 8), + ('w_Stopsig', c_uint, 8), + ('w_Filler', c_uint, 16), +] +assert sizeof(N4wait3DOLLAR_4E) == 4, sizeof(N4wait3DOLLAR_4E) +assert alignment(N4wait3DOLLAR_4E) == 4, alignment(N4wait3DOLLAR_4E) +wait._fields_ = [ + ('w_status', c_int), + ('w_T', N4wait3DOLLAR_3E), + ('w_S', N4wait3DOLLAR_4E), +] +assert sizeof(wait) == 4, sizeof(wait) +assert alignment(wait) == 4, alignment(wait) +class tm(Structure): + pass +tm._fields_ = [ + ('tm_sec', c_int), + ('tm_min', c_int), + ('tm_hour', c_int), + ('tm_mday', c_int), + ('tm_mon', c_int), + ('tm_year', c_int), + ('tm_wday', c_int), + ('tm_yday', c_int), + ('tm_isdst', c_int), + ('tm_gmtoff', c_long), + ('tm_zone', STRING), +] +assert sizeof(tm) == 44, sizeof(tm) +assert alignment(tm) == 4, alignment(tm) +uint8_t = c_ubyte +uint16_t = c_ushort +uint32_t = c_uint +uint64_t = c_ulonglong +int_least8_t = int8_t +int_least16_t = int16_t +int_least32_t = int32_t +int_least64_t = int64_t +uint_least8_t = uint8_t +uint_least16_t = uint16_t +uint_least32_t = uint32_t +uint_least64_t = uint64_t +int_fast8_t = int8_t +int_fast16_t = int16_t +int_fast32_t = int32_t +int_fast64_t = int64_t +uint_fast8_t = uint8_t +uint_fast16_t = uint16_t +uint_fast32_t = uint32_t +uint_fast64_t = uint64_t +intmax_t = c_longlong +uintmax_t = c_ulonglong +class PyFileObject(Structure): + pass +Py_ssize_t = ssize_t +class _typeobject(Structure): + pass +class _object(Structure): + pass +PyObject = _object +PyFileObject._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', POINTER(_typeobject)), + ('f_fp', POINTER(FILE)), + ('f_name', POINTER(PyObject)), + ('f_mode', POINTER(PyObject)), + ('f_close', CFUNCTYPE(c_int, POINTER(FILE))), + ('f_softspace', c_int), + ('f_binary', c_int), + ('f_buf', STRING), + ('f_bufend', STRING), + ('f_bufptr', STRING), + ('f_setbuf', STRING), + ('f_univ_newline', c_int), + ('f_newlinetypes', c_int), + ('f_skipnextlf', c_int), + ('f_encoding', POINTER(PyObject)), + ('weakreflist', POINTER(PyObject)), +] +assert sizeof(PyFileObject) == 68, sizeof(PyFileObject) +assert alignment(PyFileObject) == 4, alignment(PyFileObject) +_object._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', POINTER(_typeobject)), +] +assert sizeof(_object) == 8, sizeof(_object) +assert alignment(_object) == 4, alignment(_object) +class PyVarObject(Structure): + pass +PyVarObject._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', POINTER(_typeobject)), + ('ob_size', Py_ssize_t), +] +assert sizeof(PyVarObject) == 12, sizeof(PyVarObject) +assert alignment(PyVarObject) == 4, alignment(PyVarObject) +unaryfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject)) +binaryfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +ternaryfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +inquiry = CFUNCTYPE(c_int, POINTER(PyObject)) +lenfunc = CFUNCTYPE(Py_ssize_t, POINTER(PyObject)) +coercion = CFUNCTYPE(c_int, POINTER(POINTER(PyObject)), POINTER(POINTER(PyObject))) +intargfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), c_int) +intintargfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), c_int, c_int) +ssizeargfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), c_long) +ssizessizeargfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), c_long, c_long) +intobjargproc = CFUNCTYPE(c_int, POINTER(PyObject), c_int, POINTER(PyObject)) +intintobjargproc = CFUNCTYPE(c_int, POINTER(PyObject), c_int, c_int, POINTER(PyObject)) +ssizeobjargproc = CFUNCTYPE(c_int, POINTER(PyObject), c_long, POINTER(PyObject)) +ssizessizeobjargproc = CFUNCTYPE(c_int, POINTER(PyObject), c_long, c_long, POINTER(PyObject)) +objobjargproc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +getreadbufferproc = CFUNCTYPE(c_int, POINTER(PyObject), c_int, POINTER(c_void_p)) +getwritebufferproc = CFUNCTYPE(c_int, POINTER(PyObject), c_int, POINTER(c_void_p)) +getsegcountproc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(c_int)) +getcharbufferproc = CFUNCTYPE(c_int, POINTER(PyObject), c_int, POINTER(STRING)) +readbufferproc = CFUNCTYPE(Py_ssize_t, POINTER(PyObject), c_long, POINTER(c_void_p)) +writebufferproc = CFUNCTYPE(Py_ssize_t, POINTER(PyObject), c_long, POINTER(c_void_p)) +segcountproc = CFUNCTYPE(Py_ssize_t, POINTER(PyObject), POINTER(Py_ssize_t)) +charbufferproc = CFUNCTYPE(Py_ssize_t, POINTER(PyObject), c_long, POINTER(STRING)) +objobjproc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject)) +visitproc = CFUNCTYPE(c_int, POINTER(PyObject), c_void_p) +traverseproc = CFUNCTYPE(c_int, POINTER(PyObject), CFUNCTYPE(c_int, POINTER(PyObject), c_void_p), c_void_p) +class PyNumberMethods(Structure): + pass +PyNumberMethods._fields_ = [ + ('nb_add', binaryfunc), + ('nb_subtract', binaryfunc), + ('nb_multiply', binaryfunc), + ('nb_divide', binaryfunc), + ('nb_remainder', binaryfunc), + ('nb_divmod', binaryfunc), + ('nb_power', ternaryfunc), + ('nb_negative', unaryfunc), + ('nb_positive', unaryfunc), + ('nb_absolute', unaryfunc), + ('nb_nonzero', inquiry), + ('nb_invert', unaryfunc), + ('nb_lshift', binaryfunc), + ('nb_rshift', binaryfunc), + ('nb_and', binaryfunc), + ('nb_xor', binaryfunc), + ('nb_or', binaryfunc), + ('nb_coerce', coercion), + ('nb_int', unaryfunc), + ('nb_long', unaryfunc), + ('nb_float', unaryfunc), + ('nb_oct', unaryfunc), + ('nb_hex', unaryfunc), + ('nb_inplace_add', binaryfunc), + ('nb_inplace_subtract', binaryfunc), + ('nb_inplace_multiply', binaryfunc), + ('nb_inplace_divide', binaryfunc), + ('nb_inplace_remainder', binaryfunc), + ('nb_inplace_power', ternaryfunc), + ('nb_inplace_lshift', binaryfunc), + ('nb_inplace_rshift', binaryfunc), + ('nb_inplace_and', binaryfunc), + ('nb_inplace_xor', binaryfunc), + ('nb_inplace_or', binaryfunc), + ('nb_floor_divide', binaryfunc), + ('nb_true_divide', binaryfunc), + ('nb_inplace_floor_divide', binaryfunc), + ('nb_inplace_true_divide', binaryfunc), + ('nb_index', lenfunc), +] +assert sizeof(PyNumberMethods) == 156, sizeof(PyNumberMethods) +assert alignment(PyNumberMethods) == 4, alignment(PyNumberMethods) +class PySequenceMethods(Structure): + pass +PySequenceMethods._fields_ = [ + ('sq_length', lenfunc), + ('sq_concat', binaryfunc), + ('sq_repeat', ssizeargfunc), + ('sq_item', ssizeargfunc), + ('sq_slice', ssizessizeargfunc), + ('sq_ass_item', ssizeobjargproc), + ('sq_ass_slice', ssizessizeobjargproc), + ('sq_contains', objobjproc), + ('sq_inplace_concat', binaryfunc), + ('sq_inplace_repeat', ssizeargfunc), +] +assert sizeof(PySequenceMethods) == 40, sizeof(PySequenceMethods) +assert alignment(PySequenceMethods) == 4, alignment(PySequenceMethods) +class PyMappingMethods(Structure): + pass +PyMappingMethods._fields_ = [ + ('mp_length', lenfunc), + ('mp_subscript', binaryfunc), + ('mp_ass_subscript', objobjargproc), +] +assert sizeof(PyMappingMethods) == 12, sizeof(PyMappingMethods) +assert alignment(PyMappingMethods) == 4, alignment(PyMappingMethods) +class PyBufferProcs(Structure): + pass +PyBufferProcs._fields_ = [ + ('bf_getreadbuffer', readbufferproc), + ('bf_getwritebuffer', writebufferproc), + ('bf_getsegcount', segcountproc), + ('bf_getcharbuffer', charbufferproc), +] +assert sizeof(PyBufferProcs) == 16, sizeof(PyBufferProcs) +assert alignment(PyBufferProcs) == 4, alignment(PyBufferProcs) +freefunc = CFUNCTYPE(None, c_void_p) +destructor = CFUNCTYPE(None, POINTER(PyObject)) +printfunc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(FILE), c_int) +getattrfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), STRING) +getattrofunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +setattrfunc = CFUNCTYPE(c_int, POINTER(PyObject), STRING, POINTER(PyObject)) +setattrofunc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +cmpfunc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject)) +reprfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject)) +hashfunc = CFUNCTYPE(c_long, POINTER(PyObject)) +richcmpfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), POINTER(PyObject), c_int) +getiterfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject)) +iternextfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject)) +descrgetfunc = CFUNCTYPE(POINTER(PyObject), POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +descrsetfunc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +initproc = CFUNCTYPE(c_int, POINTER(PyObject), POINTER(PyObject), POINTER(PyObject)) +newfunc = CFUNCTYPE(POINTER(PyObject), POINTER(_typeobject), POINTER(PyObject), POINTER(PyObject)) +allocfunc = CFUNCTYPE(POINTER(PyObject), POINTER(_typeobject), c_long) +class PyMethodDef(Structure): + pass +class PyMemberDef(Structure): + pass +class PyGetSetDef(Structure): + pass +_typeobject._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', POINTER(_typeobject)), + ('ob_size', Py_ssize_t), + ('tp_name', STRING), + ('tp_basicsize', Py_ssize_t), + ('tp_itemsize', Py_ssize_t), + ('tp_dealloc', destructor), + ('tp_print', printfunc), + ('tp_getattr', getattrfunc), + ('tp_setattr', setattrfunc), + ('tp_compare', cmpfunc), + ('tp_repr', reprfunc), + ('tp_as_number', POINTER(PyNumberMethods)), + ('tp_as_sequence', POINTER(PySequenceMethods)), + ('tp_as_mapping', POINTER(PyMappingMethods)), + ('tp_hash', hashfunc), + ('tp_call', ternaryfunc), + ('tp_str', reprfunc), + ('tp_getattro', getattrofunc), + ('tp_setattro', setattrofunc), + ('tp_as_buffer', POINTER(PyBufferProcs)), + ('tp_flags', c_long), + ('tp_doc', STRING), + ('tp_traverse', traverseproc), + ('tp_clear', inquiry), + ('tp_richcompare', richcmpfunc), + ('tp_weaklistoffset', Py_ssize_t), + ('tp_iter', getiterfunc), + ('tp_iternext', iternextfunc), + ('tp_methods', POINTER(PyMethodDef)), + ('tp_members', POINTER(PyMemberDef)), + ('tp_getset', POINTER(PyGetSetDef)), + ('tp_base', POINTER(_typeobject)), + ('tp_dict', POINTER(PyObject)), + ('tp_descr_get', descrgetfunc), + ('tp_descr_set', descrsetfunc), + ('tp_dictoffset', Py_ssize_t), + ('tp_init', initproc), + ('tp_alloc', allocfunc), + ('tp_new', newfunc), + ('tp_free', freefunc), + ('tp_is_gc', inquiry), + ('tp_bases', POINTER(PyObject)), + ('tp_mro', POINTER(PyObject)), + ('tp_cache', POINTER(PyObject)), + ('tp_subclasses', POINTER(PyObject)), + ('tp_weaklist', POINTER(PyObject)), + ('tp_del', destructor), +] +assert sizeof(_typeobject) == 192, sizeof(_typeobject) +assert alignment(_typeobject) == 4, alignment(_typeobject) +PyTypeObject = _typeobject +class _heaptypeobject(Structure): + pass +_heaptypeobject._fields_ = [ + ('ht_type', PyTypeObject), + ('as_number', PyNumberMethods), + ('as_mapping', PyMappingMethods), + ('as_sequence', PySequenceMethods), + ('as_buffer', PyBufferProcs), + ('ht_name', POINTER(PyObject)), + ('ht_slots', POINTER(PyObject)), +] +assert sizeof(_heaptypeobject) == 424, sizeof(_heaptypeobject) +assert alignment(_heaptypeobject) == 4, alignment(_heaptypeobject) +PyHeapTypeObject = _heaptypeobject +Py_uintptr_t = c_uint +Py_intptr_t = c_int +__all__ = ['__uint16_t', 'PyMethodDef', 'objobjargproc', '__int16_t', + 'unaryfunc', '__darwin_pthread_condattr_t', + 'readbufferproc', '__darwin_id_t', 'key_t', + '__darwin_time_t', 'ucontext64_t', '__darwin_nl_item', + 'pthread_once_t', '_opaque_pthread_condattr_t', 'FILE', + 'pthread_mutexattr_t', 'size_t', 'Py_uintptr_t', + '_FP_ZERO', '_FP_SUPERNORMAL', 'cmpfunc', '__uint32_t', + 'mcontext_t', 'PySequenceMethods', 'uint8_t', 'fpos_t', + 'P_PGID', 'qaddr_t', '__darwin_gid_t', 'blkcnt_t', + 'uint_least16_t', '__darwin_dev_t', 'time_t', '_FP_NORMAL', + 'allocfunc', 'getiterfunc', 'int32_t', + '__darwin_fsfilcnt_t', 'lenfunc', 'intptr_t', + 'uint_least64_t', 'blksize_t', 'user_addr_t', 'PyObject', + 'int_least32_t', 'sigaltstack', 'Py_intptr_t', 'ostat', + '__darwin_pthread_t', 'u_char', 'fixpt_t', 'uid_t', + 'u_int64_t', 'u_int16_t', 'register_t', + '__darwin_ucontext_t', 'cc_t', 'in_port_t', 'ternaryfunc', + '__darwin_ssize_t', 'descrsetfunc', '__darwin_mcontext_t', + '__darwin_sigset_t', 'ct_rune_t', 'uint16_t', + '__darwin_ptrdiff_t', 'float_t', 'int_fast32_t', 'va_list', + 'uint_fast16_t', 'sigset_t', '__int32_t', 'fd_mask', + 'binaryfunc', 'fsfilcnt_t', 'ucontext', 'ssizeobjargproc', + 'uint_fast32_t', 'freefunc', 'tm', '__uint64_t', 'mode_t', + 'timespec', '__darwin_suseconds_t', 'PyNumberMethods', + '__sigaction', 'sigevent', 'user_ulong_t', 'user_ssize_t', + 'syscall_arg_t', 'reprfunc', 'int16_t', 'getattrofunc', + 'clock_t', 'ssizeargfunc', 'ssizessizeargfunc', + '__darwin_socklen_t', '__darwin_intptr_t', 'rune_t', + '__darwin_va_list', 'caddr_t', 'siginfo_t', 'ucontext_t', + '__sbuf', 'setattrfunc', 'coercion', 'int_least8_t', + 'getsegcountproc', 'N4wait3DOLLAR_4E', 'div_t', 'newfunc', + 'intintobjargproc', 'id_t', '__darwin_blksize_t', 'ldiv_t', + 'int_least16_t', '__darwin_wctrans_t', 'uint_least8_t', + 'u_int32_t', 'pthread_rwlock_t', 'charbufferproc', + '__darwin_wchar_t', 'sigval', 'PyGetSetDef', + 'intobjargproc', 'P_PID', 'sigaction', + '__darwin_natural_t', 'sig_t', '__darwin_blkcnt_t', + 'u_int', '_opaque_pthread_cond_t', '__darwin_size_t', + 'ssizessizeobjargproc', 'segsz_t', 'ushort', + '__darwin_ct_rune_t', 'pthread_t', '__darwin_ino_t', + 'pthread_attr_t', 'fd_set', '__darwin_useconds_t', + '__darwin_mcontext64_t', 'ino_t', '__darwin_clock_t', + 'uint_fast8_t', '_typeobject', '__darwin_pthread_key_t', + 'getwritebufferproc', 'traverseproc', + '__darwin_pthread_handler_rec', 'double_t', + '__darwin_pthread_mutex_t', 'initproc', 'speed_t', + '_object', '__darwin_ucontext64_t', 'getreadbufferproc', + 'rlim_t', 'hashfunc', '__darwin_fsblkcnt_t', + '__darwin_rune_t', 'fsblkcnt_t', 'u_quad_t', + '_opaque_pthread_rwlockattr_t', 'sigvec', + '_opaque_pthread_mutexattr_t', 'clockinfo', + '__darwin_pthread_rwlock_t', 'pthread_condattr_t', + 'destructor', 'rlimit', '__darwin_pthread_mutexattr_t', + 'daddr_t', '__darwin_pthread_once_t', 'stack_t', + '__darwin_mode_t', 'uint_least32_t', 'wait', 'OSBigEndian', + '__mbstate_t', 'uintptr_t', '__darwin_mach_port_t', + '__uint8_t', '__darwin_uid_t', 'itimerval', '__int8_t', + 'PyMemberDef', '_opaque_pthread_mutex_t', 'int8_t', + '__darwin_uuid_t', '_opaque_pthread_attr_t', 'uintmax_t', + 'intargfunc', 'off_t', 'gid_t', 'sigstack', 'filesec_t', + 'mcontext', 'int_least64_t', '_FP_NAN', 'pid_t', + 'visitproc', 'N4wait3DOLLAR_3E', 'quad_t', 'uint_fast64_t', + 'u_long', 'intmax_t', 'sigcontext', 'swblk_t', '__siginfo', + 'winsize', '__darwin_mbstate_t', 'useconds_t', + 'richcmpfunc', 'pthread_key_t', 'uint64_t', 'u_int8_t', + 'writebufferproc', 'pthread_cond_t', '_FP_SUBNORMAL', + '__darwin_wctype_t', '_opaque_pthread_once_t', + 'OSLittleEndian', 'int_fast16_t', 'int64_t', + '_FP_INFINITE', 'timezone', '_heaptypeobject', '__sFILE', + 'ucontext64', 'sig_atomic_t', 'u_short', 'nlink_t', + 'PyVarObject', 'mcontext64', 'lldiv_t', + '__darwin_pthread_rwlockattr_t', 'descrgetfunc', + 'OSUnknownByteOrder', 'timeval', '__darwin_stack_t', + 'inquiry', 'printfunc', 'int_fast64_t', 'int_fast8_t', + 'PyTypeObject', '__sFILEX', 'stat', 'getcharbufferproc', + 'uint32_t', 'intintargfunc', 'PyBufferProcs', + '__darwin_pthread_attr_t', '__darwin_mach_port_name_t', + 'user_time_t', 'PyFileObject', 'uint', 'termios', + '__darwin_wint_t', '__darwin_pthread_cond_t', 'objobjproc', + 'user_size_t', 'pthread_rwlockattr_t', 'rusage', + 'PyMappingMethods', 'setattrofunc', 'idtype_t', + 'segcountproc', '__darwin_pid_t', '__sigaction_u', + '_opaque_pthread_rwlock_t', 'in_addr_t', '__darwin_off_t', + '_opaque_pthread_t', 'tcflag_t', 'iternextfunc', 'P_ALL', + 'pthread_mutex_t', '__int64_t', 'getattrfunc', 'ssize_t', + 'mcontext64_t', 'user_long_t', 'dev_t', '_filesec', + 'suseconds_t', 'PyHeapTypeObject', 'Py_ssize_t'] Added: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,187 @@ +from pypy.rpython.rctypes.tool import ctypes_platform +import pypy.rpython.rctypes.implementation # this defines rctypes magic +from pypy.rpython.rctypes.aerrno import geterrno +from pypy.interpreter.error import OperationError +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from ctypes import * +import ctypes.util + +from bzlib import bz_stream, BZFILE, FILE +from fileobject import PyFileObject + +libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) + +c_void = None + +class CConfig: + _header_ = """ + #include + #include + """ + off_t = ctypes_platform.SimpleType("off_t", c_longlong) + size_t = ctypes_platform.SimpleType("size_t", c_ulong) + BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") + +constants = {} +constant_names = ['BUFSIZ', 'BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', + 'BZ_RUN_OK', 'BZ_FLUSH_OK', 'BZ_FINISH_OK', 'BZ_STREAM_END', + 'BZ_SEQUENCE_ERROR', 'BZ_PARAM_ERROR', 'BZ_MEM_ERROR', 'BZ_DATA_ERROR', + 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', + 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] +for name in constant_names: + setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + +class cConfig: + pass +cConfig.__dict__.update(ctypes_platform.configure(CConfig)) + +for name in constant_names: + value = getattr(cConfig, name) + if value is not None: + constants[name] = value +locals().update(constants) + +off_t = cConfig.off_t +BZ_OK = cConfig.BZ_OK +BZ_STREAM_END = cConfig.BZ_STREAM_END +BZ_CONFIG_ERROR = cConfig.BZ_CONFIG_ERROR +BZ_PARAM_ERROR = cConfig.BZ_PARAM_ERROR +BZ_DATA_ERROR = cConfig.BZ_DATA_ERROR +BZ_DATA_ERROR_MAGIC = cConfig.BZ_DATA_ERROR_MAGIC +BZ_IO_ERROR = cConfig.BZ_IO_ERROR +BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR +BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF +BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR + +# modes +MODE_CLOSED = 0 +MODE_READ = 1 +MODE_READ_EOF = 2 +MODE_WRITE = 3 + +# bits in f_newlinetypes +NEWLINE_UNKNOWN = 0 # No newline seen, yet +NEWLINE_CR = 1 # \r newline seen +NEWLINE_LF = 2 # \n newline seen +NEWLINE_CRLF = 4 # \r\n newline seen + +pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] +pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) +pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] +pythonapi.PyFile_SetBufSize.restype = c_void +pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] +pythonapi.PyFile_AsFile.restype = POINTER(FILE) +libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_void_p, c_int] +libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_int] +libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) + +def _catch_bz2_error(space, bzerror): + if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library was not compiled correctly")) + if bzerror == BZ_PARAM_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library has received wrong parameters")) + elif bzerror == BZ_MEM_ERROR: + raise OperationError(space.w_MemoryError, space.wrap("")) + elif bzerror in (BZ_DATA_ERROR, BZ_DATA_ERROR_MAGIC): + raise OperationError(space.w_IOError, space.wrap("invalid data stream")) + elif bzerror == BZ_IO_ERROR: + raise OperationError(space.w_IOError, space.wrap("unknown IO error")) + elif bzerror == BZ_UNEXPECTED_EOF: + raise OperationError(space.w_EOFError, + space.wrap( + "compressed file ended before the logical end-of-stream was detected")) + elif bzerror == BZ_SEQUENCE_ERROR: + raise OperationError(space.w_RuntimeError, + space.wrap("wrong sequence of bz2 library commands used")) + + +class _BZ2File(Wrappable): + def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): + self.space = space + + self.f_buf = c_char_p() # allocated readahead buffer + self.f_bufend = c_char_p() # points after last occupied position + self.f_bufptr = c_char_p() # current buffer position + + self.f_softspace = 0 # flag used by print command + + self.f_univ_newline = False # handle any newline convention + self.f_newlinetypes = 0 # types of newlines seen + self.f_skipnextlf = 0 # skip next \n + + self.mode = 0 + self.pos = 0 + self.size = 0 + + self._init_bz2file(filename, mode, buffering, compresslevel) + + def _init_bz2file(self, filename, mode_, buffering, compresslevel): + self.size = -1 + + name = filename + mode_char = "" + mode_list = mode_ + + if compresslevel < 1 or compresslevel > 9: + raise OperationError(self.space.w_ValueError, + self.space.wrap("compresslevel must be between 1 and 9")) + + i = 1 + for mode in mode_list: + error = False + + if mode in ['r', 'w']: + if mode_char: + error = True + mode_char = mode + elif mode == 'b': + pass + elif mode == 'U': + self.f_univ_newline = True + else: + error = True + + if error: + raise OperationError(self.space.w_ValueError, + self.space.wrap("invalid mode char %s" % mode)) + + if mode_char == 0: + mode_char = 'r' + mode = ('wb', 'rb')[mode_char == 'r'] + + # open the file and set the buffer + f = pythonapi.PyFile_FromString(name, mode) + if not f: + raise OperationError(self.space.w_IOError, + self.space.wrap("cannot open file %s" % name)) + pythonapi.PyFile_SetBufSize(f, buffering) + + # store the FILE object + self._file = pythonapi.PyFile_AsFile(f) + + bzerror = c_int() + if mode_char == 'r': + self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, + 0, 0, None, 0) + else: + self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, + compresslevel, 0, 0) + + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] + +_BZ2File.typedef = TypeDef("_BZ2File") + +def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): + return _BZ2File(space, filename, mode, buffering, compresslevel) +BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] + Added: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 16:10:40 2006 @@ -0,0 +1,386 @@ +from py.test import raises, skip +from pypy.conftest import gettestobjspace +import os + +def teardown_module(mod): + if os.path.exists("foo"): + os.unlink("foo") + +class AppTestBz2: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + + def test_creation(self): + from bz2 import BZ2File + + raises(ValueError, BZ2File, "foo", mode='w', compresslevel=10) + raises(ValueError, BZ2File, "foo", mode='XYZ') + raises(ValueError, BZ2File, "foo", mode='ww') + + BZ2File("foo", mode='wU', buffering=0, compresslevel=8) + BZ2File("foo", mode='wb') + # a large buf size + BZ2File("foo", mode='w', buffering=4096) + + +# #!/usr/bin/python +# from test import test_support +# from test.test_support import TESTFN +# +# import unittest +# from cStringIO import StringIO +# import os +# import popen2 +# import sys +# +# import bz2 +# from bz2 import BZ2File, BZ2Compressor, BZ2Decompressor +# +# has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") +# +# class BaseTest(unittest.TestCase): +# "Base for other testcases." +# TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' +# DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' +# DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' +# +# if has_cmdline_bunzip2: +# def decompress(self, data): +# pop = popen2.Popen3("bunzip2", capturestderr=1) +# pop.tochild.write(data) +# pop.tochild.close() +# ret = pop.fromchild.read() +# pop.fromchild.close() +# if pop.wait() != 0: +# ret = bz2.decompress(data) +# return ret +# +# else: +# # popen2.Popen3 doesn't exist on Windows, and even if it did, bunzip2 +# # isn't available to run. +# def decompress(self, data): +# return bz2.decompress(data) +# +# class BZ2FileTest(BaseTest): +# "Test BZ2File type miscellaneous methods." +# +# def setUp(self): +# self.filename = TESTFN +# +# def tearDown(self): +# if os.path.isfile(self.filename): +# os.unlink(self.filename) +# +# def createTempFile(self, crlf=0): +# f = open(self.filename, "wb") +# if crlf: +# data = self.DATA_CRLF +# else: +# data = self.DATA +# f.write(data) +# f.close() +# +# def testRead(self): +# # "Test BZ2File.read()" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# self.assertRaises(TypeError, bz2f.read, None) +# self.assertEqual(bz2f.read(), self.TEXT) +# bz2f.close() +# +# def testReadChunk10(self): +# # "Test BZ2File.read() in chunks of 10 bytes" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# text = '' +# while 1: +# str = bz2f.read(10) +# if not str: +# break +# text += str +# self.assertEqual(text, text) +# bz2f.close() +# +# def testRead100(self): +# # "Test BZ2File.read(100)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# self.assertEqual(bz2f.read(100), self.TEXT[:100]) +# bz2f.close() +# +# def testReadLine(self): +# # "Test BZ2File.readline()" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# self.assertRaises(TypeError, bz2f.readline, None) +# sio = StringIO(self.TEXT) +# for line in sio.readlines(): +# self.assertEqual(bz2f.readline(), line) +# bz2f.close() +# +# def testReadLines(self): +# # "Test BZ2File.readlines()" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# self.assertRaises(TypeError, bz2f.readlines, None) +# sio = StringIO(self.TEXT) +# self.assertEqual(bz2f.readlines(), sio.readlines()) +# bz2f.close() +# +# def testIterator(self): +# # "Test iter(BZ2File)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# sio = StringIO(self.TEXT) +# self.assertEqual(list(iter(bz2f)), sio.readlines()) +# bz2f.close() +# +# def testXReadLines(self): +# # "Test BZ2File.xreadlines()" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# sio = StringIO(self.TEXT) +# self.assertEqual(list(bz2f.xreadlines()), sio.readlines()) +# bz2f.close() +# +# def testUniversalNewlinesLF(self): +# # "Test BZ2File.read() with universal newlines (\\n)" +# self.createTempFile() +# bz2f = BZ2File(self.filename, "rU") +# self.assertEqual(bz2f.read(), self.TEXT) +# self.assertEqual(bz2f.newlines, "\n") +# bz2f.close() +# +# def testUniversalNewlinesCRLF(self): +# # "Test BZ2File.read() with universal newlines (\\r\\n)" +# self.createTempFile(crlf=1) +# bz2f = BZ2File(self.filename, "rU") +# self.assertEqual(bz2f.read(), self.TEXT) +# self.assertEqual(bz2f.newlines, "\r\n") +# bz2f.close() +# +# def testWrite(self): +# # "Test BZ2File.write()" +# bz2f = BZ2File(self.filename, "w") +# self.assertRaises(TypeError, bz2f.write) +# bz2f.write(self.TEXT) +# bz2f.close() +# f = open(self.filename, 'rb') +# self.assertEqual(self.decompress(f.read()), self.TEXT) +# f.close() +# +# def testWriteChunks10(self): +# # "Test BZ2File.write() with chunks of 10 bytes" +# bz2f = BZ2File(self.filename, "w") +# n = 0 +# while 1: +# str = self.TEXT[n*10:(n+1)*10] +# if not str: +# break +# bz2f.write(str) +# n += 1 +# bz2f.close() +# f = open(self.filename, 'rb') +# self.assertEqual(self.decompress(f.read()), self.TEXT) +# f.close() +# +# def testWriteLines(self): +# # "Test BZ2File.writelines()" +# bz2f = BZ2File(self.filename, "w") +# self.assertRaises(TypeError, bz2f.writelines) +# sio = StringIO(self.TEXT) +# bz2f.writelines(sio.readlines()) +# bz2f.close() +# f = open(self.filename, 'rb') +# self.assertEqual(self.decompress(f.read()), self.TEXT) +# f.close() +# +# def testSeekForward(self): +# # "Test BZ2File.seek(150, 0)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# self.assertRaises(TypeError, bz2f.seek) +# bz2f.seek(150) +# self.assertEqual(bz2f.read(), self.TEXT[150:]) +# bz2f.close() +# +# def testSeekBackwards(self): +# # "Test BZ2File.seek(-150, 1)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.read(500) +# bz2f.seek(-150, 1) +# self.assertEqual(bz2f.read(), self.TEXT[500-150:]) +# bz2f.close() +# +# def testSeekBackwardsFromEnd(self): +# # "Test BZ2File.seek(-150, 2)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(-150, 2) +# self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) +# bz2f.close() +# +# def testSeekPostEnd(self): +# # "Test BZ2File.seek(150000)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(150000) +# self.assertEqual(bz2f.tell(), len(self.TEXT)) +# self.assertEqual(bz2f.read(), "") +# bz2f.close() +# +# def testSeekPostEndTwice(self): +# # "Test BZ2File.seek(150000) twice" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(150000) +# bz2f.seek(150000) +# self.assertEqual(bz2f.tell(), len(self.TEXT)) +# self.assertEqual(bz2f.read(), "") +# bz2f.close() +# +# def testSeekPreStart(self): +# # "Test BZ2File.seek(-150, 0)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(-150) +# self.assertEqual(bz2f.tell(), 0) +# self.assertEqual(bz2f.read(), self.TEXT) +# bz2f.close() +# +# def testOpenDel(self): +# # "Test opening and deleting a file many times" +# self.createTempFile() +# for i in xrange(10000): +# o = BZ2File(self.filename) +# del o +# +# def testOpenNonexistent(self): +# # "Test opening a nonexistent file" +# self.assertRaises(IOError, BZ2File, "/non/existent") +# +# def testModeU(self): +# # Bug #1194181: bz2.BZ2File opened for write with mode "U" +# self.createTempFile() +# bz2f = BZ2File(self.filename, "U") +# bz2f.close() +# f = file(self.filename) +# f.seek(0, 2) +# self.assertEqual(f.tell(), len(self.DATA)) +# f.close() +# +# def testBug1191043(self): +# # readlines() for files containing no newline +# data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' +# f = open(self.filename, "wb") +# f.write(data) +# f.close() +# bz2f = BZ2File(self.filename) +# lines = bz2f.readlines() +# bz2f.close() +# self.assertEqual(lines, ['Test']) +# bz2f = BZ2File(self.filename) +# xlines = list(bz2f.xreadlines()) +# bz2f.close() +# self.assertEqual(lines, ['Test']) +# +# +# class BZ2CompressorTest(BaseTest): +# def testCompress(self): +# # "Test BZ2Compressor.compress()/flush()" +# bz2c = BZ2Compressor() +# self.assertRaises(TypeError, bz2c.compress) +# data = bz2c.compress(self.TEXT) +# data += bz2c.flush() +# self.assertEqual(self.decompress(data), self.TEXT) +# +# def testCompressChunks10(self): +# # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" +# bz2c = BZ2Compressor() +# n = 0 +# data = '' +# while 1: +# str = self.TEXT[n*10:(n+1)*10] +# if not str: +# break +# data += bz2c.compress(str) +# n += 1 +# data += bz2c.flush() +# self.assertEqual(self.decompress(data), self.TEXT) +# +# class BZ2DecompressorTest(BaseTest): +# def test_Constructor(self): +# self.assertRaises(TypeError, BZ2Decompressor, 42) +# +# def testDecompress(self): +# # "Test BZ2Decompressor.decompress()" +# bz2d = BZ2Decompressor() +# self.assertRaises(TypeError, bz2d.decompress) +# text = bz2d.decompress(self.DATA) +# self.assertEqual(text, self.TEXT) +# +# def testDecompressChunks10(self): +# # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" +# bz2d = BZ2Decompressor() +# text = '' +# n = 0 +# while 1: +# str = self.DATA[n*10:(n+1)*10] +# if not str: +# break +# text += bz2d.decompress(str) +# n += 1 +# self.assertEqual(text, self.TEXT) +# +# def testDecompressUnusedData(self): +# # "Test BZ2Decompressor.decompress() with unused data" +# bz2d = BZ2Decompressor() +# unused_data = "this is unused data" +# text = bz2d.decompress(self.DATA+unused_data) +# self.assertEqual(text, self.TEXT) +# self.assertEqual(bz2d.unused_data, unused_data) +# +# def testEOFError(self): +# # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError" +# bz2d = BZ2Decompressor() +# text = bz2d.decompress(self.DATA) +# self.assertRaises(EOFError, bz2d.decompress, "anything") +# +# +# class FuncTest(BaseTest): +# "Test module functions" +# +# def testCompress(self): +# # "Test compress() function" +# data = bz2.compress(self.TEXT) +# self.assertEqual(self.decompress(data), self.TEXT) +# +# def testDecompress(self): +# # "Test decompress() function" +# text = bz2.decompress(self.DATA) +# self.assertEqual(text, self.TEXT) +# +# def testDecompressEmpty(self): +# # "Test decompress() function with empty string" +# text = bz2.decompress("") +# self.assertEqual(text, "") +# +# def testDecompressIncomplete(self): +# # "Test decompress() function with incomplete data" +# self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) +# +# def test_main(): +# test_support.run_unittest( +# BZ2FileTest, +# BZ2CompressorTest, +# BZ2DecompressorTest, +# FuncTest +# ) +# test_support.reap_children() +# +# if __name__ == '__main__': +# test_main() +# +# # vim:ts=4:sw=4 From rhymes at codespeak.net Fri Aug 4 16:27:49 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 16:27:49 +0200 (CEST) Subject: [pypy-svn] r31004 - pypy/dist/pypy/module/bz2 Message-ID: <20060804142749.D96A31007C@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 16:27:46 2006 New Revision: 31004 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: BZ2File destruction Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 16:27:46 2006 @@ -73,12 +73,20 @@ pythonapi.PyFile_SetBufSize.restype = c_void pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] pythonapi.PyFile_AsFile.restype = POINTER(FILE) +pythonapi.PyMem_Free.argtypes = [c_char_p] +pythonapi.PyMem_Free.restype = c_void libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, c_int, c_void_p, c_int] libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, c_int, c_int] libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] +libbz2.BZ2_bzReadClose.restype = c_void +libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), + c_int, POINTER(c_uint), POINTER(c_uint)] +libbz2.BZ2_bzWriteClose.restype = c_void + def _catch_bz2_error(space, bzerror): if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: @@ -101,6 +109,10 @@ raise OperationError(space.w_RuntimeError, space.wrap("wrong sequence of bz2 library commands used")) +def _drop_readahead(obj): + if obj.f_buf: + pythonapi.PyMem_Free(obj.f_buf) + obj.f_buf = c_char_p() class _BZ2File(Wrappable): def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): @@ -178,6 +190,16 @@ _catch_bz2_error(self.space, bzerror) self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] + + def __del__(self): + bzerror = c_int() + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + _drop_readahead(self) _BZ2File.typedef = TypeDef("_BZ2File") From rhymes at codespeak.net Fri Aug 4 16:32:38 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 16:32:38 +0200 (CEST) Subject: [pypy-svn] r31006 - pypy/dist/pypy/module/bz2 Message-ID: <20060804143238.86B3110078@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 16:32:35 2006 New Revision: 31006 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: add doc for BZFile factory function Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 16:32:35 2006 @@ -204,6 +204,20 @@ _BZ2File.typedef = TypeDef("_BZ2File") def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): + """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object + + Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or + writing. When opened for writing, the file will be created if it doesn't + exist, and truncated otherwise. If the buffering argument is given, 0 means + unbuffered, and larger numbers specify the buffer size. If compresslevel + is given, must be a number between 1 and 9. + + Add a 'U' to mode to open the file for input with universal newline + support. Any line ending in the input file will be seen as a '\\n' in + Python. Also, a file so opened gains the attribute 'newlines'; the value + for this attribute is one of None (no newline read yet), '\\r', '\\n', + '\\r\\n' or a tuple containing all the newline types seen. Universal + newlines are available only when reading.""" return _BZ2File(space, filename, mode, buffering, compresslevel) BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] From rhymes at codespeak.net Fri Aug 4 17:02:33 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 17:02:33 +0200 (CEST) Subject: [pypy-svn] r31007 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060804150233.05FB210072@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 17:02:30 2006 New Revision: 31007 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: close() method. This implementation cannot be called twice or more due to fclose() in the implementation. Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 17:02:30 2006 @@ -1,4 +1,5 @@ from pypy.rpython.rctypes.tool import ctypes_platform +from pypy.rpython.rctypes.tool.libc import libc import pypy.rpython.rctypes.implementation # this defines rctypes magic from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError @@ -19,6 +20,7 @@ _header_ = """ #include #include + #include """ off_t = ctypes_platform.SimpleType("off_t", c_longlong) size_t = ctypes_platform.SimpleType("size_t", c_ulong) @@ -86,7 +88,14 @@ libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), c_int, POINTER(c_uint), POINTER(c_uint)] libbz2.BZ2_bzWriteClose.restype = c_void - +libc.strerror.restype = c_char_p +libc.strerror.argtypes = [c_int] +libc.fclose.argtypes = [POINTER(FILE)] +libc.fclose.restype = c_int + +def _get_error_msg(): + errno = geterrno() + return libc.strerror(errno) def _catch_bz2_error(space, bzerror): if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: @@ -200,8 +209,41 @@ libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) _drop_readahead(self) + + def close(self): + """close() -> None or (perhaps) an integer + + Close the file. Sets data attribute .closed to true. A closed file + cannot be used for further I/O operations.""" + + # this feature is not supported due to fclose(): + # close() may be called more than once without error. + + bzerror = c_int(BZ_OK) + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + self.mode = MODE_CLOSED + + # close the underline file + ret = libc.fclose(self._file) + if ret != 0: + raise OperationError(self.space.w_IOError, + self.space.wrap(_get_error_msg())) + + if bzerror != BZ_OK: + return _catch_bz2_error(self.space, bzerror) + + return ret + close.unwrap_spec = ['self'] -_BZ2File.typedef = TypeDef("_BZ2File") +_BZ2File.typedef = TypeDef("_BZ2File", + close = interp2app(_BZ2File.close, + unwrap_spec=_BZ2File.close.unwrap_spec), +) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 17:02:30 2006 @@ -22,7 +22,19 @@ BZ2File("foo", mode='wb') # a large buf size BZ2File("foo", mode='w', buffering=4096) + + def test_close(self): + from bz2 import BZ2File + # readonly + bz2f = BZ2File("foo", mode='w') + bz2f.close() + # since we use fclose() internally you can't close it twice + # bz2f.close() + + # writeonly + bz2f = BZ2File("foo", mode='r') + bz2f.close() # #!/usr/bin/python # from test import test_support From rhymes at codespeak.net Fri Aug 4 17:11:57 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 17:11:57 +0200 (CEST) Subject: [pypy-svn] r31008 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060804151157.51DF31007A@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 17:11:54 2006 New Revision: 31008 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: tell() implemented Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 17:11:54 2006 @@ -239,10 +239,24 @@ return ret close.unwrap_spec = ['self'] + + def tell(self): + """tell() -> int + + Return the current file position, an integer (may be a long integer).""" + + if self.mode == MODE_CLOSED: + raise OperationError(self.space.w_ValueError, + self.space.wrap("I/O operation on closed file")) + + return self.space.wrap(self.pos) + tell.unwrap_spec = ['self'] _BZ2File.typedef = TypeDef("_BZ2File", close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), + tell = interp2app(_BZ2File.tell, + unwrap_spec=_BZ2File.tell.unwrap_spec), ) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 17:11:54 2006 @@ -26,16 +26,28 @@ def test_close(self): from bz2 import BZ2File - # readonly + # writeonly bz2f = BZ2File("foo", mode='w') bz2f.close() # since we use fclose() internally you can't close it twice # bz2f.close() - # writeonly + # readonly bz2f = BZ2File("foo", mode='r') bz2f.close() + def test_tell(self): + from bz2 import BZ2File + + bz2f = BZ2File("foo", mode='w') + bz2f.close() + raises(ValueError, bz2f.tell) + + bz2f = BZ2File("foo", mode='w') + pos = bz2f.tell() + assert pos == 0 + + # #!/usr/bin/python # from test import test_support # from test.test_support import TESTFN From mwh at codespeak.net Fri Aug 4 17:18:38 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 4 Aug 2006 17:18:38 +0200 (CEST) Subject: [pypy-svn] r31009 - pypy/dist/pypy/translator/stackless Message-ID: <20060804151838.8A1BB1007A@code0.codespeak.net> Author: mwh Date: Fri Aug 4 17:18:37 2006 New Revision: 31009 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: cleanups, delete some dead code. no changes in behaviour Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Fri Aug 4 17:18:37 2006 @@ -138,18 +138,12 @@ # cast blocks will (conceptually at least) have a switch on the # restate subcase at the end of it. # -# conditions for reuse: -# -# 1. retrieval block: the erased types being read out -# -# 2. the return value blocks: the erased the erased return type and the exception -# and target of any exception links -# -# 3. the cast blocks: the exact types required. -# # note that we order types by the index of the erased type in # STORAGE_TYPES, to increase the chance that we can resuse the types. - +# +# in simple cases this approach creates graphs that are more +# complicated than needed, so we run the graph through a few +# simplifications to join blocks and remove unused variables. class FrameTyper: @@ -161,13 +155,13 @@ self.transformer = transformer - def _key_for_types(self, types): + def _key_for_types(self, TYPES): counts = {} - for tt in types: - if tt is lltype.Void: + for EXACT_TYPE in TYPES: + if EXACT_TYPE is lltype.Void: continue - t = storage_type(tt) - counts[t] = counts.get(t, 0) + 1 + ERASED_TYPE = storage_type(EXACT_TYPE) + counts[ERASED_TYPE] = counts.get(ERASED_TYPE, 0) + 1 key = lltype.frozendict(counts) return key @@ -190,11 +184,10 @@ resulttype=lltype.Ptr(FRAME_TYPE)) for fieldname in FRAME_TYPE._names[1:]: # skip the 'header' field - TYPE = FRAME_TYPE._flds[fieldname] - var = varoftype(TYPE) - save_block.inputargs.append(var) + v_arg = varoftype(FRAME_TYPE._flds[fieldname]) + save_block.inputargs.append(v_arg) llops.genop('setfield', - [v_state, model.Constant(fieldname, lltype.Void), var], + [v_state, model.Constant(fieldname, lltype.Void), v_arg], resulttype=lltype.Void) v_header = gen_cast(llops, lltype.Ptr(STATE_HEADER), v_state) @@ -220,7 +213,6 @@ key = self._key_for_types([v.concretetype for v in vars]) if key not in self.frametypes: fields = [] - fieldsbytype = {} tcounts = [] for t in STORAGE_TYPES: tcount = key.get(t, 0) @@ -228,15 +220,12 @@ for j in range(tcount): fname = 'state_%s_%d' % (STORAGE_FIELDS[t], j) fields.append((fname, t)) - fieldsbytype.setdefault(t, []).append(fname) FRAME_TYPE = frame.make_state_header_type( "FrameState_"+'_'.join(tcounts), *fields) - self.frametypes[key] = (FRAME_TYPE, - self.saving_function_for_type(FRAME_TYPE), - fieldsbytype) + self.frametypes[key] = (FRAME_TYPE, self.saving_function_for_type(FRAME_TYPE)) - T, save_state_funcptr, fieldsbytype = self.frametypes[key] + T, save_state_funcptr = self.frametypes[key] varsforcall = list(vars) def key(v): return STORAGE_TYPES.index(storage_type(v.concretetype)) @@ -245,21 +234,17 @@ varsforcall.sort(mycmp) return T, varsforcall, save_state_funcptr - def ensure_frame_type_for_types(self, frame_type): - assert len(frame_type._names[1:]) <= 1, "too lazy" - if len(frame_type._names[1:]) == 1: - fname, = frame_type._names[1:] - t = frame_type._flds[fname] - fieldsbytype = {t:[fname]} - key = self._key_for_types([t]) + def ensure_frame_type_for_types(self, FRAME_TYPE): + assert len(FRAME_TYPE._names[1:]) <= 1, "too lazy" + if len(FRAME_TYPE._names[1:]) == 1: + fname, = FRAME_TYPE._names[1:] + T = FRAME_TYPE._flds[fname] + key = self._key_for_types([T]) else: key = self._key_for_types([]) - fieldsbytype = {} if key in self.frametypes: - assert self.frametypes[key][0] is frame_type - self.frametypes[key] = (frame_type, - self.saving_function_for_type(frame_type), - fieldsbytype) + assert self.frametypes[key][0] is FRAME_TYPE + self.frametypes[key] = (FRAME_TYPE, self.saving_function_for_type(FRAME_TYPE)) class StacklessAnalyzer(graphanalyze.GraphAnalyzer): @@ -279,7 +264,6 @@ def analyze_external_call(self, op): callable = op.args[0].value._obj._callable - #assert getattr(callable, 'suggested_primitive', False) return callable in [ll_stack.ll_stack_unwind, ll_stack.ll_stack_capture, ll_stackless.ll_stackless_stack_frames_depth, ll_stackless.ll_stackless_switch] @@ -294,7 +278,7 @@ and arg not in args \ and arg not in [l.last_exception, l.last_exc_value]: args.append(arg) - return args + return args class StacklessTransformer(object): @@ -517,8 +501,6 @@ self.insert_resume_handling(graph) self.generate_restart_infos(graph) - model.checkgraph(graph) - if SAVE_STATISTICS: pot_exact_save_count = 0 for t, count in self.stats.cur_rp_exact_types.items(): @@ -536,48 +518,30 @@ self.curr_graph = None self.curr_graph_save_blocks = None - - def ops_read_global_state_field(self, targetvar, fieldname): - ops = [] - llfieldname = "inst_%s" % fieldname - llfieldtype = self.ll_global_state.value._T._flds[llfieldname] - if llfieldtype == targetvar.concretetype: - tmpvar = targetvar - else: - assert isinstance(llfieldtype, lltype.Ptr) - tmpvar = varoftype(llfieldtype) - - ops.append(model.SpaceOperation( - "getfield", - [self.ll_global_state, - model.Constant(llfieldname, lltype.Void)], - tmpvar)) - if tmpvar is not targetvar: - ops.append(model.SpaceOperation( - "cast_pointer", [tmpvar], - targetvar)) - return ops + self.curr_graph_resume_retrieval_blocks = None + self.curr_graph_resume_return_blocks = None + self.curr_graph_resume_cast_blocks = None def insert_resume_handling(self, graph): old_start_block = graph.startblock newinputargs = [unsimplify.copyvar(self.translator.annotator, v) for v in old_start_block.inputargs] new_start_block = model.Block(newinputargs) - var_resume_state = varoftype(lltype.Signed) + v_resume_substate = varoftype(lltype.Signed) new_start_block.operations.append( model.SpaceOperation("getfield", [self.ll_global_state, self.c_restart_substate_name], - var_resume_state)) + v_resume_substate)) not_resuming_link = model.Link(newinputargs, old_start_block, -1) not_resuming_link.llexitcase = -1 resuming_links = [] for resume_index, resume_block in enumerate(self.resume_blocks): resuming_links.append( - model.Link([var_resume_state], resume_block, resume_index)) + model.Link([v_resume_substate], resume_block, resume_index)) resuming_links[-1].llexitcase = resume_index - new_start_block.exitswitch = var_resume_state + new_start_block.exitswitch = v_resume_substate new_start_block.closeblock(not_resuming_link, *resuming_links) old_start_block.isstartblock = False @@ -637,16 +601,15 @@ args = args res = op.result - (frame_type, - varsforcall, saver) = self.frametyper.frame_type_for_vars(parms[1:]) + (FRAME_TYPE, varsforcall, saver) = self.frametyper.frame_type_for_vars(parms[1:]) if label in self.explicit_resume_point_data: - other_type = self.explicit_resume_point_data[label] - assert frame_type == other_type, "inconsistent types for label %r"%(label,) + OTHER_TYPE = self.explicit_resume_point_data[label] + assert FRAME_TYPE == OTHER_TYPE, "inconsistent types for label %r"%(label,) else: - self.explicit_resume_point_data[label] = frame_type + self.explicit_resume_point_data[label] = FRAME_TYPE - self._make_resume_handling(frame_type, varsforcall, res, block.exits) + self._make_resume_handling(FRAME_TYPE, varsforcall, res, block.exits) restart_number = len(self.masterarray1) + len(self.resume_blocks) - 1 @@ -663,16 +626,15 @@ def handle_resume_state_create(self, block, i): op = block.operations[i] llops = LowLevelOpList() - # XXX we do not look at op.args[0], the prevstate, at all label = op.args[1].value parms = op.args[2:] - FRAME, varsforcall, saver = self.frametyper.frame_type_for_vars(parms) + FRAME_TYPE, varsforcall, saver = self.frametyper.frame_type_for_vars(parms) if label in self.explicit_resume_point_data: - other_type = self.explicit_resume_point_data[label] - assert FRAME == other_type, "inconsistent types for label %r"%(label,) + OTHER_TYPE = self.explicit_resume_point_data[label] + assert FRAME_TYPE == OTHER_TYPE, "inconsistent types for label %r"%(label,) else: - self.explicit_resume_point_data[label] = FRAME + self.explicit_resume_point_data[label] = FRAME_TYPE if label in self.symbolic_restart_numbers: symb = self.symbolic_restart_numbers[label] @@ -697,18 +659,18 @@ model.Constant(symb, lltype.Signed)] + realvarsforcall, resulttype = lltype.Void) v_state = varoftype(lltype.Ptr(frame.STATE_HEADER)) - llops.extend(self.ops_read_global_state_field(v_state, "top")) + v_state_hdr = llops.genop("getfield", + [self.ll_global_state, self.c_inst_top_name], + resulttype=lltype.Ptr(STATE_HEADER)) + v_state = gen_cast(llops, lltype.Ptr(FRAME_TYPE), v_state_hdr) llops.genop("setfield", - [self.ll_global_state, - self.c_inst_top_name, - self.c_null_state]) - - v_prevstate = llops.genop('cast_opaque_ptr', [op.args[0]], - resulttype=lltype.Ptr(frame.STATE_HEADER)) - llops.genop('setfield', [v_state, + [self.ll_global_state, self.c_inst_top_name, self.c_null_state]) + + v_prevstate = gen_cast(llops, lltype.Ptr(frame.STATE_HEADER), op.args[0]) + llops.genop('setfield', [v_state_hdr, model.Constant('f_back', lltype.Void), v_prevstate]) - llops.append(model.SpaceOperation('cast_opaque_ptr', [v_state], op.result)) + llops.append(model.SpaceOperation('cast_opaque_ptr', [v_state_hdr], op.result)) block.operations[i:i+1] = llops def handle_resume_state_invoke(self, block): @@ -800,20 +762,20 @@ block.recloseblock(*newexits) self.translator.rtyper._convert_link(block, newlink) - var_unwind_exception = varoftype(evalue) + v_unwind_exception = varoftype(evalue) op = block.operations[i] args = vars_to_save(block) save_block, varsforcall = self.generate_save_and_resume_blocks( - args, var_unwind_exception, op.result, block.exits) + args, v_unwind_exception, op.result, block.exits) - newlink = model.Link(varsforcall + [var_unwind_exception], + newlink = model.Link(varsforcall + [v_unwind_exception], save_block, code.UnwindException) newlink.last_exception = model.Constant(code.UnwindException, etype) - newlink.last_exc_value = var_unwind_exception + newlink.last_exc_value = v_unwind_exception newexits = list(block.exits) newexits.insert(1, newlink) block.recloseblock(*newexits) @@ -883,8 +845,8 @@ else: i += 1 - def generate_save_and_resume_blocks(self, varstosave, var_exception, - var_result, links_to_resumption): + def generate_save_and_resume_blocks(self, varstosave, v_exception, + v_result, links_to_resumption): frame_type, varsforcall, saver = self.frametyper.frame_type_for_vars(varstosave) if SAVE_STATISTICS: self.stats.rp_count += 1 @@ -893,7 +855,7 @@ inc(self.stats.rp_per_graph_type_counts.setdefault(gkey(self.curr_graph), {}), frame_type) exact_key = [v.concretetype for v in varstosave] exact_key.sort() - exact_key = (tuple(exact_key), var_result.concretetype) + exact_key = (tuple(exact_key), v_result.concretetype) inc(self.stats.cur_rp_exact_types, exact_key) inc(self.stats.cur_rp_erased_types, frame_type) @@ -903,12 +865,12 @@ varsforcall = [v for v in varsforcall if v.concretetype != lltype.Void] self._make_resume_handling(frame_type, varsforcall0, - var_result, links_to_resumption) + v_result, links_to_resumption) - return (self._generate_save_block(varsforcall, var_exception, saver), + return (self._generate_save_block(varsforcall, v_exception, saver), varsforcall) - def _generate_save_block(self, varsforcall, var_unwind_exception, saver): + def _generate_save_block(self, varsforcall, v_unwind_exception, saver): conc_types = tuple([v.concretetype for v in varsforcall]) if conc_types in self.curr_graph_save_blocks: return self.curr_graph_save_blocks[conc_types] @@ -916,20 +878,15 @@ edata = rtyper.getexceptiondata() etype = edata.lltype_of_exception_type evalue = edata.lltype_of_exception_value - def cv(v): - if isinstance(v, model.Variable): - return unsimplify.copyvar(None, v) - else: - return varoftype(v.concretetype) - inputargs = [cv(v) for v in varsforcall] - var_unwind_exception = unsimplify.copyvar(None, var_unwind_exception) + inputargs = [copyvar(v) for v in varsforcall] + v_unwind_exception = copyvar(v_unwind_exception) - save_state_block = model.Block(inputargs + [var_unwind_exception]) + save_state_block = model.Block(inputargs + [v_unwind_exception]) saveops = LowLevelOpList() - var_exc = gen_cast(saveops, self.unwind_exception_type, var_unwind_exception) + v_exc = gen_cast(saveops, self.unwind_exception_type, v_unwind_exception) - realvarsforcall = [var_exc] + realvarsforcall = [v_exc] for v in inputargs: realvarsforcall.append(gen_cast(saveops, storage_type(v.concretetype), v)) @@ -946,7 +903,7 @@ if not hasattr(self.curr_graph.exceptblock.inputargs[1], 'concretetype'): self.curr_graph.exceptblock.inputargs[1].concretetype = evalue save_state_block.closeblock(model.Link( - [c_unwindexception, var_unwind_exception], + [c_unwindexception, v_unwind_exception], self.curr_graph.exceptblock)) self.translator.rtyper._convert_link( save_state_block, save_state_block.exits[0]) @@ -955,72 +912,6 @@ self.curr_graph_save_blocks[conc_types] = save_state_block return save_state_block - def _generate_resume_block(self, varsinfieldorder, frame_type, - var_result, links_to_resumption): - newblock = model.Block([]) - newargs = [] - llops = LowLevelOpList() - llops.genop("setfield", - [self.ll_global_state, - self.c_restart_substate_name, - self.c_minus_one]) - frame_top = varoftype(lltype.Ptr(frame_type)) - llops.extend(self.ops_read_global_state_field(frame_top, "top")) - llops.genop("setfield", - [self.ll_global_state, - self.c_inst_top_name, - self.c_null_state]) - varmap = {} - fielditer = iter(frame_type._names[1:]) - for arg in varsinfieldorder: - assert arg is not var_result - t = storage_type(arg.concretetype) - if t is lltype.Void: - v_newarg = model.Constant(None, lltype.Void) - else: - fname = model.Constant(fielditer.next(), lltype.Void) - assert frame_type._flds[fname.value] is t - v_newarg = llops.genop('getfield', [frame_top, fname], - resulttype = t) - v_newarg = gen_cast(llops, arg.concretetype, v_newarg) - varmap[arg] = v_newarg - - rettype = storage_type(var_result.concretetype) - getretval = self.fetch_retvals[rettype] - retval = llops.genop("direct_call", [getretval], - resulttype = rettype) - varmap[var_result] = retval - - newblock.operations.extend(llops) - - def rename(arg): - if isinstance(arg, model.Variable): - if arg in varmap: - return varmap[arg] - else: - assert arg in [l.last_exception, l.last_exc_value] - r = unsimplify.copyvar(None, arg) - varmap[arg] = r - return r - else: - return arg - - newblock.closeblock(*[l.copy(rename) - for l in links_to_resumption]) - # this check is a bit implicit! - if len(links_to_resumption) > 1: - newblock.exitswitch = model.c_last_exception - else: - newblock.exitswitch = None - - if var_result.concretetype != rettype: - self.insert_return_conversion( - newblock.exits[0], var_result.concretetype, retval) - - if SAVE_STATISTICS: - self.stats.resumeops += len(newblock.operations) - return newblock - def _make_resume_handling(self, FRAME_TYPE, sorted_vars, v_retval, links_to_resumption): resume_substate = len(self.resume_blocks) @@ -1078,11 +969,11 @@ [self.ll_global_state, self.c_inst_top_name, self.c_null_state]) output_args = [retrieve_block.inputargs[0]] assert len(FRAME_TYPE._names[1:]) == len(erased_types) - for fieldname, typ in zip(FRAME_TYPE._names[1:], erased_types): - assert FRAME_TYPE._flds[fieldname] == typ + for fieldname, TYPE in zip(FRAME_TYPE._names[1:], erased_types): + assert FRAME_TYPE._flds[fieldname] == TYPE output_args.append(llops.genop("getfield", [v_state, model.Constant(fieldname, lltype.Void)], - resulttype=typ)) + resulttype=TYPE)) retrieve_block.operations = llops return retrieve_block, output_args From rhymes at codespeak.net Fri Aug 4 18:58:43 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 18:58:43 +0200 (CEST) Subject: [pypy-svn] r31010 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060804165843.B2E7B1006E@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 18:58:37 2006 New Revision: 31010 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: seek() and fix of tab/spaces stuff Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Aug 4 18:58:37 2006 @@ -1,279 +1,457 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -import pypy.rpython.rctypes.implementation # this defines rctypes magic -from pypy.rpython.rctypes.aerrno import geterrno -from pypy.interpreter.error import OperationError -from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app -from ctypes import * -import ctypes.util - -from bzlib import bz_stream, BZFILE, FILE -from fileobject import PyFileObject - -libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) - -c_void = None - -class CConfig: - _header_ = """ - #include - #include - #include - """ - off_t = ctypes_platform.SimpleType("off_t", c_longlong) - size_t = ctypes_platform.SimpleType("size_t", c_ulong) - BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") - -constants = {} -constant_names = ['BUFSIZ', 'BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', - 'BZ_RUN_OK', 'BZ_FLUSH_OK', 'BZ_FINISH_OK', 'BZ_STREAM_END', - 'BZ_SEQUENCE_ERROR', 'BZ_PARAM_ERROR', 'BZ_MEM_ERROR', 'BZ_DATA_ERROR', - 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', - 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] -for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) - -class cConfig: - pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) - -for name in constant_names: - value = getattr(cConfig, name) - if value is not None: - constants[name] = value -locals().update(constants) - -off_t = cConfig.off_t -BZ_OK = cConfig.BZ_OK -BZ_STREAM_END = cConfig.BZ_STREAM_END -BZ_CONFIG_ERROR = cConfig.BZ_CONFIG_ERROR -BZ_PARAM_ERROR = cConfig.BZ_PARAM_ERROR -BZ_DATA_ERROR = cConfig.BZ_DATA_ERROR -BZ_DATA_ERROR_MAGIC = cConfig.BZ_DATA_ERROR_MAGIC -BZ_IO_ERROR = cConfig.BZ_IO_ERROR -BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR -BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF -BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR - -# modes -MODE_CLOSED = 0 -MODE_READ = 1 -MODE_READ_EOF = 2 -MODE_WRITE = 3 - -# bits in f_newlinetypes -NEWLINE_UNKNOWN = 0 # No newline seen, yet -NEWLINE_CR = 1 # \r newline seen -NEWLINE_LF = 2 # \n newline seen -NEWLINE_CRLF = 4 # \r\n newline seen - -pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] -pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) -pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] -pythonapi.PyFile_SetBufSize.restype = c_void -pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] -pythonapi.PyFile_AsFile.restype = POINTER(FILE) -pythonapi.PyMem_Free.argtypes = [c_char_p] -pythonapi.PyMem_Free.restype = c_void -libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_void_p, c_int] -libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_int] -libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] -libbz2.BZ2_bzReadClose.restype = c_void -libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), - c_int, POINTER(c_uint), POINTER(c_uint)] -libbz2.BZ2_bzWriteClose.restype = c_void -libc.strerror.restype = c_char_p -libc.strerror.argtypes = [c_int] -libc.fclose.argtypes = [POINTER(FILE)] -libc.fclose.restype = c_int - -def _get_error_msg(): - errno = geterrno() - return libc.strerror(errno) - -def _catch_bz2_error(space, bzerror): - if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: - raise OperationError(space.w_SystemError, - space.wrap("the bz2 library was not compiled correctly")) - if bzerror == BZ_PARAM_ERROR: - raise OperationError(space.w_SystemError, - space.wrap("the bz2 library has received wrong parameters")) - elif bzerror == BZ_MEM_ERROR: - raise OperationError(space.w_MemoryError, space.wrap("")) - elif bzerror in (BZ_DATA_ERROR, BZ_DATA_ERROR_MAGIC): - raise OperationError(space.w_IOError, space.wrap("invalid data stream")) - elif bzerror == BZ_IO_ERROR: - raise OperationError(space.w_IOError, space.wrap("unknown IO error")) - elif bzerror == BZ_UNEXPECTED_EOF: - raise OperationError(space.w_EOFError, - space.wrap( - "compressed file ended before the logical end-of-stream was detected")) - elif bzerror == BZ_SEQUENCE_ERROR: - raise OperationError(space.w_RuntimeError, - space.wrap("wrong sequence of bz2 library commands used")) - -def _drop_readahead(obj): - if obj.f_buf: - pythonapi.PyMem_Free(obj.f_buf) - obj.f_buf = c_char_p() - -class _BZ2File(Wrappable): - def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): - self.space = space - - self.f_buf = c_char_p() # allocated readahead buffer - self.f_bufend = c_char_p() # points after last occupied position - self.f_bufptr = c_char_p() # current buffer position - - self.f_softspace = 0 # flag used by print command - - self.f_univ_newline = False # handle any newline convention - self.f_newlinetypes = 0 # types of newlines seen - self.f_skipnextlf = 0 # skip next \n - - self.mode = 0 - self.pos = 0 - self.size = 0 - - self._init_bz2file(filename, mode, buffering, compresslevel) - - def _init_bz2file(self, filename, mode_, buffering, compresslevel): - self.size = -1 - - name = filename - mode_char = "" - mode_list = mode_ - - if compresslevel < 1 or compresslevel > 9: - raise OperationError(self.space.w_ValueError, - self.space.wrap("compresslevel must be between 1 and 9")) - - i = 1 - for mode in mode_list: - error = False - - if mode in ['r', 'w']: - if mode_char: - error = True - mode_char = mode - elif mode == 'b': - pass - elif mode == 'U': - self.f_univ_newline = True - else: - error = True - - if error: - raise OperationError(self.space.w_ValueError, - self.space.wrap("invalid mode char %s" % mode)) - - if mode_char == 0: - mode_char = 'r' - mode = ('wb', 'rb')[mode_char == 'r'] - - # open the file and set the buffer - f = pythonapi.PyFile_FromString(name, mode) - if not f: - raise OperationError(self.space.w_IOError, - self.space.wrap("cannot open file %s" % name)) - pythonapi.PyFile_SetBufSize(f, buffering) - - # store the FILE object - self._file = pythonapi.PyFile_AsFile(f) - - bzerror = c_int() - if mode_char == 'r': - self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, - 0, 0, None, 0) - else: - self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, - compresslevel, 0, 0) - - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) - - self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] - - def __del__(self): - bzerror = c_int() - - if self.mode in (MODE_READ, MODE_READ_EOF): - libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - elif self.mode == MODE_WRITE: - libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) - - _drop_readahead(self) - - def close(self): - """close() -> None or (perhaps) an integer - - Close the file. Sets data attribute .closed to true. A closed file - cannot be used for further I/O operations.""" - - # this feature is not supported due to fclose(): - # close() may be called more than once without error. - - bzerror = c_int(BZ_OK) - - if self.mode in (MODE_READ, MODE_READ_EOF): - libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - elif self.mode == MODE_WRITE: - libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) - - self.mode = MODE_CLOSED - - # close the underline file - ret = libc.fclose(self._file) - if ret != 0: - raise OperationError(self.space.w_IOError, - self.space.wrap(_get_error_msg())) - - if bzerror != BZ_OK: - return _catch_bz2_error(self.space, bzerror) - - return ret - close.unwrap_spec = ['self'] - - def tell(self): - """tell() -> int - - Return the current file position, an integer (may be a long integer).""" - - if self.mode == MODE_CLOSED: - raise OperationError(self.space.w_ValueError, - self.space.wrap("I/O operation on closed file")) - - return self.space.wrap(self.pos) - tell.unwrap_spec = ['self'] - -_BZ2File.typedef = TypeDef("_BZ2File", - close = interp2app(_BZ2File.close, - unwrap_spec=_BZ2File.close.unwrap_spec), - tell = interp2app(_BZ2File.tell, - unwrap_spec=_BZ2File.tell.unwrap_spec), -) - -def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): - """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object - - Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or - writing. When opened for writing, the file will be created if it doesn't - exist, and truncated otherwise. If the buffering argument is given, 0 means - unbuffered, and larger numbers specify the buffer size. If compresslevel - is given, must be a number between 1 and 9. - - Add a 'U' to mode to open the file for input with universal newline - support. Any line ending in the input file will be seen as a '\\n' in - Python. Also, a file so opened gains the attribute 'newlines'; the value - for this attribute is one of None (no newline read yet), '\\r', '\\n', - '\\r\\n' or a tuple containing all the newline types seen. Universal - newlines are available only when reading.""" - return _BZ2File(space, filename, mode, buffering, compresslevel) -BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] - +from pypy.rpython.rctypes.tool import ctypes_platform +from pypy.rpython.rctypes.tool.libc import libc +import pypy.rpython.rctypes.implementation # this defines rctypes magic +from pypy.rpython.rctypes.aerrno import geterrno +from pypy.interpreter.error import OperationError +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from ctypes import * +import ctypes.util + +from bzlib import bz_stream, BZFILE, FILE +from fileobject import PyFileObject + +libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) + +c_void = None + +class CConfig: + _header_ = """ + #include + #include + #include + """ + off_t = ctypes_platform.SimpleType("off_t", c_longlong) + size_t = ctypes_platform.SimpleType("size_t", c_ulong) + BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") + SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET") + +constants = {} +constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', + 'BZ_RUN_OK', 'BZ_FLUSH_OK', 'BZ_FINISH_OK', 'BZ_STREAM_END', + 'BZ_SEQUENCE_ERROR', 'BZ_PARAM_ERROR', 'BZ_MEM_ERROR', 'BZ_DATA_ERROR', + 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', + 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] +for name in constant_names: + setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + +class cConfig: + pass +cConfig.__dict__.update(ctypes_platform.configure(CConfig)) + +for name in constant_names: + value = getattr(cConfig, name) + if value is not None: + constants[name] = value +locals().update(constants) + +off_t = cConfig.off_t +BUFSIZ = cConfig.BUFSIZ +SEEK_SET = cConfig.SEEK_SET +BZ_OK = cConfig.BZ_OK +BZ_STREAM_END = cConfig.BZ_STREAM_END +BZ_CONFIG_ERROR = cConfig.BZ_CONFIG_ERROR +BZ_PARAM_ERROR = cConfig.BZ_PARAM_ERROR +BZ_DATA_ERROR = cConfig.BZ_DATA_ERROR +BZ_DATA_ERROR_MAGIC = cConfig.BZ_DATA_ERROR_MAGIC +BZ_IO_ERROR = cConfig.BZ_IO_ERROR +BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR +BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF +BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR + +# modes +MODE_CLOSED = 0 +MODE_READ = 1 +MODE_READ_EOF = 2 +MODE_WRITE = 3 + +# bits in f_newlinetypes +NEWLINE_UNKNOWN = 0 # No newline seen, yet +NEWLINE_CR = 1 # \r newline seen +NEWLINE_LF = 2 # \n newline seen +NEWLINE_CRLF = 4 # \r\n newline seen + +if BUFSIZ < 8192: + SMALLCHUNK = 8192 +else: + SMALLCHUNK = BUFSIZ + + +pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] +pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) +pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] +pythonapi.PyFile_SetBufSize.restype = c_void +pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] +pythonapi.PyFile_AsFile.restype = POINTER(FILE) +pythonapi.PyMem_Free.argtypes = [c_char_p] +pythonapi.PyMem_Free.restype = c_void + +libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_void_p, c_int] +libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_int] +libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] +libbz2.BZ2_bzReadClose.restype = c_void +libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), + c_int, POINTER(c_uint), POINTER(c_uint)] +libbz2.BZ2_bzWriteClose.restype = c_void +libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] +libbz2.BZ2_bzRead.restype = c_int + +libc.strerror.restype = c_char_p +libc.strerror.argtypes = [c_int] +libc.fclose.argtypes = [POINTER(FILE)] +libc.fclose.restype = c_int +libc.fseek.argtypes = [POINTER(FILE), c_int, c_int] +libc.fseek.restype = c_int + +def _get_error_msg(): + errno = geterrno() + return libc.strerror(errno) + +def _catch_bz2_error(space, bzerror): + if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library was not compiled correctly")) + if bzerror == BZ_PARAM_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library has received wrong parameters")) + elif bzerror == BZ_MEM_ERROR: + raise OperationError(space.w_MemoryError, space.wrap("")) + elif bzerror in (BZ_DATA_ERROR, BZ_DATA_ERROR_MAGIC): + raise OperationError(space.w_IOError, space.wrap("invalid data stream")) + elif bzerror == BZ_IO_ERROR: + raise OperationError(space.w_IOError, space.wrap("unknown IO error")) + elif bzerror == BZ_UNEXPECTED_EOF: + raise OperationError(space.w_EOFError, + space.wrap( + "compressed file ended before the logical end-of-stream was detected")) + elif bzerror == BZ_SEQUENCE_ERROR: + raise OperationError(space.w_RuntimeError, + space.wrap("wrong sequence of bz2 library commands used")) + +def _drop_readahead(obj): + if obj.f_buf: + pythonapi.PyMem_Free(obj.f_buf) + obj.f_buf = c_char_p() + +def _univ_newline_read(bzerror, stream, buf, n, obj): + dst = buf + + if not obj.f_univ_newline: + return libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + + newlinetypes = obj.f_newlinetypes + skipnextlf = obj.f_skipnextlf + + while n: + src = dst + + nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + n -= nread # assuming 1 byte out for each in; will adjust + shortread = n != 0 # True iff EOF or error + + # needed to operate with "pointers" + src_lst = list(src.value) + src_pos = 0 + dst_lst = list(dst.value) + dst_pos = 0 + while nread: + nread -= 1 + + c = src_lst[src_pos] + src_pos += 1 + + if c == '\r': + # save as LF and set flag to skip next LF. + dst_lst[dst_pos] = '\n' + dst_pos += 1 + skipnextlf = True + elif skipnextlf and c == '\n': + # skip LF, and remember we saw CR LF. + skipnextlf = False + newlinetypes |= NEWLINE_CRLF + n += 1 + else: + # normal char to be stored in buffer. Also + # update the newlinetypes flag if either this + # is an LF or the previous char was a CR. + if c == '\n': + newlinetypes |= NEWLINE_LF + elif skipnextlf: + newlinetypes |= NEWLINE_CR + + dst_lst[dst_pos] = c + dst_pos += 1 + + skipnextlf = False + + if shortread: + # if this is EOF, update type flags. + if skipnextlf and (bzerror == BZ_STREAM_END): + newlinetypes |= NEWLINE_CR + break + + obj.f_newlinetypes = newlinetypes + obj.f_skipnextlf = skipnextlf + + buf = c_char_p("".join(dst_lst)) + + return dst_pos + + +class _BZ2File(Wrappable): + def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): + self.space = space + + self.f_buf = c_char_p() # allocated readahead buffer + self.f_bufend = c_char_p() # points after last occupied position + self.f_bufptr = c_char_p() # current buffer position + + self.f_softspace = 0 # flag used by print command + + self.f_univ_newline = False # handle any newline convention + self.f_newlinetypes = 0 # types of newlines seen + self.f_skipnextlf = 0 # skip next \n + + self.mode = 0 + self.pos = 0 + self.size = 0 + + self._init_bz2file(filename, mode, buffering, compresslevel) + + def _init_bz2file(self, filename, mode_, buffering, compresslevel): + self.size = -1 + + name = filename + mode_char = "" + mode_list = mode_ + + if compresslevel < 1 or compresslevel > 9: + raise OperationError(self.space.w_ValueError, + self.space.wrap("compresslevel must be between 1 and 9")) + + for mode in mode_list: + error = False + + if mode in ['r', 'w']: + if mode_char: + error = True + mode_char = mode + elif mode == 'b': + pass + elif mode == 'U': + self.f_univ_newline = True + else: + error = True + + if error: + raise OperationError(self.space.w_ValueError, + self.space.wrap("invalid mode char %s" % mode)) + + if mode_char == 0: + mode_char = 'r' + mode = ('wb', 'rb')[mode_char == 'r'] + + # open the file and set the buffer + f = pythonapi.PyFile_FromString(name, mode) + if not f: + raise OperationError(self.space.w_IOError, + self.space.wrap("cannot open file %s" % name)) + pythonapi.PyFile_SetBufSize(f, buffering) + + # store the FILE object + self._file = pythonapi.PyFile_AsFile(f) + + bzerror = c_int() + if mode_char == 'r': + self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, + 0, 0, None, 0) + else: + self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, + compresslevel, 0, 0) + + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] + + def __del__(self): + bzerror = c_int() + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + _drop_readahead(self) + + def _check_if_close(self): + if self.mode == MODE_CLOSED: + raise OperationError(self.space.w_ValueError, + self.space.wrap("I/O operation on closed file")) + + def close(self): + """close() -> None or (perhaps) an integer + + Close the file. Sets data attribute .closed to true. A closed file + cannot be used for further I/O operations.""" + + # this feature is not supported due to fclose(): + # close() may be called more than once without error. + + bzerror = c_int(BZ_OK) + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + self.mode = MODE_CLOSED + + # close the underline file + ret = libc.fclose(self._file) + if ret != 0: + raise OperationError(self.space.w_IOError, + self.space.wrap(_get_error_msg())) + + if bzerror != BZ_OK: + return _catch_bz2_error(self.space, bzerror) + + return ret + close.unwrap_spec = ['self'] + + def tell(self): + """tell() -> int + + Return the current file position, an integer (may be a long integer).""" + + self._check_if_close() + + return self.space.wrap(self.pos) + tell.unwrap_spec = ['self'] + + def seek(self, offset, whence=0): + """"seek(offset [, whence]) -> None + + Move to new file position. Argument offset is a byte count. Optional + argument whence defaults to 0 (offset from start of file, offset + should be >= 0); other values are 1 (move relative to current position, + positive or negative), and 2 (move relative to end of file, usually + negative, although many platforms allow seeking beyond the end of a file). + + Note that seeking of bz2 files is emulated, and depending on the parameters + the operation may be extremely slow.""" + + _drop_readahead(self) + self._check_if_close() + + buf = c_char_p() + bufsize = SMALLCHUNK + bytesread = 0 + bzerror = c_int() + + if self.mode not in (MODE_READ, MODE_READ_EOF): + raise OperationError(self.space.w_IOError, + self.space.wrap("seek works only while reading")) + + if whence == 2: + if self.size == -1: + while True: + chunksize = _univ_newline_read(bzerror, self.fp, buf, + bufsize, self) + self.pos += chunksize + bytesread += chunksize + + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_OK: + _catch_bz2_error(bzerror) + + self.mode = MODE_READ_EOF + self.size = self.pos + bytesread = 0 + offset += self.size + elif whence == 1: + offset += self.pos + + # Before getting here, offset must be the absolute position the file + # pointer should be set to. + if offset >= self.pos: + # we can move forward + offset -= self.pos + else: + # we cannot move back, so rewind the stream + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + if bzerror != BZ_OK: + _catch_bz2_error(bzerror) + + ret = libc.fseek(self._file, 0, SEEK_SET) + if ret != 0: + raise OperationError(self.space.w_IOError, + self.space.wrap(_get_error_msg())) + + self.pos = 0 + self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, + 0, 0, None, 0) + if bzerror != BZ_OK: + _catch_bz2_error(bzerror) + + self.mode = MODE_READ + + if offset <= 0 or self.mode == MODE_READ_EOF: + return + + # Before getting here, offset must be set to the number of bytes + # to walk forward. + while True: + if (offset - bytesread) > bufsize: + readsize = bufsize + else: + # offset might be wider that readsize, but the result + # of the subtraction is bound by buffersize (see the + # condition above). bufsize is 8192. + readsize = offset - bytesread + + chunksize = _univ_newline_read(bzerror, self.fp, buf, readsize, self) + self.pos += chunksize + bytesread += chunksize + + if bzerror == BZ_STREAM_END: + self.size = self.pos + self.mode = MODE_READ_EOF + elif bzerror != BZ_OK: + _catch_bz2_error(bzerror) + + if bytesread == offset: + break + seek.unwrap_spec = ['self', int, int] + + +_BZ2File.typedef = TypeDef("_BZ2File", + close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), + tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), + seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), +) + +def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): + """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object + + Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or + writing. When opened for writing, the file will be created if it doesn't + exist, and truncated otherwise. If the buffering argument is given, 0 means + unbuffered, and larger numbers specify the buffer size. If compresslevel + is given, must be a number between 1 and 9. + + Add a 'U' to mode to open the file for input with universal newline + support. Any line ending in the input file will be seen as a '\\n' in + Python. Also, a file so opened gains the attribute 'newlines'; the value + for this attribute is one of None (no newline read yet), '\\r', '\\n', + '\\r\\n' or a tuple containing all the newline types seen. Universal + newlines are available only when reading.""" + return _BZ2File(space, filename, mode, buffering, compresslevel) +BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] + Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 18:58:37 2006 @@ -46,7 +46,37 @@ bz2f = BZ2File("foo", mode='w') pos = bz2f.tell() assert pos == 0 + + def test_seek(self): + from bz2 import BZ2File + # hack to create a foo file + open("foo", "w").close() + + # cannot seek if close + bz2f = BZ2File("foo", mode='r') + bz2f.close() + raises(ValueError, bz2f.seek, 0) + + # cannot seek if 'w' + bz2f = BZ2File("foo", mode='w') + raises(IOError, bz2f.seek, 0) + bz2f.close() + + bz2f = BZ2File("foo", mode='r') + raises(TypeError, bz2f.seek, "foo") + raises(TypeError, bz2f.seek, 0, "foo") + + bz2f.seek(0) + assert bz2f.tell() == 0 + assert False + # bz2f.read(1) + # bz2f.seek(1, 1) + # assert bz2f.tell() == 2 + # bz2f.seek(0) + # bz2f.seek(-1, 2) + # assert bz2f.tell() == 5 + # #!/usr/bin/python # from test import test_support From rhymes at codespeak.net Fri Aug 4 19:00:57 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 19:00:57 +0200 (CEST) Subject: [pypy-svn] r31011 - pypy/dist/pypy/module/bz2/test Message-ID: <20060804170057.4ED5B1006E@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 19:00:54 2006 New Revision: 31011 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: mask in MS Windows by now Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 19:00:54 2006 @@ -2,6 +2,9 @@ from pypy.conftest import gettestobjspace import os +if os.name == "nt": + skip("fcntl module is not available on Windows") + def teardown_module(mod): if os.path.exists("foo"): os.unlink("foo") From rhymes at codespeak.net Fri Aug 4 19:06:34 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 4 Aug 2006 19:06:34 +0200 (CEST) Subject: [pypy-svn] r31012 - pypy/dist/pypy/module/bz2/test Message-ID: <20060804170634.0EEEB1006E@code0.codespeak.net> Author: rhymes Date: Fri Aug 4 19:06:32 2006 New Revision: 31012 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: oops Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Fri Aug 4 19:06:32 2006 @@ -72,7 +72,6 @@ bz2f.seek(0) assert bz2f.tell() == 0 - assert False # bz2f.read(1) # bz2f.seek(1, 1) # assert bz2f.tell() == 2 From fijal at codespeak.net Fri Aug 4 23:22:10 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Aug 2006 23:22:10 +0200 (CEST) Subject: [pypy-svn] r31013 - pypy/dist/pypy/doc/discussion Message-ID: <20060804212210.436A91006C@code0.codespeak.net> Author: fijal Date: Fri Aug 4 23:22:07 2006 New Revision: 31013 Added: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt (contents, props changed) Log: Added preliminary version of my SoP proposal. Added: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt Fri Aug 4 23:22:07 2006 @@ -0,0 +1,31 @@ +============================================ +Summer of PyPy proposal: Distributed py.test +============================================ + + +Purpose: +======== + +The main purpose of distributing py.test is to speedup tests +of actual applications (running all pypy tests already takes +ages). + +Method: +======= + +XXX: some draft + +- remote import (which I've already implemented) +- run on session level probably (like RemoteSession) +- do not run on low level (around call(*args)) [previous, + Executor approach], because there are problems with displaying and + we need everything to get every data. +- We need to get data on higher level - perform remote import of py lib + and run session there (ServerSession or so) to send data to client + and get info. Probably more sophisticated stuff such as PDB, or any other + input will get run by re-running test on client side (transfering every pdb + call seems hard and what if they implement readline-like approach in pdb?) + +XXX: Remote import, how it works + +Sending data as new files. Possible additions of some kind of checksums. \ No newline at end of file From guido at codespeak.net Sat Aug 5 00:32:33 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Sat, 5 Aug 2006 00:32:33 +0200 (CEST) Subject: [pypy-svn] r31016 - pypy/branch/guido-config-testing Message-ID: <20060804223233.1744410076@code0.codespeak.net> Author: guido Date: Sat Aug 5 00:32:26 2006 New Revision: 31016 Added: pypy/branch/guido-config-testing/ - copied from r31015, pypy/dist/ Log: Creating branch to test the new config stuff a bit. From guido at codespeak.net Sat Aug 5 00:39:59 2006 From: guido at codespeak.net (guido at codespeak.net) Date: Sat, 5 Aug 2006 00:39:59 +0200 (CEST) Subject: [pypy-svn] r31017 - pypy/branch/guido-config-testing/pypy Message-ID: <20060804223959.EE83710076@code0.codespeak.net> Author: guido Date: Sat Aug 5 00:39:56 2006 New Revision: 31017 Modified: pypy/branch/guido-config-testing/pypy/conftest.py Log: Using pypy.config now to get config options for py.test. Modified: pypy/branch/guido-config-testing/pypy/conftest.py ============================================================================== --- pypy/branch/guido-config-testing/pypy/conftest.py (original) +++ pypy/branch/guido-config-testing/pypy/conftest.py Sat Aug 5 00:39:56 2006 @@ -3,6 +3,7 @@ from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport from pypy.tool.option import make_config +from pypy.config.config import to_optparse from inspect import isclass rootdir = py.magic.autopath().dirpath() @@ -20,38 +21,27 @@ def usemodules_callback(option, opt, value, parser): parser.values.usemodules.append(value) -# XXX these options should go away - -option = py.test.Config.addoptions("pypy options", - Option('-O', '--objspace', action="store", default=None, - type="string", dest="objspace", - help="object space to run tests on."), - Option('--oldstyle', action="store_true",dest="oldstyle", default=False, - help="enable oldstyle classes as default metaclass"), - Option('--uselibfile', action="store_true", - dest="uselibfile", default=False, - help="enable our applevel file implementation"), - Option('--nofaking', action="store_true", - dest="nofaking", default=False, - help="avoid faking of modules and objects completely."), - Option('--allpypy', action="store_true",dest="allpypy", default=False, - help="run everything possible on top of PyPy."), - Option('--usemodules', action="callback", type="string", metavar="NAME", - callback=usemodules_callback, default=[], - help="(mixed) modules to use."), - Option('--compiler', action="store", type="string", dest="compiler", - metavar="[ast|cpython]", default='ast', - help="""select compiling approach. see pypy/doc/README.compiling"""), - Option('--view', action="store_true", dest="view", default=False, - help="view translation tests' flow graphs with Pygame"), - Option('--gc', action="store", default=None, - type="choice", dest="gcpolicy", - choices=['ref', 'boehm', 'none', 'framework', 'exact_boehm'], - help="GcPolicy class to use for genc tests"), - Option('-A', '--runappdirect', action="store_true", - default=False, dest="runappdirect", - help="run applevel tests directly on python interpreter (not through PyPy)"), - ) +_config = make_config(None) +_opts = to_optparse(_config, _config.getpaths()).option_list[1:] + [ + # XXX these options should go away too... but are not yet available in + # pypy.config.Config format yet + # XXX strange... this one I would have expected to find in the pypy_optdesc + Option('--gc', action="store", default=None, + type="choice", dest="gcpolicy", + choices=['ref', 'boehm', 'none', 'framework', 'exact_boehm'], + help="GcPolicy class to use for genc tests"), + + # XXX the options below are specific to the tests + Option('--allpypy', action="store_true",dest="allpypy", default=False, + help="run everything possible on top of PyPy."), + Option('--view', action="store_true", dest="view", default=False, + help="view translation tests' flow graphs with Pygame"), + Option('-A', '--runappdirect', action="store_true", + default=False, dest="runappdirect", + help=("run applevel tests directly on python interpreter (not " + "through PyPy)")), +] +option = py.test.Config.addoptions('pypy options', *_opts) _SPACECACHE={} def getobjspace(name=None, **kwds): From hpk at codespeak.net Sat Aug 5 09:30:49 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 5 Aug 2006 09:30:49 +0200 (CEST) Subject: [pypy-svn] r31018 - pypy/dist/pypy/doc/discussion Message-ID: <20060805073049.9AA5310078@code0.codespeak.net> Author: hpk Date: Sat Aug 5 09:30:48 2006 New Revision: 31018 Modified: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt Log: ReST fix Modified: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt (original) +++ pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt Sat Aug 5 09:30:48 2006 @@ -17,7 +17,7 @@ - remote import (which I've already implemented) - run on session level probably (like RemoteSession) -- do not run on low level (around call(*args)) [previous, +- do not run on low level (around `call(*args)`) [previous, Executor approach], because there are problems with displaying and we need everything to get every data. - We need to get data on higher level - perform remote import of py lib @@ -28,4 +28,4 @@ XXX: Remote import, how it works -Sending data as new files. Possible additions of some kind of checksums. \ No newline at end of file +Sending data as new files. Possible additions of some kind of checksums. From ale at codespeak.net Sat Aug 5 12:34:11 2006 From: ale at codespeak.net (ale at codespeak.net) Date: Sat, 5 Aug 2006 12:34:11 +0200 (CEST) Subject: [pypy-svn] r31019 - in pypy/dist/pypy/lib/pyontology: . test Message-ID: <20060805103411.25FED1007A@code0.codespeak.net> Author: ale Date: Sat Aug 5 12:34:10 2006 New Revision: 31019 Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py pypy/dist/pypy/lib/pyontology/pyontology.py pypy/dist/pypy/lib/pyontology/test/test_ontology.py Log: Bugfixes and a perfomance improvement Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/constraint_classes.py (original) +++ pypy/dist/pypy/lib/pyontology/constraint_classes.py Sat Aug 5 12:34:10 2006 @@ -2,6 +2,11 @@ ConsistencyFailure from rdflib import URIRef +import py +from pypy.tool.ansi_print import ansi_log +log = py.log.Producer("Constraint") +py.log.setconsumer("Constraint", ansi_log) + class OwlConstraint(AbstractConstraint): cost = 1 @@ -25,11 +30,6 @@ card = 0 return card -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer("Constraint") -py.log.setconsumer("Constraint", ansi_log) - class CardinalityConstraint(AbstractConstraint): cost = 10 @@ -281,13 +281,11 @@ for cls, val in obj_domain: for v in val: if not (v,cls) in sub_domain: - raise ConsistencyFailure("Inverseof failed for (%r, %r) in %r" % - (val, cls, sub_domain) ) + domains[self.variable].addValue(v, cls) for cls, val in sub_domain: for v in val: if not (val,cls) in obj_domain: - raise ConsistencyFailure("Inverseof failed for (%r, %r) in %r" % - (val, cls, obj_domain)) + domains[self.object].addValue(val, cls) class DifferentfromConstraint(SubClassConstraint): @@ -449,55 +447,61 @@ def narrow(self, domains): val = domains[self.List].getValues() - property = domains[self.variable].property - cls = domains[self.variable].getValues()[0] + dom = domains[self.variable] + property = dom.property + indi = dom.getValues() prop = Linkeddict(domains[property].getValues()) - for v in prop[cls]: - if v in val: - break - else: - raise ConsistencyFailure( - "The value of the property %s in the class %s has no values from %r" - %(property, cls, val)) - + for v in indi: + if not v in prop.keys(): + dom.removeValue(v) + else: + prop_val = prop[v] + for p in prop_val: + if p in val: + break + else: + dom.removeValue(v) + class AllValueConstraint(OneofPropertyConstraint): - + """ AllValuesfrom property restriction is used to define the class + of individuals for which the values for the property (defined + by the onProperty triple) all comes from the class description + which is the object of this triple. + The constraint shall narrow the domain of the subject class to + only contain individuals satisfying the above condition + """ cost = 100 def narrow(self, domains): val = domains[self.List].getValues() - property = domains[self.variable].property - cls = domains[self.variable].getValues()[0] + dom = domains[self.variable] + property = dom.property + indi = dom.getValues() prop = Linkeddict(domains[property].getValues()) - for v in prop[cls]: - if not v in val: - raise ConsistencyFailure( - "The value of the property %s in the class %s has a value not from %r" - %(property, cls, val)) - -class HasvalueConstraint(AbstractConstraint): + for v in indi: + if not v in prop.keys(): + dom.removeValue(v) + else: + prop_val = prop[v] + for p in prop_val: + if not p in val: + dom.removeValue(v) - def __init__(self, variable, property, value): - AbstractConstraint.__init__(self, [variable]) - self.variable = variable - self.property = property - self.value = value +class HasvalueConstraint(OneofPropertyConstraint): cost = 100 - def estimateCost(self, domains): - return self.cost - def narrow(self, domains): - """ This is to check the assertion that the class self.variable has a value of self.value - for the property """ - val = self.value - prop = domains[self.property].getValuesPrKey(self.variable) - for v in prop: - if v == val: - break - else: - raise ConsistencyFailure( - "The value of the property %s in the class %s has a value not from %r" - %(self.property, self.variable, self.value)) + val = self.List + dom = domains[self.variable] + property = dom.property + indi = dom.getValues() + prop = Linkeddict(domains[property].getValues()) + for v in indi: + if not v in prop.keys(): + dom.removeValue(v) + else: + prop_val = prop[v] + if not val in prop_val: + dom.removeValue(v) Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Sat Aug 5 12:34:10 2006 @@ -48,7 +48,7 @@ format = "n3" return format -class ClassDomain(fd, object): +class ClassDomain(AbstractDomain, object): # Class domain is intended as a (abstract/virtual) domain for implementing # Class axioms. Working on class descriptions the class domain should allow @@ -133,10 +133,13 @@ def removeValues(self, values): for val in values: - self.values.pop(val) + self.removeValue(val) + + def removeValue(self, value): + self.values.pop(value) if not self.values: - raise ConsistencyFailure - + raise ConsistencyFailure + def getBases(self): return self._bases @@ -161,6 +164,8 @@ class FixedClassDomain(ClassDomain): + finished = False + def removeValues(self, values): pass #raise ConsistencyFailure @@ -177,8 +182,8 @@ def __init__(self, name, uri=None, values=[], bases=[]): self.name = name self.uri = uri - self.sameas = [] - self.differentfrom = [] + self.sameas = set() + self.differentfrom = set() def __repr__(self): return "<%s( %s, %s)>"%(self.__class__.__name__, self.name, self.uri) @@ -187,7 +192,7 @@ return hash(self.uri) def __eq__(self, other): - log("CMP %r,%r"%(self,other)) + log("CMP %r,%r %i"%(self.name,other, len(self.differentfrom))) if ((hasattr(other,'uri') and self.uri == other.uri) or (not hasattr(other,'uri') and self.uri == other) or other in self.sameas): @@ -220,7 +225,7 @@ self.un_constraint = [] self.in_constraint = [] self.bases = [] - + self.finished = True def finish(self, var, constraints): return var, constraints @@ -257,7 +262,7 @@ self._dict[k] = [ x for x in vals if x != v] def __contains__(self, (cls, val)): - if not cls in self._dict.keys(): + if not cls in self._dict: return False vals = self._dict[cls] if val in vals: @@ -381,6 +386,7 @@ self.store = store if store != 'default': self.graph.open(py.path.local().join("db").strpath) + self.store_path = py.path.local().join("db").strpath self.variables = {} self.constraints = [] self.seen = {} @@ -404,11 +410,12 @@ def attach_fd(self): #while len(list(self.graph.triples((None,)*3))) != len(self.seen.keys()): + self.time = time.time() + self.nr_of_triples = 0 for (s, p, o) in (self.graph.triples((None,)*3)): self.consider_triple((s, p, o)) - log("%s %s" %(s,p)) log("=============================") - assert len(list(self.graph.triples((None,)*3))) == len(self.seen.keys()) +# assert len(list(self.graph.triples((None,)*3))) == len(self.seen.keys()) def finish(self): for key in list(self.variables.keys()): @@ -416,10 +423,13 @@ self.variables[key].finish(self.variables, self.constraints) def consider_triple(self,(s, p, o)): - log("Trying %r" % ((s, p, o),)) - if (s, p, o) in self.seen.keys(): + if (s, p, o) in self.seen: return - log("Doing %r" % ((s, p, o),)) + self.nr_of_triples += 1 + log("Doing triple nr %i: %r" % (self.nr_of_triples,(s, p, o))) + tim = time.time() + log.considerTriple("Triples per second %f" %(1./(tim-self.time))) + self.time = tim self.seen[(s, p, o)] = True if p.find('#') != -1: ns, func = p.split('#') @@ -646,8 +656,8 @@ self.variables[svar].setValues(res) def intersectionOf(self, s, var): - var = self.flatten_rdf_list(var) - vals = [self.make_var(ClassDomain, x) for x in self.variables[var].getValues()] + var_list = self.flatten_rdf_list(var) + vals = [self.make_var(ClassDomain, x) for x in self.variables[var_list].getValues()] res = vals[0] for l in vals[1:]: @@ -692,20 +702,10 @@ self.constraints.append(cons) def inverseOf(self, s, var): - self.resolve_predicate(s) - self.resolve_predicate(var) avar = self.make_var(Property, var) svar = self.make_var(Property, s) -# con = InverseofConstraint(svar, avar) -# self.constraints.append(con) - avals = self.variables[avar].getValues() - svals = self.variables[svar].getValues() - for pair in avals: - if not (pair[1], pair[0]) in svals: - self.variables[svar].addValue(pair[1], pair[0]) - for pair in svals: - if not (pair[1], pair[0]) in avals: - self.variables[avar].addValue(pair[1], pair[0]) + con = InverseofConstraint(svar, avar) + self.constraints.append(con) #---Property restrictions------------------------------------------------------ @@ -750,38 +750,22 @@ """ The hasValue restriction defines a class having as an extension all Individuals that have a property with the value of var. To make an assertion we need to know for which class the restriction applies""" - def Hasvalue(cls ,prop, val): - var = "%s_%s_hasvalue" %(cls, prop.name) - dom = {var : fd(prop.getValues( ))} - cons = Expression([cls, var], " %s[1] == %s and %s.cmp( %s[0])" %( var, val, cls, var)) - log("HASVALUE %r %r"%(prop.getValues(),dom)) - return dom, [cons] - - self.value_helper(s, var, Hasvalue) + sub = self.make_var(Restriction, s) + cons = HasvalueConstraint(sub, var) + self.constraints.append(cons) def allValuesFrom(self, s, var): - def allvalue(cls ,prop, val): - # This creates a temporary domain to be able to help find the classes that only has values - # from val - var = "%s_%s_allvalue" %(cls, prop.name) - dom = {var : fd([(x,tuple(y)) for (x,y) in prop.getValuesPrKey( )])} - # The condition should return true if - cons = Expression([cls, var], "%s == %s[1] and %s == %s[0]" %(val, var, cls, var)) - return dom, [cons] - self.value_helper(s, var, allvalue) + sub = self.make_var(Restriction, s) + obj = self.make_var(ClassDomain, var) + cons = AllValueConstraint(sub, obj) + self.constraints.append(cons) def someValuesFrom(self, s, var): - #Maybe just add hasvalue - def somevalue(cls ,prop, val): - # This creates a temporary domain to be able to help find the classes that only has values - # from val - var = "%s_%s_allvalue" %(cls, prop.name) - dom = {var : fd(prop.getValues( ))} - # The condition should return true if - cons = Expression([cls, var, val], " %s[1] in %s and %s == %s[0]" %(var, val, cls, var)) - return dom, [cons] - self.value_helper(s, var, somevalue) - + sub = self.make_var(Restriction, s) + obj = self.make_var(ClassDomain, var) + cons = SomeValueConstraint(sub, obj) + self.constraints.append(cons) + # ----------------- ---------------- def imports(self, s, var): Modified: pypy/dist/pypy/lib/pyontology/test/test_ontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_ontology.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_ontology.py Sat Aug 5 12:34:10 2006 @@ -313,6 +313,7 @@ O.type(sub, obj) O.variables['owner_'].setValues([('Bob_','Fiat_')]) O.inverseOf(own, owned) + O.consistency() assert ('Fiat_','Bob_') in O.variables['ownedby_'].getValues() def test_hasvalue(): From fijal at codespeak.net Sat Aug 5 13:21:00 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 5 Aug 2006 13:21:00 +0200 (CEST) Subject: [pypy-svn] r31020 - pypy/dist/pypy/doc/discussion Message-ID: <20060805112100.97B311007A@code0.codespeak.net> Author: fijal Date: Sat Aug 5 13:20:59 2006 New Revision: 31020 Modified: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt Log: Loaded full version of proposal. Modified: pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt (original) +++ pypy/dist/pypy/doc/discussion/summer-of-pypy-pytest.txt Sat Aug 5 13:20:59 2006 @@ -13,19 +13,44 @@ Method: ======= -XXX: some draft +Remote imports: +--------------- -- remote import (which I've already implemented) -- run on session level probably (like RemoteSession) -- do not run on low level (around `call(*args)`) [previous, - Executor approach], because there are problems with displaying and - we need everything to get every data. -- We need to get data on higher level - perform remote import of py lib - and run session there (ServerSession or so) to send data to client - and get info. Probably more sophisticated stuff such as PDB, or any other - input will get run by re-running test on client side (transfering every pdb - call seems hard and what if they implement readline-like approach in pdb?) +On the beggining of communication, master server sends to client +import hook code, which then can import all needed libraries. -XXX: Remote import, how it works +Libraries are uploaded server -> client if they're needed (when +__import__ is called). Possible extension is to add some kind of +checksum (md5?) and store files in some directory. -Sending data as new files. Possible additions of some kind of checksums. +Previous experiments: +--------------------- + +Previous experiments tried to run on the lowest level - when function/ +method is called. This is pretty clear (you run as few code on client +side as possible), but has got some drawbacks: + +- You must simulate *everything* and transform it to server side in + case of need of absolutely anything (tracebacks, short and long, + source code etc.) +- It's sometimes hard to catch exceptions. +- Top level code in testing module does not work at all. + +Possible approach: +------------------ + +On client side (side really running tests) run some kind of cut-down +session, which is imported by remote import at the very beginning and +after that, we run desired tests (probably by importing whole test +file which allows us to have top-level imports). + +Then we transfer output data to server as string, possibly tweaking +file names (which is quite easy). + +Delivarables: +============= + +- better use of testing machines +- cut down test time +- possible extension to run distributed code testing, by running and + controlling several distributed parts on different machines. From fijal at codespeak.net Sat Aug 5 14:23:58 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 5 Aug 2006 14:23:58 +0200 (CEST) Subject: [pypy-svn] r31021 - in pypy/dist/pypy/translator/js: . demo/jsdemo jssrc modules test Message-ID: <20060805122358.5C07510078@code0.codespeak.net> Author: fijal Date: Sat Aug 5 14:23:55 2006 New Revision: 31021 Modified: pypy/dist/pypy/translator/js/_class.py pypy/dist/pypy/translator/js/database.py pypy/dist/pypy/translator/js/demo/jsdemo/consserv.py pypy/dist/pypy/translator/js/jssrc/misc.js pypy/dist/pypy/translator/js/modules/_dom.py pypy/dist/pypy/translator/js/opcodes.py pypy/dist/pypy/translator/js/test/test_exception.py Log: Added some test. Mostly finished DOM interface. It's mostly working right now, I don't think more tweaks are needed now. Maybe later if someone needs it really, really hard. Fixed several opcode issues. Removed dead code. Modified: pypy/dist/pypy/translator/js/_class.py ============================================================================== --- pypy/dist/pypy/translator/js/_class.py (original) +++ pypy/dist/pypy/translator/js/_class.py Sat Aug 5 14:23:55 2006 @@ -71,10 +71,3 @@ def basename(self, name): return name.replace('.', '_')#[-1] - - #def _ctor(self): - # self.ilasm.begin_function('.ctor', [], 'void', False, 'specialname', 'rtspecialname', 'instance') - # self.ilasm.opcode('ldarg.0') - # self.ilasm.call('instance void %s::.ctor()' % self.get_base_class()) - # self.ilasm.opcode('ret') - # self.ilasm.end_function() Modified: pypy/dist/pypy/translator/js/database.py ============================================================================== --- pypy/dist/pypy/translator/js/database.py (original) +++ pypy/dist/pypy/translator/js/database.py Sat Aug 5 14:23:55 2006 @@ -465,6 +465,8 @@ self.db = db self.const = const self.name = self.get_name() + self.depends = set() + self.depends_on = set() def __eq__(self, other): return self.name == other.name Modified: pypy/dist/pypy/translator/js/demo/jsdemo/consserv.py ============================================================================== --- pypy/dist/pypy/translator/js/demo/jsdemo/consserv.py (original) +++ pypy/dist/pypy/translator/js/demo/jsdemo/consserv.py Sat Aug 5 14:23:55 2006 @@ -19,10 +19,6 @@ from pypy.translator.js.modules.bltns import date class ConsoleRoot(BasicExternal, Root): - _methods = { - 'run_command' : MethodDesc([('str_to_eval', 'ss'), ('callback', lambda : None)], {'aa':'aa'}) - } - @turbogears.expose(format="json") def str_to_eval(self, str_to_eval = "some_string"): return dict() @@ -37,7 +33,7 @@ @turbogears.expose(format="json") @described(retval={'aa':'aa'}) - def run_command(self, str_to_eval): + def run_command(self, str_to_eval = "aa"): # we need what has changed # we try to run it... Modified: pypy/dist/pypy/translator/js/jssrc/misc.js ============================================================================== --- pypy/dist/pypy/translator/js/jssrc/misc.js (original) +++ pypy/dist/pypy/translator/js/jssrc/misc.js Sat Aug 5 14:23:55 2006 @@ -1,6 +1,12 @@ // starts hand written code MALLOC_ZERO_FILLED = 0 +try { + log; + print = log; +} catch(e) { +} + Function.prototype.method = function (name, func) { this.prototype[name] = func; return this; @@ -26,28 +32,12 @@ return (false); } -/*function alloc_and_set(L0, len, elem) { - l = []; - for(i = 0; i < len; ++i){ - l[i] = elem; - } - return(l); -}*/ - -function strconcat(s1, s2) { - return(s1+s2); -} - -function stritem(cl, s, it) { - return(s[it]); -} - -function delitem(fn, l, i) { +/*function delitem(fn, l, i) { for(j = i; j < l.length-1; ++j) { l[j] = l[j+1]; } l.length--; -} +}*/ function strcmp(s1, s2) { if ( s1 < s2 ) { Modified: pypy/dist/pypy/translator/js/modules/_dom.py ============================================================================== --- pypy/dist/pypy/translator/js/modules/_dom.py (original) +++ pypy/dist/pypy/translator/js/modules/_dom.py Sat Aug 5 14:23:55 2006 @@ -36,6 +36,9 @@ class Window(Node): pass +class Form(Node): + pass + class Style(BasicExternal): _fields = { 'azimuth' : 'aa', @@ -256,15 +259,132 @@ 'supports' : MethodDesc(["aa", 1.0]), } -Document._methods = Element._methods +Document._methods = Element._methods.copy() Document._methods.update({ - 'getElementById' : MethodDesc(["aa"], Element()), + 'clear' : MethodDesc([]), + 'close' : MethodDesc([]), + 'createAttribute' : MethodDesc(["aa"], Element()), + 'createDocumentFragment' : MethodDesc([], Element()), 'createElement' : MethodDesc(["aa"], Element()), + 'createElementNS' : MethodDesc(["aa", "aa"], Element()), + 'createTextNode' : MethodDesc(["aa"], Element()), + 'createEvent' : MethodDesc(["aa"], Event()), + #'createRange' : MethodDesc(["aa"], Range()) - don't know what to do here + 'getElementById' : MethodDesc(["aa"], Element()), + 'getElementsByName' : MethodDesc(["aa"], [Element(), Element()]), + 'getElementsByTagName' : MethodDesc(["aa"], [Element(), Element()]), + 'importNode' : MethodDesc([Element(), True], Element()), + 'open' : MethodDesc([]), + 'write' : MethodDesc(["aa"]), + 'writeln' : MethodDesc(["aa"]), }) -Document._fields = Element._fields +Document._fields = Element._fields.copy() Document._fields.update({ + 'alinkColor' : "aa", + 'bgColor' : "aa", 'body' : Element(), + 'characterSet' : "aa", + 'cookie' : "aa", + 'contentWindow' : Window(), + 'defaultView' : Window(), + 'doctype' : "aa", + 'documentElement' : Element(), + 'domain' : "aa", + 'embeds' : [Element(), Element()], + 'fgColor' : "aa", + 'firstChild' : Element(), + 'forms' : [Form(), Form()], + 'height' : 123, + 'images' : [Element(), Element()], + 'lastModified' : "aa", + 'linkColor' : "aa", + 'links' : [Element(), Element()], + 'location' : "aa", + 'namespaceURI' : "aa", + 'referrer' : "aa", + 'styleSheets' : [Style(), Style()], + 'title' : "aa", + 'URL' : "aa", + 'vlinkColor' : "aa", + 'width' : 123, +}) + +Window._fields = Element._fields.copy() +Window._fields.update({ + 'content' : Window(), + 'closed' : True, + #'crypto' : Crypto() - not implemented in Gecko, leave alone + 'defaultStatus' : "aa", + 'document' : Document(), + # 'frameElement' : - leave alone + 'frames' : [Window(), Window()], + 'history' : ["aa", "aa"], + 'innerHeight' : 123, + 'innerWidth' : 123, + 'length' : 12, + 'location' : "aa", + 'name' : "aa", + # 'preference' : # denied in gecko + 'opener' : Window(), + 'outerHeight' : 123, + 'outerWidth' : 123, + 'pageXOffset' : 12, + 'pageYOffset' : 12, + 'parent' : Window(), + # 'personalbar' : - disallowed + # 'screen' : Screen() - not part of the standard, allow it if you want + 'screenX' : 12, + 'screenY' : 12, + 'scrollMaxX' : 12, + 'scrollMaxY' : 12, + 'scrollX' : 12, + 'scrollY' : 12, + 'self' : Window(), + 'status' : "asd", + 'top' : Window(), + 'window' : Window(), +}) + +Window._methods = Element._methods.copy() +Window._methods.update({ + 'alert' : MethodDesc(["aa"]), + 'atob' : MethodDesc(["aa"], "aa"), + 'back' : MethodDesc([]), + 'blur' : MethodDesc([]), + 'btoa' : MethodDesc(["aa"], "aa"), + 'close' : MethodDesc([]), + 'confirm' : MethodDesc(["aa"], True), + 'dump' : MethodDesc(["aa"]), + 'escape' : MethodDesc(["aa"], "aa"), + #'find' : MethodDesc(["aa"], - gecko only + 'focus' : MethodDesc([]), + 'forward' : MethodDesc([]), + 'getComputedStyle' : MethodDesc([Element(), "aa"], Style()), + 'home' : MethodDesc([]), + 'open' : MethodDesc(["aa", "aa"]), + 'onabort' : MethodDesc([Event()]), + 'onblur' : MethodDesc([Event()]), + 'onchange' : MethodDesc([Event()]), + 'onclick' : MethodDesc([MouseEvent()]), + 'onclose' : MethodDesc([MouseEvent()]), + 'ondragdrop' : MethodDesc([MouseEvent()]), + 'onerror' : MethodDesc([MouseEvent()]), + 'onfocus' : MethodDesc([Event()]), + 'onkeydown' : MethodDesc([KeyEvent()]), + 'onkeypress' : MethodDesc([KeyEvent()]), + 'onkeyup' : MethodDesc([KeyEvent()]), + 'onload' : MethodDesc([KeyEvent()]), + 'onmousedown' : MethodDesc([MouseEvent()]), + 'onmousemove' : MethodDesc([MouseEvent()]), + 'onmouseup' : MethodDesc([MouseEvent()]), + 'onmouseover' : MethodDesc([MouseEvent()]), + 'onmouseup' : MethodDesc([MouseEvent()]), + 'onresize' : MethodDesc([MouseEvent()]), + 'onscroll' : MethodDesc([MouseEvent()]), + 'onselect' : MethodDesc([MouseEvent()]), + 'onsubmit' : MethodDesc([MouseEvent()]), + 'onunload' : MethodDesc([Event()]), }) def get_document(): Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Sat Aug 5 14:23:55 2006 @@ -104,6 +104,7 @@ 'int_is_true': [PushAllArgs,_Prefix('!!')], 'uint_is_true': [PushAllArgs,_Prefix('!!')], 'float_is_true': [PushAllArgs,_Prefix('!!')], + 'is_true': [PushAllArgs,_Prefix('!!')], 'direct_call' : [_CallDispatcher(Builtins, class_map)], 'indirect_call' : [IndirectCall], Modified: pypy/dist/pypy/translator/js/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_exception.py (original) +++ pypy/dist/pypy/translator/js/test/test_exception.py Sat Aug 5 14:23:55 2006 @@ -278,7 +278,10 @@ pass def function_raise1(i): - function_raise2(i) + try: + function_raise2(i) + except Exception, e: + return str(e) fn = compile_function(function_raise1, [int]) - fn() + fn(3) From rhymes at codespeak.net Sat Aug 5 17:59:32 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 17:59:32 +0200 (CEST) Subject: [pypy-svn] r31022 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805155932.41B6F10078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 17:59:27 2006 New Revision: 31022 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: readline() works Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 17:59:27 2006 @@ -1,457 +1,570 @@ -from pypy.rpython.rctypes.tool import ctypes_platform -from pypy.rpython.rctypes.tool.libc import libc -import pypy.rpython.rctypes.implementation # this defines rctypes magic -from pypy.rpython.rctypes.aerrno import geterrno -from pypy.interpreter.error import OperationError -from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app -from ctypes import * -import ctypes.util - -from bzlib import bz_stream, BZFILE, FILE -from fileobject import PyFileObject - -libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) - -c_void = None - -class CConfig: - _header_ = """ - #include - #include - #include - """ - off_t = ctypes_platform.SimpleType("off_t", c_longlong) - size_t = ctypes_platform.SimpleType("size_t", c_ulong) - BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") - SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET") - -constants = {} -constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', - 'BZ_RUN_OK', 'BZ_FLUSH_OK', 'BZ_FINISH_OK', 'BZ_STREAM_END', - 'BZ_SEQUENCE_ERROR', 'BZ_PARAM_ERROR', 'BZ_MEM_ERROR', 'BZ_DATA_ERROR', - 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', - 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] -for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) - -class cConfig: - pass -cConfig.__dict__.update(ctypes_platform.configure(CConfig)) - -for name in constant_names: - value = getattr(cConfig, name) - if value is not None: - constants[name] = value -locals().update(constants) - -off_t = cConfig.off_t -BUFSIZ = cConfig.BUFSIZ -SEEK_SET = cConfig.SEEK_SET -BZ_OK = cConfig.BZ_OK -BZ_STREAM_END = cConfig.BZ_STREAM_END -BZ_CONFIG_ERROR = cConfig.BZ_CONFIG_ERROR -BZ_PARAM_ERROR = cConfig.BZ_PARAM_ERROR -BZ_DATA_ERROR = cConfig.BZ_DATA_ERROR -BZ_DATA_ERROR_MAGIC = cConfig.BZ_DATA_ERROR_MAGIC -BZ_IO_ERROR = cConfig.BZ_IO_ERROR -BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR -BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF -BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR - -# modes -MODE_CLOSED = 0 -MODE_READ = 1 -MODE_READ_EOF = 2 -MODE_WRITE = 3 - -# bits in f_newlinetypes -NEWLINE_UNKNOWN = 0 # No newline seen, yet -NEWLINE_CR = 1 # \r newline seen -NEWLINE_LF = 2 # \n newline seen -NEWLINE_CRLF = 4 # \r\n newline seen - -if BUFSIZ < 8192: - SMALLCHUNK = 8192 -else: - SMALLCHUNK = BUFSIZ - - -pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] -pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) -pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] -pythonapi.PyFile_SetBufSize.restype = c_void -pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] -pythonapi.PyFile_AsFile.restype = POINTER(FILE) -pythonapi.PyMem_Free.argtypes = [c_char_p] -pythonapi.PyMem_Free.restype = c_void - -libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_void_p, c_int] -libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_int] -libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) -libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] -libbz2.BZ2_bzReadClose.restype = c_void -libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), - c_int, POINTER(c_uint), POINTER(c_uint)] -libbz2.BZ2_bzWriteClose.restype = c_void -libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] -libbz2.BZ2_bzRead.restype = c_int - -libc.strerror.restype = c_char_p -libc.strerror.argtypes = [c_int] -libc.fclose.argtypes = [POINTER(FILE)] -libc.fclose.restype = c_int -libc.fseek.argtypes = [POINTER(FILE), c_int, c_int] -libc.fseek.restype = c_int - -def _get_error_msg(): - errno = geterrno() - return libc.strerror(errno) - -def _catch_bz2_error(space, bzerror): - if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: - raise OperationError(space.w_SystemError, - space.wrap("the bz2 library was not compiled correctly")) - if bzerror == BZ_PARAM_ERROR: - raise OperationError(space.w_SystemError, - space.wrap("the bz2 library has received wrong parameters")) - elif bzerror == BZ_MEM_ERROR: - raise OperationError(space.w_MemoryError, space.wrap("")) - elif bzerror in (BZ_DATA_ERROR, BZ_DATA_ERROR_MAGIC): - raise OperationError(space.w_IOError, space.wrap("invalid data stream")) - elif bzerror == BZ_IO_ERROR: - raise OperationError(space.w_IOError, space.wrap("unknown IO error")) - elif bzerror == BZ_UNEXPECTED_EOF: - raise OperationError(space.w_EOFError, - space.wrap( - "compressed file ended before the logical end-of-stream was detected")) - elif bzerror == BZ_SEQUENCE_ERROR: - raise OperationError(space.w_RuntimeError, - space.wrap("wrong sequence of bz2 library commands used")) - -def _drop_readahead(obj): - if obj.f_buf: - pythonapi.PyMem_Free(obj.f_buf) - obj.f_buf = c_char_p() - -def _univ_newline_read(bzerror, stream, buf, n, obj): - dst = buf - - if not obj.f_univ_newline: - return libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) - - newlinetypes = obj.f_newlinetypes - skipnextlf = obj.f_skipnextlf - - while n: - src = dst - - nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) - n -= nread # assuming 1 byte out for each in; will adjust - shortread = n != 0 # True iff EOF or error - - # needed to operate with "pointers" - src_lst = list(src.value) - src_pos = 0 - dst_lst = list(dst.value) - dst_pos = 0 - while nread: - nread -= 1 - - c = src_lst[src_pos] - src_pos += 1 - - if c == '\r': - # save as LF and set flag to skip next LF. - dst_lst[dst_pos] = '\n' - dst_pos += 1 - skipnextlf = True - elif skipnextlf and c == '\n': - # skip LF, and remember we saw CR LF. - skipnextlf = False - newlinetypes |= NEWLINE_CRLF - n += 1 - else: - # normal char to be stored in buffer. Also - # update the newlinetypes flag if either this - # is an LF or the previous char was a CR. - if c == '\n': - newlinetypes |= NEWLINE_LF - elif skipnextlf: - newlinetypes |= NEWLINE_CR - - dst_lst[dst_pos] = c - dst_pos += 1 - - skipnextlf = False - - if shortread: - # if this is EOF, update type flags. - if skipnextlf and (bzerror == BZ_STREAM_END): - newlinetypes |= NEWLINE_CR - break - - obj.f_newlinetypes = newlinetypes - obj.f_skipnextlf = skipnextlf - - buf = c_char_p("".join(dst_lst)) - - return dst_pos - - -class _BZ2File(Wrappable): - def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): - self.space = space - - self.f_buf = c_char_p() # allocated readahead buffer - self.f_bufend = c_char_p() # points after last occupied position - self.f_bufptr = c_char_p() # current buffer position - - self.f_softspace = 0 # flag used by print command - - self.f_univ_newline = False # handle any newline convention - self.f_newlinetypes = 0 # types of newlines seen - self.f_skipnextlf = 0 # skip next \n - - self.mode = 0 - self.pos = 0 - self.size = 0 - - self._init_bz2file(filename, mode, buffering, compresslevel) - - def _init_bz2file(self, filename, mode_, buffering, compresslevel): - self.size = -1 - - name = filename - mode_char = "" - mode_list = mode_ - - if compresslevel < 1 or compresslevel > 9: - raise OperationError(self.space.w_ValueError, - self.space.wrap("compresslevel must be between 1 and 9")) - - for mode in mode_list: - error = False - - if mode in ['r', 'w']: - if mode_char: - error = True - mode_char = mode - elif mode == 'b': - pass - elif mode == 'U': - self.f_univ_newline = True - else: - error = True - - if error: - raise OperationError(self.space.w_ValueError, - self.space.wrap("invalid mode char %s" % mode)) - - if mode_char == 0: - mode_char = 'r' - mode = ('wb', 'rb')[mode_char == 'r'] - - # open the file and set the buffer - f = pythonapi.PyFile_FromString(name, mode) - if not f: - raise OperationError(self.space.w_IOError, - self.space.wrap("cannot open file %s" % name)) - pythonapi.PyFile_SetBufSize(f, buffering) - - # store the FILE object - self._file = pythonapi.PyFile_AsFile(f) - - bzerror = c_int() - if mode_char == 'r': - self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, - 0, 0, None, 0) - else: - self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, - compresslevel, 0, 0) - - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) - - self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] - - def __del__(self): - bzerror = c_int() - - if self.mode in (MODE_READ, MODE_READ_EOF): - libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - elif self.mode == MODE_WRITE: - libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) - - _drop_readahead(self) - - def _check_if_close(self): - if self.mode == MODE_CLOSED: - raise OperationError(self.space.w_ValueError, - self.space.wrap("I/O operation on closed file")) - - def close(self): - """close() -> None or (perhaps) an integer - - Close the file. Sets data attribute .closed to true. A closed file - cannot be used for further I/O operations.""" - - # this feature is not supported due to fclose(): - # close() may be called more than once without error. - - bzerror = c_int(BZ_OK) - - if self.mode in (MODE_READ, MODE_READ_EOF): - libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - elif self.mode == MODE_WRITE: - libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) - - self.mode = MODE_CLOSED - - # close the underline file - ret = libc.fclose(self._file) - if ret != 0: - raise OperationError(self.space.w_IOError, - self.space.wrap(_get_error_msg())) - - if bzerror != BZ_OK: - return _catch_bz2_error(self.space, bzerror) - - return ret - close.unwrap_spec = ['self'] - - def tell(self): - """tell() -> int - - Return the current file position, an integer (may be a long integer).""" - - self._check_if_close() - - return self.space.wrap(self.pos) - tell.unwrap_spec = ['self'] - - def seek(self, offset, whence=0): - """"seek(offset [, whence]) -> None - - Move to new file position. Argument offset is a byte count. Optional - argument whence defaults to 0 (offset from start of file, offset - should be >= 0); other values are 1 (move relative to current position, - positive or negative), and 2 (move relative to end of file, usually - negative, although many platforms allow seeking beyond the end of a file). - - Note that seeking of bz2 files is emulated, and depending on the parameters - the operation may be extremely slow.""" - - _drop_readahead(self) - self._check_if_close() - - buf = c_char_p() - bufsize = SMALLCHUNK - bytesread = 0 - bzerror = c_int() - - if self.mode not in (MODE_READ, MODE_READ_EOF): - raise OperationError(self.space.w_IOError, - self.space.wrap("seek works only while reading")) - - if whence == 2: - if self.size == -1: - while True: - chunksize = _univ_newline_read(bzerror, self.fp, buf, - bufsize, self) - self.pos += chunksize - bytesread += chunksize - - if bzerror == BZ_STREAM_END: - break - elif bzerror != BZ_OK: - _catch_bz2_error(bzerror) - - self.mode = MODE_READ_EOF - self.size = self.pos - bytesread = 0 - offset += self.size - elif whence == 1: - offset += self.pos - - # Before getting here, offset must be the absolute position the file - # pointer should be set to. - if offset >= self.pos: - # we can move forward - offset -= self.pos - else: - # we cannot move back, so rewind the stream - libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - if bzerror != BZ_OK: - _catch_bz2_error(bzerror) - - ret = libc.fseek(self._file, 0, SEEK_SET) - if ret != 0: - raise OperationError(self.space.w_IOError, - self.space.wrap(_get_error_msg())) - - self.pos = 0 - self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, - 0, 0, None, 0) - if bzerror != BZ_OK: - _catch_bz2_error(bzerror) - - self.mode = MODE_READ - - if offset <= 0 or self.mode == MODE_READ_EOF: - return - - # Before getting here, offset must be set to the number of bytes - # to walk forward. - while True: - if (offset - bytesread) > bufsize: - readsize = bufsize - else: - # offset might be wider that readsize, but the result - # of the subtraction is bound by buffersize (see the - # condition above). bufsize is 8192. - readsize = offset - bytesread - - chunksize = _univ_newline_read(bzerror, self.fp, buf, readsize, self) - self.pos += chunksize - bytesread += chunksize - - if bzerror == BZ_STREAM_END: - self.size = self.pos - self.mode = MODE_READ_EOF - elif bzerror != BZ_OK: - _catch_bz2_error(bzerror) - - if bytesread == offset: - break - seek.unwrap_spec = ['self', int, int] - - -_BZ2File.typedef = TypeDef("_BZ2File", - close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), - tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), - seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), -) - -def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): - """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object - - Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or - writing. When opened for writing, the file will be created if it doesn't - exist, and truncated otherwise. If the buffering argument is given, 0 means - unbuffered, and larger numbers specify the buffer size. If compresslevel - is given, must be a number between 1 and 9. - - Add a 'U' to mode to open the file for input with universal newline - support. Any line ending in the input file will be seen as a '\\n' in - Python. Also, a file so opened gains the attribute 'newlines'; the value - for this attribute is one of None (no newline read yet), '\\r', '\\n', - '\\r\\n' or a tuple containing all the newline types seen. Universal - newlines are available only when reading.""" - return _BZ2File(space, filename, mode, buffering, compresslevel) -BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] - +from pypy.rpython.rctypes.tool import ctypes_platform +from pypy.rpython.rctypes.tool.libc import libc +import pypy.rpython.rctypes.implementation # this defines rctypes magic +from pypy.rpython.rctypes.aerrno import geterrno +from pypy.interpreter.error import OperationError +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from ctypes import * +import ctypes.util +import sys + +from bzlib import bz_stream, BZFILE, FILE +from fileobject import PyFileObject + +libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2")) + +c_void = None + +class CConfig: + _header_ = """ + #include + #include + #include + """ + off_t = ctypes_platform.SimpleType("off_t", c_longlong) + size_t = ctypes_platform.SimpleType("size_t", c_ulong) + BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") + SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET") + +constants = {} +constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK', + 'BZ_RUN_OK', 'BZ_FLUSH_OK', 'BZ_FINISH_OK', 'BZ_STREAM_END', + 'BZ_SEQUENCE_ERROR', 'BZ_PARAM_ERROR', 'BZ_MEM_ERROR', 'BZ_DATA_ERROR', + 'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF', + 'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR'] +for name in constant_names: + setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) + +class cConfig: + pass +cConfig.__dict__.update(ctypes_platform.configure(CConfig)) + +for name in constant_names: + value = getattr(cConfig, name) + if value is not None: + constants[name] = value +locals().update(constants) + +off_t = cConfig.off_t +BUFSIZ = cConfig.BUFSIZ +SEEK_SET = cConfig.SEEK_SET +BZ_OK = cConfig.BZ_OK +BZ_STREAM_END = cConfig.BZ_STREAM_END +BZ_CONFIG_ERROR = cConfig.BZ_CONFIG_ERROR +BZ_PARAM_ERROR = cConfig.BZ_PARAM_ERROR +BZ_DATA_ERROR = cConfig.BZ_DATA_ERROR +BZ_DATA_ERROR_MAGIC = cConfig.BZ_DATA_ERROR_MAGIC +BZ_IO_ERROR = cConfig.BZ_IO_ERROR +BZ_MEM_ERROR = cConfig.BZ_MEM_ERROR +BZ_UNEXPECTED_EOF = cConfig.BZ_UNEXPECTED_EOF +BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR + +# modes +MODE_CLOSED = 0 +MODE_READ = 1 +MODE_READ_EOF = 2 +MODE_WRITE = 3 + +# bits in f_newlinetypes +NEWLINE_UNKNOWN = 0 # No newline seen, yet +NEWLINE_CR = 1 # \r newline seen +NEWLINE_LF = 2 # \n newline seen +NEWLINE_CRLF = 4 # \r\n newline seen + +if BUFSIZ < 8192: + SMALLCHUNK = 8192 +else: + SMALLCHUNK = BUFSIZ + +MAXINT = sys.maxint + +pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] +pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) +pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] +pythonapi.PyFile_SetBufSize.restype = c_void +pythonapi.PyFile_AsFile.argtypes = [POINTER(PyFileObject)] +pythonapi.PyFile_AsFile.restype = POINTER(FILE) +pythonapi.PyMem_Free.argtypes = [c_char_p] +pythonapi.PyMem_Free.restype = c_void + +libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_void_p, c_int] +libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, + c_int, c_int] +libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE) +libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)] +libbz2.BZ2_bzReadClose.restype = c_void +libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), + c_int, POINTER(c_uint), POINTER(c_uint)] +libbz2.BZ2_bzWriteClose.restype = c_void +libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] +libbz2.BZ2_bzRead.restype = c_int + +libc.strerror.restype = c_char_p +libc.strerror.argtypes = [c_int] +libc.fclose.argtypes = [POINTER(FILE)] +libc.fclose.restype = c_int +libc.fseek.argtypes = [POINTER(FILE), c_int, c_int] +libc.fseek.restype = c_int + +def _get_error_msg(): + errno = geterrno() + return libc.strerror(errno) + +def _catch_bz2_error(space, bzerror): + if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library was not compiled correctly")) + if bzerror == BZ_PARAM_ERROR: + raise OperationError(space.w_SystemError, + space.wrap("the bz2 library has received wrong parameters")) + elif bzerror == BZ_MEM_ERROR: + raise OperationError(space.w_MemoryError, space.wrap("")) + elif bzerror in (BZ_DATA_ERROR, BZ_DATA_ERROR_MAGIC): + raise OperationError(space.w_IOError, space.wrap("invalid data stream")) + elif bzerror == BZ_IO_ERROR: + raise OperationError(space.w_IOError, space.wrap("unknown IO error")) + elif bzerror == BZ_UNEXPECTED_EOF: + raise OperationError(space.w_EOFError, + space.wrap( + "compressed file ended before the logical end-of-stream was detected")) + elif bzerror == BZ_SEQUENCE_ERROR: + raise OperationError(space.w_RuntimeError, + space.wrap("wrong sequence of bz2 library commands used")) + +def _drop_readahead(obj): + if obj.f_buf: + pythonapi.PyMem_Free(obj.f_buf) + obj.f_buf = c_char_p() + +def _univ_newline_read(bzerror, stream, buf, n, obj): + dst = buf + + if not obj.f_univ_newline: + return libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + + newlinetypes = obj.f_newlinetypes + skipnextlf = obj.f_skipnextlf + + while n: + src = dst + + nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + n -= nread # assuming 1 byte out for each in; will adjust + shortread = n != 0 # True iff EOF or error + + # needed to operate with "pointers" + src_lst = list(src.value) + src_pos = 0 + dst_lst = list(dst.value) + dst_pos = 0 + while nread: + nread -= 1 + + c = src_lst[src_pos] + src_pos += 1 + + if c == '\r': + # save as LF and set flag to skip next LF. + dst_lst[dst_pos] = '\n' + dst_pos += 1 + skipnextlf = True + elif skipnextlf and c == '\n': + # skip LF, and remember we saw CR LF. + skipnextlf = False + newlinetypes |= NEWLINE_CRLF + n += 1 + else: + # normal char to be stored in buffer. Also + # update the newlinetypes flag if either this + # is an LF or the previous char was a CR. + if c == '\n': + newlinetypes |= NEWLINE_LF + elif skipnextlf: + newlinetypes |= NEWLINE_CR + + dst_lst[dst_pos] = c + dst_pos += 1 + + skipnextlf = False + + if shortread: + # if this is EOF, update type flags. + if skipnextlf and (bzerror == BZ_STREAM_END): + newlinetypes |= NEWLINE_CR + break + + obj.f_newlinetypes = newlinetypes + obj.f_skipnextlf = skipnextlf + + buf = c_char_p("".join(dst_lst)) + + return dst_pos + +def _getline(space, obj, size): + used_v_size = 0 # no. used slots in buffer + increment = 0 # amount to increment the buffer + bzerror = c_int() + + newlinetypes = obj.f_newlinetypes + skipnextlf = obj.f_skipnextlf + univ_newline = obj.f_univ_newline + + total_v_size = (100, size)[size > 0] # total no. of slots in buffer + buf_lst = [] + buf_pos = 0 + + end_pos = buf_pos + total_v_size + + ch = c_char() + while True: + if univ_newline: + while True: + libbz2.BZ2_bzRead(byref(bzerror), obj.fp, byref(ch), 1) + obj.pos += 1 + if bzerror != BZ_OK or buf_pos == end_pos: + break + + if skipnextlf: + skipnextlf = False + if ch.value == '\n': + # Seeing a \n here with + # skipnextlf true means we saw a \r before. + newlinetypes |= NEWLINE_CRLF + libbz2.BZ2_bzRead(byref(bzerror), obj.fp, byref(ch), 1) + if bzerror != BZ_OK: break + else: + newlinetypes |= NEWLINE_CR + + if ch.value == '\r': + skipnextlf = True + ch.value = '\n' + elif ch.value == '\n': + newlinetypes |= NEWLINE_LF + buf_lst.append(ch.value) + buf_pos += 1 + + if ch.value == '\n': break + if bzerror == BZ_STREAM_END and skipnextlf: + newlinetypes |= NEWLINE_CR + else: # if not universal newlines use the normal loop + while True: + libbz2.BZ2_bzRead(byref(bzerror), obj.fp, byref(ch), 1) + obj.pos += 1 + buf_lst.append(ch.value) + buf_pos += 1 + + if not (bzerror == BZ_OK and ch.value != '\n' and buf_pos != end_pos): + break + + obj.f_newlinetypes = newlinetypes + obj.f_skipnextlf = skipnextlf + + if bzerror.value == BZ_STREAM_END: + obj.size = obj.pos + obj.mode = MODE_READ_EOF + break + elif bzerror.value != BZ_OK: + _catch_bz2_error(space, bzerror) + + if ch.value == '\n': break + # must be because buf_pos == end_pos + if size > 0: + break + + used_v_size = total_v_size + increment = total_v_size >> 2 # mild exponential growth + total_v_size += increment + + if total_v_size > MAXINT: + raise OperationError(space.w_OverflowError, + space.wrap("line is longer than a Python string can hold")) + + buf_pos += used_v_size + end_pos += total_v_size + + used_v_size = buf_pos + if used_v_size != total_v_size: + return "".join(buf_lst[:used_v_size]) + return "".join(buf_lst) + +class _BZ2File(Wrappable): + def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): + self.space = space + + self.f_buf = c_char_p() # allocated readahead buffer + self.f_bufend = c_char_p() # points after last occupied position + self.f_bufptr = c_char_p() # current buffer position + + self.f_softspace = 0 # flag used by print command + + self.f_univ_newline = False # handle any newline convention + self.f_newlinetypes = 0 # types of newlines seen + self.f_skipnextlf = 0 # skip next \n + + self.mode = 0 + self.pos = 0 + self.size = 0 + + self._init_bz2file(filename, mode, buffering, compresslevel) + + def _init_bz2file(self, filename, mode_, buffering, compresslevel): + self.size = -1 + + name = filename + mode_char = "" + mode_list = mode_ + + if compresslevel < 1 or compresslevel > 9: + raise OperationError(self.space.w_ValueError, + self.space.wrap("compresslevel must be between 1 and 9")) + + for mode in mode_list: + error = False + + if mode in ['r', 'w']: + if mode_char: + error = True + mode_char = mode + elif mode == 'b': + pass + elif mode == 'U': + self.f_univ_newline = True + else: + error = True + + if error: + raise OperationError(self.space.w_ValueError, + self.space.wrap("invalid mode char %s" % mode)) + + if mode_char == 0: + mode_char = 'r' + mode = ('wb', 'rb')[mode_char == 'r'] + + # open the file and set the buffer + f = pythonapi.PyFile_FromString(name, mode) + if not f: + raise OperationError(self.space.w_IOError, + self.space.wrap("cannot open file %s" % name)) + pythonapi.PyFile_SetBufSize(f, buffering) + + # store the FILE object + self._file = pythonapi.PyFile_AsFile(f) + + bzerror = c_int() + if mode_char == 'r': + self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, + 0, 0, None, 0) + else: + self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, + compresslevel, 0, 0) + + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] + + def __del__(self): + bzerror = c_int() + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + _drop_readahead(self) + + def _check_if_close(self): + if self.mode == MODE_CLOSED: + raise OperationError(self.space.w_ValueError, + self.space.wrap("I/O operation on closed file")) + + def close(self): + """close() -> None or (perhaps) an integer + + Close the file. Sets data attribute .closed to true. A closed file + cannot be used for further I/O operations.""" + + # this feature is not supported due to fclose(): + # close() may be called more than once without error. + + bzerror = c_int(BZ_OK) + + if self.mode in (MODE_READ, MODE_READ_EOF): + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + elif self.mode == MODE_WRITE: + libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) + + self.mode = MODE_CLOSED + + # close the underline file + ret = libc.fclose(self._file) + if ret != 0: + raise OperationError(self.space.w_IOError, + self.space.wrap(_get_error_msg())) + + if bzerror != BZ_OK: + return _catch_bz2_error(self.space, bzerror) + + return ret + close.unwrap_spec = ['self'] + + def tell(self): + """tell() -> int + + Return the current file position, an integer (may be a long integer).""" + + self._check_if_close() + + return self.space.wrap(self.pos) + tell.unwrap_spec = ['self'] + + def seek(self, offset, whence=0): + """"seek(offset [, whence]) -> None + + Move to new file position. Argument offset is a byte count. Optional + argument whence defaults to 0 (offset from start of file, offset + should be >= 0); other values are 1 (move relative to current position, + positive or negative), and 2 (move relative to end of file, usually + negative, although many platforms allow seeking beyond the end of a file). + + Note that seeking of bz2 files is emulated, and depending on the parameters + the operation may be extremely slow.""" + + _drop_readahead(self) + self._check_if_close() + + buf = c_char_p() + bufsize = SMALLCHUNK + bytesread = 0 + bzerror = c_int() + + if self.mode not in (MODE_READ, MODE_READ_EOF): + raise OperationError(self.space.w_IOError, + self.space.wrap("seek works only while reading")) + + if whence == 2: + if self.size == -1: + while True: + chunksize = _univ_newline_read(bzerror, self.fp, buf, + bufsize, self) + self.pos += chunksize + bytesread += chunksize + + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.mode = MODE_READ_EOF + self.size = self.pos + bytesread = 0 + offset += self.size + elif whence == 1: + offset += self.pos + + # Before getting here, offset must be the absolute position the file + # pointer should be set to. + if offset >= self.pos: + # we can move forward + offset -= self.pos + else: + # we cannot move back, so rewind the stream + libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + ret = libc.fseek(self._file, 0, SEEK_SET) + if ret != 0: + raise OperationError(self.space.w_IOError, + self.space.wrap(_get_error_msg())) + + self.pos = 0 + self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, + 0, 0, None, 0) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.mode = MODE_READ + + if offset <= 0 or self.mode == MODE_READ_EOF: + return + + # Before getting here, offset must be set to the number of bytes + # to walk forward. + while True: + if (offset - bytesread) > bufsize: + readsize = bufsize + else: + # offset might be wider that readsize, but the result + # of the subtraction is bound by buffersize (see the + # condition above). bufsize is 8192. + readsize = offset - bytesread + + chunksize = _univ_newline_read(bzerror, self.fp, buf, readsize, self) + self.pos += chunksize + bytesread += chunksize + + if bzerror == BZ_STREAM_END: + self.size = self.pos + self.mode = MODE_READ_EOF + elif bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + if bytesread == offset: + break + seek.unwrap_spec = ['self', int, int] + + def readline(self, size=-1): + """readline([size]) -> string + + Return the next line from the file, as a string, retaining newline. + A non-negative size argument will limit the maximum number of bytes to + return (an incomplete line may be returned then). Return an empty + string at EOF.""" + + self._check_if_close() + + if self.mode == MODE_READ_EOF: + return self.space.wrap("") + elif not self.mode == MODE_READ: + raise OperationError(self.space.w_IOError, + self.space.wrap("file is not ready for reading")) + + if size == 0: + return self.space.wrap("") + else: + size = (size, 0)[size < 0] + return self.space.wrap(_getline(self.space, self, size)) + readline.unwrap_spec = ['self', int] + + +_BZ2File.typedef = TypeDef("_BZ2File", + close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), + tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), + seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), + readline = interp2app(_BZ2File.readline, + unwrap_spec=_BZ2File.readline.unwrap_spec) +) + +def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): + """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object + + Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or + writing. When opened for writing, the file will be created if it doesn't + exist, and truncated otherwise. If the buffering argument is given, 0 means + unbuffered, and larger numbers specify the buffer size. If compresslevel + is given, must be a number between 1 and 9. + + Add a 'U' to mode to open the file for input with universal newline + support. Any line ending in the input file will be seen as a '\\n' in + Python. Also, a file so opened gains the attribute 'newlines'; the value + for this attribute is one of None (no newline read yet), '\\r', '\\n', + '\\r\\n' or a tuple containing all the newline types seen. Universal + newlines are available only when reading.""" + return _BZ2File(space, filename, mode, buffering, compresslevel) +BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] + Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 17:59:27 2006 @@ -9,7 +9,7 @@ if os.path.exists("foo"): os.unlink("foo") -class AppTestBz2: +class AppTestBz2File: def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space @@ -78,7 +78,31 @@ # bz2f.seek(0) # bz2f.seek(-1, 2) # assert bz2f.tell() == 5 + + def test_readline(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + from bz2 import BZ2File + from StringIO import StringIO + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + raises(TypeError, bz2f.readline, None) + sio = StringIO(TEXT) + for line in sio.readlines(): + line_read = bz2f.readline() + assert line_read == line + bz2f.close() # #!/usr/bin/python # from test import test_support @@ -164,17 +188,7 @@ # bz2f = BZ2File(self.filename) # self.assertEqual(bz2f.read(100), self.TEXT[:100]) # bz2f.close() -# -# def testReadLine(self): -# # "Test BZ2File.readline()" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# self.assertRaises(TypeError, bz2f.readline, None) -# sio = StringIO(self.TEXT) -# for line in sio.readlines(): -# self.assertEqual(bz2f.readline(), line) -# bz2f.close() -# + # def testReadLines(self): # # "Test BZ2File.readlines()" # self.createTempFile() From rhymes at codespeak.net Sat Aug 5 18:42:00 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 18:42:00 +0200 (CEST) Subject: [pypy-svn] r31023 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805164200.38D4110078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 18:41:56 2006 New Revision: 31023 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: BZ2File.read() is in place too Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 18:41:56 2006 @@ -78,6 +78,11 @@ else: SMALLCHUNK = BUFSIZ +if sizeof(c_int) > 4: + BIGCHUNK = 512 * 32 +else: + BIGCHUNK = 512 * 1024 + MAXINT = sys.maxint pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] @@ -225,7 +230,7 @@ while True: libbz2.BZ2_bzRead(byref(bzerror), obj.fp, byref(ch), 1) obj.pos += 1 - if bzerror != BZ_OK or buf_pos == end_pos: + if bzerror.value != BZ_OK or buf_pos == end_pos: break if skipnextlf: @@ -235,7 +240,7 @@ # skipnextlf true means we saw a \r before. newlinetypes |= NEWLINE_CRLF libbz2.BZ2_bzRead(byref(bzerror), obj.fp, byref(ch), 1) - if bzerror != BZ_OK: break + if bzerror.value != BZ_OK: break else: newlinetypes |= NEWLINE_CR @@ -248,7 +253,7 @@ buf_pos += 1 if ch.value == '\n': break - if bzerror == BZ_STREAM_END and skipnextlf: + if bzerror.value == BZ_STREAM_END and skipnextlf: newlinetypes |= NEWLINE_CR else: # if not universal newlines use the normal loop while True: @@ -268,7 +273,7 @@ obj.mode = MODE_READ_EOF break elif bzerror.value != BZ_OK: - _catch_bz2_error(space, bzerror) + _catch_bz2_error(space, bzerror.value) if ch.value == '\n': break # must be because buf_pos == end_pos @@ -289,7 +294,17 @@ used_v_size = buf_pos if used_v_size != total_v_size: return "".join(buf_lst[:used_v_size]) - return "".join(buf_lst) + return "".join(buf_lst) + +def _new_buffer_size(current_size): + if current_size > SMALLCHUNK: + # keep doubling until we reach BIGCHUNK + # then keep adding BIGCHUNK + if current_size <= BIGCHUNK: + return current_size + current_size + else: + return current_size + BIGCHUNK + return current_size + SMALLCHUNK class _BZ2File(Wrappable): def __init__(self, space, filename, mode='r', buffering=-1, compresslevel=9): @@ -362,8 +377,8 @@ self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, compresslevel, 0, 0) - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + if bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] @@ -406,8 +421,8 @@ raise OperationError(self.space.w_IOError, self.space.wrap(_get_error_msg())) - if bzerror != BZ_OK: - return _catch_bz2_error(self.space, bzerror) + if bzerror.value != BZ_OK: + return _catch_bz2_error(self.space, bzerror.value) return ret close.unwrap_spec = ['self'] @@ -454,10 +469,10 @@ self.pos += chunksize bytesread += chunksize - if bzerror == BZ_STREAM_END: + if bzerror.value == BZ_STREAM_END: break - elif bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + elif bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) self.mode = MODE_READ_EOF self.size = self.pos @@ -474,8 +489,8 @@ else: # we cannot move back, so rewind the stream libbz2.BZ2_bzReadClose(byref(bzerror), self.fp) - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + if bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) ret = libc.fseek(self._file, 0, SEEK_SET) if ret != 0: @@ -485,8 +500,8 @@ self.pos = 0 self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, 0, 0, None, 0) - if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + if bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) self.mode = MODE_READ @@ -508,11 +523,11 @@ self.pos += chunksize bytesread += chunksize - if bzerror == BZ_STREAM_END: + if bzerror.value == BZ_STREAM_END: self.size = self.pos self.mode = MODE_READ_EOF - elif bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + elif bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) if bytesread == offset: break @@ -540,14 +555,63 @@ size = (size, 0)[size < 0] return self.space.wrap(_getline(self.space, self, size)) readline.unwrap_spec = ['self', int] - + + def read(self, size=-1): + """read([size]) -> string + + Read at most size uncompressed bytes, returned as a string. If the size + argument is negative or omitted, read until EOF is reached.""" + + self._check_if_close() + + if self.mode == MODE_READ_EOF: + return self.space.wrap("") + elif not self.mode == MODE_READ: + raise OperationError(self.space.w_IOError, + self.space.wrap("file is not ready for reading")) + + bufsize = (size, _new_buffer_size(0))[size < 0] + + if bufsize > MAXINT: + raise OperationError(self.space.w_OverflowError, + self.space.wrap( + "requested number of bytes is more than a Python string can hold")) + + bytesread = 0 + buf = create_string_buffer(bufsize) + bzerror = c_int() + while True: + chunksize = _univ_newline_read(bzerror, self.fp, buf, + bufsize - bytesread, self) + self.pos += chunksize + bytesread += chunksize + + if bzerror.value == BZ_STREAM_END: + self.size = self.pos + self.mode = MODE_READ_EOF + break + elif bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror.value) + + if size < 0: + bufsize = _new_buffer_size(bufsize) + else: + break + + buf_lst = list(buf.value) + if bytesread != bufsize: + return self.space.wrap("".join(buf_lst[:bytesread])) + return self.space.wrap("".join(buf_lst)) + read.unwrap_spec = ['self', int] + _BZ2File.typedef = TypeDef("_BZ2File", close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), readline = interp2app(_BZ2File.readline, - unwrap_spec=_BZ2File.readline.unwrap_spec) + unwrap_spec=_BZ2File.readline.unwrap_spec), + read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec) ) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 18:41:56 2006 @@ -103,6 +103,29 @@ line_read = bz2f.readline() assert line_read == line bz2f.close() + + def test_read(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + raises(TypeError, bz2f.read, None) + text_read = bz2f.read() + assert text_read == TEXT + bz2f.close() + # #!/usr/bin/python # from test import test_support @@ -161,14 +184,7 @@ # f.write(data) # f.close() # -# def testRead(self): -# # "Test BZ2File.read()" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# self.assertRaises(TypeError, bz2f.read, None) -# self.assertEqual(bz2f.read(), self.TEXT) -# bz2f.close() -# + # def testReadChunk10(self): # # "Test BZ2File.read() in chunks of 10 bytes" # self.createTempFile() From rhymes at codespeak.net Sat Aug 5 18:47:58 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 18:47:58 +0200 (CEST) Subject: [pypy-svn] r31024 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805164758.C04E510078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 18:47:56 2006 New Revision: 31024 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: more tests for read() Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 18:47:56 2006 @@ -125,8 +125,53 @@ text_read = bz2f.read() assert text_read == TEXT bz2f.close() - + + def test_read_chunk10(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + text_read = "" + while True: + data = bz2f.read(10) + if not data: + break + text_read = "%s%s" % (text_read, data) + assert text_read == TEXT + bz2f.close() + + def test_read_100_bytes(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + assert bz2f.read(100) == TEXT[:100] + bz2f.close() + # #!/usr/bin/python # from test import test_support # from test.test_support import TESTFN @@ -167,44 +212,7 @@ # # class BZ2FileTest(BaseTest): # "Test BZ2File type miscellaneous methods." -# -# def setUp(self): -# self.filename = TESTFN -# -# def tearDown(self): -# if os.path.isfile(self.filename): -# os.unlink(self.filename) -# -# def createTempFile(self, crlf=0): -# f = open(self.filename, "wb") -# if crlf: -# data = self.DATA_CRLF -# else: -# data = self.DATA -# f.write(data) -# f.close() -# - -# def testReadChunk10(self): -# # "Test BZ2File.read() in chunks of 10 bytes" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# text = '' -# while 1: -# str = bz2f.read(10) -# if not str: -# break -# text += str -# self.assertEqual(text, text) -# bz2f.close() -# -# def testRead100(self): -# # "Test BZ2File.read(100)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# self.assertEqual(bz2f.read(100), self.TEXT[:100]) -# bz2f.close() - +# # def testReadLines(self): # # "Test BZ2File.readlines()" # self.createTempFile() From rhymes at codespeak.net Sat Aug 5 18:56:25 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 18:56:25 +0200 (CEST) Subject: [pypy-svn] r31025 - pypy/dist/pypy/module/bz2 Message-ID: <20060805165625.3A40D10078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 18:56:21 2006 New Revision: 31025 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: small fix to make seek really work Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 18:56:21 2006 @@ -452,8 +452,8 @@ _drop_readahead(self) self._check_if_close() - buf = c_char_p() bufsize = SMALLCHUNK + buf = create_string_buffer(bufsize) bytesread = 0 bzerror = c_int() From rhymes at codespeak.net Sat Aug 5 19:07:11 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 19:07:11 +0200 (CEST) Subject: [pypy-svn] r31026 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805170711.744C910078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 19:07:08 2006 New Revision: 31026 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: break means to break out. do not forget it young padawan! Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 19:07:08 2006 @@ -526,6 +526,7 @@ if bzerror.value == BZ_STREAM_END: self.size = self.pos self.mode = MODE_READ_EOF + break elif bzerror.value != BZ_OK: _catch_bz2_error(self.space, bzerror.value) Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 19:07:08 2006 @@ -67,18 +67,120 @@ bz2f.close() bz2f = BZ2File("foo", mode='r') + raises(TypeError, bz2f.seek) raises(TypeError, bz2f.seek, "foo") raises(TypeError, bz2f.seek, 0, "foo") bz2f.seek(0) assert bz2f.tell() == 0 - # bz2f.read(1) - # bz2f.seek(1, 1) - # assert bz2f.tell() == 2 - # bz2f.seek(0) - # bz2f.seek(-1, 2) - # assert bz2f.tell() == 5 + def test_seek_forward(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.seek(150) # (150, 0) + assert bz2f.read() == TEXT[150:] + bz2f.close() + + def test_seek_backwards(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.read(500) + bz2f.seek(-150, 1) + assert bz2f.read() == TEXT[500 - 150:] + bz2f.close() + + def test_seek_backwards_from_end(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.seek(-150, 2) + assert bz2f.read() == TEXT[len(TEXT) - 150:] + bz2f.close() + + def test_seek_post_end(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.seek(150000) + assert bz2f.tell() == len(TEXT) + assert bz2f.read() == "" + bz2f.close() + +# def testSeekPostEndTwice(self): +# # "Test BZ2File.seek(150000) twice" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(150000) +# bz2f.seek(150000) +# self.assertEqual(bz2f.tell(), len(self.TEXT)) +# self.assertEqual(bz2f.read(), "") +# bz2f.close() +# +# def testSeekPreStart(self): +# # "Test BZ2File.seek(-150, 0)" +# self.createTempFile() +# bz2f = BZ2File(self.filename) +# bz2f.seek(-150) +# self.assertEqual(bz2f.tell(), 0) +# self.assertEqual(bz2f.read(), self.TEXT) +# bz2f.close() +# + + def test_readline(self): def create_temp_file(crlf=False): DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' @@ -289,61 +391,6 @@ # f = open(self.filename, 'rb') # self.assertEqual(self.decompress(f.read()), self.TEXT) # f.close() -# -# def testSeekForward(self): -# # "Test BZ2File.seek(150, 0)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# self.assertRaises(TypeError, bz2f.seek) -# bz2f.seek(150) -# self.assertEqual(bz2f.read(), self.TEXT[150:]) -# bz2f.close() -# -# def testSeekBackwards(self): -# # "Test BZ2File.seek(-150, 1)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.read(500) -# bz2f.seek(-150, 1) -# self.assertEqual(bz2f.read(), self.TEXT[500-150:]) -# bz2f.close() -# -# def testSeekBackwardsFromEnd(self): -# # "Test BZ2File.seek(-150, 2)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(-150, 2) -# self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) -# bz2f.close() -# -# def testSeekPostEnd(self): -# # "Test BZ2File.seek(150000)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(150000) -# self.assertEqual(bz2f.tell(), len(self.TEXT)) -# self.assertEqual(bz2f.read(), "") -# bz2f.close() -# -# def testSeekPostEndTwice(self): -# # "Test BZ2File.seek(150000) twice" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(150000) -# bz2f.seek(150000) -# self.assertEqual(bz2f.tell(), len(self.TEXT)) -# self.assertEqual(bz2f.read(), "") -# bz2f.close() -# -# def testSeekPreStart(self): -# # "Test BZ2File.seek(-150, 0)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(-150) -# self.assertEqual(bz2f.tell(), 0) -# self.assertEqual(bz2f.read(), self.TEXT) -# bz2f.close() -# # def testOpenDel(self): # # "Test opening and deleting a file many times" # self.createTempFile() From rhymes at codespeak.net Sat Aug 5 19:09:32 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 19:09:32 +0200 (CEST) Subject: [pypy-svn] r31027 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805170932.22A5610078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 19:09:29 2006 New Revision: 31027 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: more tests for seek. now is definitely rock solid Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 19:09:29 2006 @@ -159,27 +159,51 @@ assert bz2f.tell() == len(TEXT) assert bz2f.read() == "" bz2f.close() + + def test_seek_post_end_twice(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' -# def testSeekPostEndTwice(self): -# # "Test BZ2File.seek(150000) twice" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(150000) -# bz2f.seek(150000) -# self.assertEqual(bz2f.tell(), len(self.TEXT)) -# self.assertEqual(bz2f.read(), "") -# bz2f.close() -# -# def testSeekPreStart(self): -# # "Test BZ2File.seek(-150, 0)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# bz2f.seek(-150) -# self.assertEqual(bz2f.tell(), 0) -# self.assertEqual(bz2f.read(), self.TEXT) -# bz2f.close() -# + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.seek(150000) + bz2f.seek(150000) + assert bz2f.tell() == len(TEXT) + assert bz2f.read() == "" + bz2f.close() + + def test_seek_pre_start(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + bz2f.seek(-150) + assert bz2f.tell() == 0 + assert bz2f.read() == TEXT + bz2f.close() def test_readline(self): def create_temp_file(crlf=False): From rhymes at codespeak.net Sat Aug 5 19:41:51 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 19:41:51 +0200 (CEST) Subject: [pypy-svn] r31029 - pypy/dist/pypy/module/bz2 Message-ID: <20060805174151.3478610078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 19:41:47 2006 New Revision: 31029 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: exposed newlines property. indicates which kind of end line terminator you encounter while reading with universal new lines mode Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 19:41:47 2006 @@ -4,7 +4,7 @@ from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app from ctypes import * import ctypes.util @@ -604,15 +604,39 @@ return self.space.wrap("".join(buf_lst[:bytesread])) return self.space.wrap("".join(buf_lst)) read.unwrap_spec = ['self', int] + + # accessor for newlines property + def fget_newlines(space, self): + if self.f_newlinetypes == NEWLINE_UNKNOWN: + return space.wrap(None) + elif self.f_newlinetypes == NEWLINE_CR: + return space.wrap('\r') + elif self.f_newlinetypes == NEWLINE_LF: + return space.wrap('\n') + elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_LF: + return space.wrap(('\r', '\n')) + elif self.f_newlinetypes == NEWLINE_CRLF: + return space.wrap("\r\n") + elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_CRLF: + return space.wrap(('\r', "\r\n")) + elif self.f_newlinetypes == NEWLINE_LF|NEWLINE_CRLF: + return space.wrap(('\n', "\r\n")) + elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF: + return space.wrap(('\r', '\n', "\r\n")) + else: + raise OperationError(space.w_SystemError, + space.wrap( + "Unknown newlines value 0x%d\n" % hex(self.f_newlinetypes))) - +get_newlines = GetSetProperty(_BZ2File.fget_newlines, cls=_BZ2File) _BZ2File.typedef = TypeDef("_BZ2File", close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), readline = interp2app(_BZ2File.readline, unwrap_spec=_BZ2File.readline.unwrap_spec), - read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec) + read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), + newlines = get_newlines, ) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): From rhymes at codespeak.net Sat Aug 5 20:18:49 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 20:18:49 +0200 (CEST) Subject: [pypy-svn] r31034 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805181849.0EB6E1007A@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 20:18:46 2006 New Revision: 31034 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: tests on different newlines work correctly Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 20:18:46 2006 @@ -149,7 +149,8 @@ dst = buf if not obj.f_univ_newline: - return libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + return nread, buf newlinetypes = obj.f_newlinetypes skipnextlf = obj.f_skipnextlf @@ -205,9 +206,12 @@ obj.f_newlinetypes = newlinetypes obj.f_skipnextlf = skipnextlf - buf = c_char_p("".join(dst_lst)) + data = "".join(dst_lst) + buf = create_string_buffer(len(data)) + for i, ch in enumerate(data): + buf[i] = ch - return dst_pos + return dst_pos, buf def _getline(space, obj, size): used_v_size = 0 # no. used slots in buffer @@ -464,7 +468,7 @@ if whence == 2: if self.size == -1: while True: - chunksize = _univ_newline_read(bzerror, self.fp, buf, + chunksize, buf = _univ_newline_read(bzerror, self.fp, buf, bufsize, self) self.pos += chunksize bytesread += chunksize @@ -519,7 +523,8 @@ # condition above). bufsize is 8192. readsize = offset - bytesread - chunksize = _univ_newline_read(bzerror, self.fp, buf, readsize, self) + chunksize, buf = _univ_newline_read(bzerror, self.fp, buf, + readsize, self) self.pos += chunksize bytesread += chunksize @@ -582,7 +587,7 @@ buf = create_string_buffer(bufsize) bzerror = c_int() while True: - chunksize = _univ_newline_read(bzerror, self.fp, buf, + chunksize, buf = _univ_newline_read(bzerror, self.fp, buf, bufsize - bytesread, self) self.pos += chunksize bytesread += chunksize Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 20:18:46 2006 @@ -298,6 +298,49 @@ assert bz2f.read(100) == TEXT[:100] bz2f.close() + def test_universal_newlines_lf(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo", "rU") + assert bz2f.read() == TEXT + assert bz2f.newlines == "\n" + bz2f.close() + + def test_universal_newlines_crlf(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file(crlf=True) + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo", "rU") + data = bz2f.read() + assert data == TEXT + assert bz2f.newlines == "\r\n" + bz2f.close() + # #!/usr/bin/python # from test import test_support # from test.test_support import TESTFN @@ -364,22 +407,6 @@ # self.assertEqual(list(bz2f.xreadlines()), sio.readlines()) # bz2f.close() # -# def testUniversalNewlinesLF(self): -# # "Test BZ2File.read() with universal newlines (\\n)" -# self.createTempFile() -# bz2f = BZ2File(self.filename, "rU") -# self.assertEqual(bz2f.read(), self.TEXT) -# self.assertEqual(bz2f.newlines, "\n") -# bz2f.close() -# -# def testUniversalNewlinesCRLF(self): -# # "Test BZ2File.read() with universal newlines (\\r\\n)" -# self.createTempFile(crlf=1) -# bz2f = BZ2File(self.filename, "rU") -# self.assertEqual(bz2f.read(), self.TEXT) -# self.assertEqual(bz2f.newlines, "\r\n") -# bz2f.close() -# # def testWrite(self): # # "Test BZ2File.write()" # bz2f = BZ2File(self.filename, "w") From rhymes at codespeak.net Sat Aug 5 20:22:17 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 20:22:17 +0200 (CEST) Subject: [pypy-svn] r31035 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805182217.6A0621007C@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 20:22:15 2006 New Revision: 31035 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: test open/close/del operations Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 20:22:15 2006 @@ -340,6 +340,25 @@ assert data == TEXT assert bz2f.newlines == "\r\n" bz2f.close() + + def test_open_close_del(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + for i in range(10000): + f = BZ2File("foo") + f.close() + del f # #!/usr/bin/python # from test import test_support From rhymes at codespeak.net Sat Aug 5 20:25:18 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 20:25:18 +0200 (CEST) Subject: [pypy-svn] r31036 - pypy/dist/pypy/module/bz2 Message-ID: <20060805182518.220D310080@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 20:25:15 2006 New Revision: 31036 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: catch the exception dude! Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 20:25:15 2006 @@ -364,8 +364,9 @@ mode = ('wb', 'rb')[mode_char == 'r'] # open the file and set the buffer - f = pythonapi.PyFile_FromString(name, mode) - if not f: + try: + f = pythonapi.PyFile_FromString(name, mode) + except IOError: raise OperationError(self.space.w_IOError, self.space.wrap("cannot open file %s" % name)) pythonapi.PyFile_SetBufSize(f, buffering) From rhymes at codespeak.net Sat Aug 5 22:46:39 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 22:46:39 +0200 (CEST) Subject: [pypy-svn] r31048 - pypy/dist/pypy/module/bz2 Message-ID: <20060805204639.F0A8210075@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 22:46:37 2006 New Revision: 31048 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: mode_char is never equals 0. fixed it. Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 22:46:37 2006 @@ -359,7 +359,7 @@ raise OperationError(self.space.w_ValueError, self.space.wrap("invalid mode char %s" % mode)) - if mode_char == 0: + if mode_char == "": mode_char = 'r' mode = ('wb', 'rb')[mode_char == 'r'] From rhymes at codespeak.net Sat Aug 5 22:47:07 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 22:47:07 +0200 (CEST) Subject: [pypy-svn] r31049 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805204707.64EA710075@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 22:47:04 2006 New Revision: 31049 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: some more tests Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 22:47:04 2006 @@ -25,7 +25,7 @@ BZ2File("foo", mode='wb') # a large buf size BZ2File("foo", mode='w', buffering=4096) - + def test_close(self): from bz2 import BZ2File @@ -73,6 +73,52 @@ bz2f.seek(0) assert bz2f.tell() == 0 + + def test_open_close_del(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + for i in range(10000): + f = BZ2File("foo") + f.close() + del f + + def test_open_non_existent(self): + from bz2 import BZ2File + raises(IOError, BZ2File, "/non/existent/path") + + def test_open_mode_U(self): + # bug #1194181: bz2.BZ2File opened for write with mode "U" + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + def create_temp_file(crlf=False): + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + create_temp_file() + + bz2f = BZ2File("foo", "U") + bz2f.close() + f = open("foo") + f.seek(0, 2) + data = f.read() + assert f.tell() == len(DATA) + f.close() def test_seek_forward(self): def create_temp_file(crlf=False): @@ -340,25 +386,7 @@ assert data == TEXT assert bz2f.newlines == "\r\n" bz2f.close() - - def test_open_close_del(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() - - for i in range(10000): - f = BZ2File("foo") - f.close() - del f # #!/usr/bin/python # from test import test_support @@ -468,20 +496,6 @@ # o = BZ2File(self.filename) # del o # -# def testOpenNonexistent(self): -# # "Test opening a nonexistent file" -# self.assertRaises(IOError, BZ2File, "/non/existent") -# -# def testModeU(self): -# # Bug #1194181: bz2.BZ2File opened for write with mode "U" -# self.createTempFile() -# bz2f = BZ2File(self.filename, "U") -# bz2f.close() -# f = file(self.filename) -# f.seek(0, 2) -# self.assertEqual(f.tell(), len(self.DATA)) -# f.close() -# # def testBug1191043(self): # # readlines() for files containing no newline # data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' From rhymes at codespeak.net Sat Aug 5 23:07:27 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 23:07:27 +0200 (CEST) Subject: [pypy-svn] r31050 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805210727.A839910075@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 23:07:24 2006 New Revision: 31050 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: name, mode, closed BZ2File properties are in Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 23:07:24 2006 @@ -5,6 +5,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import interp2app from ctypes import * import ctypes.util @@ -328,12 +329,15 @@ self.pos = 0 self.size = 0 - self._init_bz2file(filename, mode, buffering, compresslevel) + self.filename = filename + self.mode_string = "" + + self._init_bz2file(mode, buffering, compresslevel) - def _init_bz2file(self, filename, mode_, buffering, compresslevel): + def _init_bz2file(self, mode_, buffering, compresslevel): self.size = -1 - name = filename + name = self.filename mode_char = "" mode_list = mode_ @@ -362,6 +366,7 @@ if mode_char == "": mode_char = 'r' mode = ('wb', 'rb')[mode_char == 'r'] + self.mode_string = mode # open the file and set the buffer try: @@ -611,7 +616,7 @@ return self.space.wrap("".join(buf_lst)) read.unwrap_spec = ['self', int] - # accessor for newlines property + # accessors for properties def fget_newlines(space, self): if self.f_newlinetypes == NEWLINE_UNKNOWN: return space.wrap(None) @@ -633,8 +638,12 @@ raise OperationError(space.w_SystemError, space.wrap( "Unknown newlines value 0x%d\n" % hex(self.f_newlinetypes))) + + def fget_closed(space, self): + return space.wrap(self.mode == MODE_CLOSED) get_newlines = GetSetProperty(_BZ2File.fget_newlines, cls=_BZ2File) +get_closed = GetSetProperty(_BZ2File.fget_closed, cls=_BZ2File) _BZ2File.typedef = TypeDef("_BZ2File", close = interp2app(_BZ2File.close, unwrap_spec=_BZ2File.close.unwrap_spec), tell = interp2app(_BZ2File.tell, unwrap_spec=_BZ2File.tell.unwrap_spec), @@ -643,6 +652,9 @@ unwrap_spec=_BZ2File.readline.unwrap_spec), read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), newlines = get_newlines, + name = interp_attrproperty("filename", _BZ2File), + mode = interp_attrproperty("mode_string", _BZ2File), + closed = get_closed, ) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 23:07:24 2006 @@ -13,6 +13,17 @@ def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space + + def test_attributes(self): + from bz2 import BZ2File + + bz2f = BZ2File("foo", mode="w") + assert bz2f.name == "foo" + assert bz2f.newlines == None + assert bz2f.mode == "wb" + assert bz2f.closed == False + bz2f.close() + assert bz2f.closed == True def test_creation(self): from bz2 import BZ2File @@ -489,13 +500,7 @@ # f = open(self.filename, 'rb') # self.assertEqual(self.decompress(f.read()), self.TEXT) # f.close() -# def testOpenDel(self): -# # "Test opening and deleting a file many times" -# self.createTempFile() -# for i in xrange(10000): -# o = BZ2File(self.filename) -# del o -# +# # def testBug1191043(self): # # readlines() for files containing no newline # data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' From rhymes at codespeak.net Sat Aug 5 23:13:44 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 23:13:44 +0200 (CEST) Subject: [pypy-svn] r31051 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805211344.52AD310078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 23:13:41 2006 New Revision: 31051 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: softspace property is in too Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 23:13:41 2006 @@ -652,9 +652,10 @@ unwrap_spec=_BZ2File.readline.unwrap_spec), read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), newlines = get_newlines, + closed = get_closed, name = interp_attrproperty("filename", _BZ2File), mode = interp_attrproperty("mode_string", _BZ2File), - closed = get_closed, + softspace = interp_attrproperty("f_softspace", _BZ2File), ) def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 23:13:41 2006 @@ -21,6 +21,7 @@ assert bz2f.name == "foo" assert bz2f.newlines == None assert bz2f.mode == "wb" + assert bz2f.softspace == False assert bz2f.closed == False bz2f.close() assert bz2f.closed == True From rhymes at codespeak.net Sat Aug 5 23:47:57 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 5 Aug 2006 23:47:57 +0200 (CEST) Subject: [pypy-svn] r31052 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805214757.68A2F10078@code0.codespeak.net> Author: rhymes Date: Sat Aug 5 23:47:52 2006 New Revision: 31052 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: readlines() Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 5 23:47:52 2006 @@ -401,7 +401,7 @@ libbz2.BZ2_bzWriteClose(byref(bzerror), self.fp, 0, None, None) _drop_readahead(self) - + def _check_if_close(self): if self.mode == MODE_CLOSED: raise OperationError(self.space.w_ValueError, @@ -616,6 +616,27 @@ return self.space.wrap("".join(buf_lst)) read.unwrap_spec = ['self', int] + def readlines(self, size=0): + """"readlines([size]) -> list + + Call readline() repeatedly and return a list of lines read. + The optional size argument, if given, is an approximate bound on the + total number of bytes in the lines returned.""" + + # it seems size definitely ignored in CPython, so... + lines = [] + + while True: + w_line = self.readline() + line = self.space.str_w(w_line) + if not line: + break + + lines.append(line) + + return self.space.wrap(lines) + readlines.unwrap_spec = ['self', int] + # accessors for properties def fget_newlines(space, self): if self.f_newlinetypes == NEWLINE_UNKNOWN: @@ -650,6 +671,8 @@ seek = interp2app(_BZ2File.seek, unwrap_spec=_BZ2File.seek.unwrap_spec), readline = interp2app(_BZ2File.readline, unwrap_spec=_BZ2File.readline.unwrap_spec), + readlines = interp2app(_BZ2File.readlines, + unwrap_spec=_BZ2File.readlines.unwrap_spec), read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), newlines = get_newlines, closed = get_closed, Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sat Aug 5 23:47:52 2006 @@ -399,6 +399,29 @@ assert bz2f.newlines == "\r\n" bz2f.close() + def test_readlines(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + from StringIO import StringIO + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + raises(TypeError, bz2f.readlines, None) + sio = StringIO(TEXT) + assert bz2f.readlines() == sio.readlines() + bz2f.close() + # #!/usr/bin/python # from test import test_support @@ -441,14 +464,6 @@ # class BZ2FileTest(BaseTest): # "Test BZ2File type miscellaneous methods." # -# def testReadLines(self): -# # "Test BZ2File.readlines()" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# self.assertRaises(TypeError, bz2f.readlines, None) -# sio = StringIO(self.TEXT) -# self.assertEqual(bz2f.readlines(), sio.readlines()) -# bz2f.close() # # def testIterator(self): # # "Test iter(BZ2File)" From rhymes at codespeak.net Sun Aug 6 00:05:39 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 00:05:39 +0200 (CEST) Subject: [pypy-svn] r31053 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805220539.3243510078@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 00:05:36 2006 New Revision: 31053 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: UGLY hack because there's no support of special methods outside so I can't use the iterator protocol (no __iter__), next() method is not implemented, it's useless right now. xreadlines() is an alias of get_iterator(). Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 00:05:36 2006 @@ -623,6 +623,8 @@ The optional size argument, if given, is an approximate bound on the total number of bytes in the lines returned.""" + self._check_if_close() + # it seems size definitely ignored in CPython, so... lines = [] @@ -662,6 +664,27 @@ def fget_closed(space, self): return space.wrap(self.mode == MODE_CLOSED) + + # XXX: I have to hack-in this UGLY thing because there's no support of + # special methods outside so I can't use the iterator protocol (no __iter__) + # next() method is not implemented, it's useless right now + # XXX no 2: unwrap() is the evil itself! XXX + def get_iterator(self): + w_lines = self.readlines() + lines = self.space.unwrap(w_lines) + return self.space.wrap(iter(lines)) + get_iterator.unwrap_spec = ['self'] + + def xreadlines(self): + """xreadlines() -> self + + For backward compatibility. BZ2File objects now include the performance + optimizations previously implemented in the xreadlines module.""" + + # this method should use the iterator protocol one day... + return self.get_iterator() + xreadlines.unwrap_spec = ['self'] + get_newlines = GetSetProperty(_BZ2File.fget_newlines, cls=_BZ2File) get_closed = GetSetProperty(_BZ2File.fget_closed, cls=_BZ2File) @@ -674,6 +697,10 @@ readlines = interp2app(_BZ2File.readlines, unwrap_spec=_BZ2File.readlines.unwrap_spec), read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), + get_iterator = interp2app(_BZ2File.get_iterator, + unwrap_spec=_BZ2File.get_iterator.unwrap_spec), + xreadlines = interp2app(_BZ2File.xreadlines, + unwrap_spec=_BZ2File.xreadlines.unwrap_spec), newlines = get_newlines, closed = get_closed, name = interp_attrproperty("filename", _BZ2File), Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 00:05:36 2006 @@ -422,6 +422,51 @@ assert bz2f.readlines() == sio.readlines() bz2f.close() + def test_iterator(self): + # XXX: to fix when the actual iterator protocol will be available + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + from StringIO import StringIO + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + sio = StringIO(TEXT) + assert list(bz2f.get_iterator()) == sio.readlines() + bz2f.close() + + def test_xreadlines(self): + def create_temp_file(crlf=False): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + from bz2 import BZ2File + from StringIO import StringIO + create_temp_file() + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo") + sio = StringIO(TEXT) + assert list(bz2f.xreadlines()) == sio.readlines() + bz2f.close() + # #!/usr/bin/python # from test import test_support @@ -463,24 +508,6 @@ # # class BZ2FileTest(BaseTest): # "Test BZ2File type miscellaneous methods." -# -# -# def testIterator(self): -# # "Test iter(BZ2File)" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# sio = StringIO(self.TEXT) -# self.assertEqual(list(iter(bz2f)), sio.readlines()) -# bz2f.close() -# -# def testXReadLines(self): -# # "Test BZ2File.xreadlines()" -# self.createTempFile() -# bz2f = BZ2File(self.filename) -# sio = StringIO(self.TEXT) -# self.assertEqual(list(bz2f.xreadlines()), sio.readlines()) -# bz2f.close() -# # def testWrite(self): # # "Test BZ2File.write()" # bz2f = BZ2File(self.filename, "w") From rhymes at codespeak.net Sun Aug 6 00:08:02 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 00:08:02 +0200 (CEST) Subject: [pypy-svn] r31054 - pypy/dist/pypy/module/bz2 Message-ID: <20060805220802.B3A6C10078@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 00:08:00 2006 New Revision: 31054 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: s/_check_if_close/_check_if_closed -- makes more sens Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 00:08:00 2006 @@ -402,7 +402,7 @@ _drop_readahead(self) - def _check_if_close(self): + def _check_if_closed(self): if self.mode == MODE_CLOSED: raise OperationError(self.space.w_ValueError, self.space.wrap("I/O operation on closed file")) @@ -442,7 +442,7 @@ Return the current file position, an integer (may be a long integer).""" - self._check_if_close() + self._check_if_closed() return self.space.wrap(self.pos) tell.unwrap_spec = ['self'] @@ -460,7 +460,7 @@ the operation may be extremely slow.""" _drop_readahead(self) - self._check_if_close() + self._check_if_closed() bufsize = SMALLCHUNK buf = create_string_buffer(bufsize) @@ -553,7 +553,7 @@ return (an incomplete line may be returned then). Return an empty string at EOF.""" - self._check_if_close() + self._check_if_closed() if self.mode == MODE_READ_EOF: return self.space.wrap("") @@ -574,7 +574,7 @@ Read at most size uncompressed bytes, returned as a string. If the size argument is negative or omitted, read until EOF is reached.""" - self._check_if_close() + self._check_if_closed() if self.mode == MODE_READ_EOF: return self.space.wrap("") @@ -623,7 +623,7 @@ The optional size argument, if given, is an approximate bound on the total number of bytes in the lines returned.""" - self._check_if_close() + self._check_if_closed() # it seems size definitely ignored in CPython, so... lines = [] From rhymes at codespeak.net Sun Aug 6 00:14:41 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 00:14:41 +0200 (CEST) Subject: [pypy-svn] r31055 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805221441.C103E10078@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 00:14:36 2006 New Revision: 31055 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: one more test for readlines()/xreadlines() Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 00:14:36 2006 @@ -467,6 +467,24 @@ assert list(bz2f.xreadlines()) == sio.readlines() bz2f.close() + def test_readlines_bug_1191043(self): + # readlines()/xreadlines() for files containing no newline + from bz2 import BZ2File + + data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' + f = open("foo", "wb") + f.write(data) + f.close() + + bz2f = BZ2File("foo") + lines = bz2f.readlines() + bz2f.close() + assert lines == ['Test'] + + bz2f = BZ2File("foo") + xlines = list(bz2f.xreadlines()) + bz2f.close() + assert xlines == ['Test'] # #!/usr/bin/python # from test import test_support @@ -544,20 +562,6 @@ # self.assertEqual(self.decompress(f.read()), self.TEXT) # f.close() # -# def testBug1191043(self): -# # readlines() for files containing no newline -# data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' -# f = open(self.filename, "wb") -# f.write(data) -# f.close() -# bz2f = BZ2File(self.filename) -# lines = bz2f.readlines() -# bz2f.close() -# self.assertEqual(lines, ['Test']) -# bz2f = BZ2File(self.filename) -# xlines = list(bz2f.xreadlines()) -# bz2f.close() -# self.assertEqual(lines, ['Test']) # # # class BZ2CompressorTest(BaseTest): From rhymes at codespeak.net Sun Aug 6 01:07:13 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 01:07:13 +0200 (CEST) Subject: [pypy-svn] r31056 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805230713.3199010072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 01:07:10 2006 New Revision: 31056 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: write() and s/StringIO/cStringIO in tests. decompress() utility function is not implemented for now Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 01:07:10 2006 @@ -108,6 +108,8 @@ libbz2.BZ2_bzWriteClose.restype = c_void libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] libbz2.BZ2_bzRead.restype = c_int +libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] +libbz2.BZ2_bzWrite.restype = c_void libc.strerror.restype = c_char_p libc.strerror.argtypes = [c_int] @@ -319,11 +321,11 @@ self.f_bufend = c_char_p() # points after last occupied position self.f_bufptr = c_char_p() # current buffer position - self.f_softspace = 0 # flag used by print command + self.f_softspace = False # flag used by print command self.f_univ_newline = False # handle any newline convention self.f_newlinetypes = 0 # types of newlines seen - self.f_skipnextlf = 0 # skip next \n + self.f_skipnextlf = False # skip next \n self.mode = 0 self.pos = 0 @@ -639,6 +641,30 @@ return self.space.wrap(lines) readlines.unwrap_spec = ['self', int] + def write(self, data): + """write(data) -> None + + Write the 'data' string to file. Note that due to buffering, close() may + be needed before the file on disk reflects the data written.""" + + self._check_if_closed() + + if not self.mode == MODE_WRITE: + raise OperationError(self.space.w_IOError, + self.space.wrap("file is not ready for writing")) + + self.f_softspace = False + + bzerror = c_int() + bufsize = len(data) + buf = c_char_p(data) + libbz2.BZ2_bzWrite(byref(bzerror), self.fp, buf, bufsize) + self.pos += bufsize + + if bzerror.value != BZ_OK: + _catch_bz2_error(self.space, bzerror) + write.unwrap_spec = ['self', str] + # accessors for properties def fget_newlines(space, self): if self.f_newlinetypes == NEWLINE_UNKNOWN: @@ -701,6 +727,7 @@ unwrap_spec=_BZ2File.get_iterator.unwrap_spec), xreadlines = interp2app(_BZ2File.xreadlines, unwrap_spec=_BZ2File.xreadlines.unwrap_spec), + write = interp2app(_BZ2File.write, unwrap_spec=_BZ2File.write.unwrap_spec), newlines = get_newlines, closed = get_closed, name = interp_attrproperty("filename", _BZ2File), Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 01:07:10 2006 @@ -275,7 +275,7 @@ f.close() from bz2 import BZ2File - from StringIO import StringIO + from cStringIO import StringIO create_temp_file() TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' @@ -411,7 +411,7 @@ f.close() from bz2 import BZ2File - from StringIO import StringIO + from cStringIO import StringIO create_temp_file() TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' @@ -435,7 +435,7 @@ f.close() from bz2 import BZ2File - from StringIO import StringIO + from cStringIO import StringIO create_temp_file() TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' @@ -457,7 +457,7 @@ f.close() from bz2 import BZ2File - from StringIO import StringIO + from cStringIO import StringIO create_temp_file() TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' @@ -485,6 +485,41 @@ xlines = list(bz2f.xreadlines()) bz2f.close() assert xlines == ['Test'] + + def test_write(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import BZ2File + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo", 'w') + raises(TypeError, bz2f.write) + bz2f.write(TEXT) + bz2f.close() + + f = open("foo", "rb") + assert decompress(f.read()) == TEXT + f.close() + +# def decompress(self, data): +# pop = popen2.Popen3("bunzip2", capturestderr=1) +# pop.tochild.write(data) +# pop.tochild.close() +# ret = pop.fromchild.read() +# pop.fromchild.close() +# if pop.wait() != 0: +# ret = bz2.decompress(data) +# return ret # #!/usr/bin/python # from test import test_support From rhymes at codespeak.net Sun Aug 6 01:10:33 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 01:10:33 +0200 (CEST) Subject: [pypy-svn] r31057 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805231033.28E8510072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 01:10:26 2006 New Revision: 31057 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: purged useless commented code Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 01:10:26 2006 @@ -511,53 +511,24 @@ assert decompress(f.read()) == TEXT f.close() -# def decompress(self, data): -# pop = popen2.Popen3("bunzip2", capturestderr=1) -# pop.tochild.write(data) -# pop.tochild.close() -# ret = pop.fromchild.read() -# pop.fromchild.close() -# if pop.wait() != 0: -# ret = bz2.decompress(data) -# return ret - -# #!/usr/bin/python -# from test import test_support -# from test.test_support import TESTFN -# -# import unittest -# from cStringIO import StringIO -# import os -# import popen2 -# import sys -# -# import bz2 -# from bz2 import BZ2File, BZ2Compressor, BZ2Decompressor -# # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # -# class BaseTest(unittest.TestCase): -# "Base for other testcases." -# TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' -# DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' -# DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' -# -# if has_cmdline_bunzip2: -# def decompress(self, data): -# pop = popen2.Popen3("bunzip2", capturestderr=1) -# pop.tochild.write(data) -# pop.tochild.close() -# ret = pop.fromchild.read() -# pop.fromchild.close() -# if pop.wait() != 0: -# ret = bz2.decompress(data) -# return ret -# -# else: -# # popen2.Popen3 doesn't exist on Windows, and even if it did, bunzip2 -# # isn't available to run. -# def decompress(self, data): -# return bz2.decompress(data) +# if has_cmdline_bunzip2: +# def decompress(self, data): +# pop = popen2.Popen3("bunzip2", capturestderr=1) +# pop.tochild.write(data) +# pop.tochild.close() +# ret = pop.fromchild.read() +# pop.fromchild.close() +# if pop.wait() != 0: +# ret = bz2.decompress(data) +# return ret +# +# else: +# # popen2.Popen3 doesn't exist on Windows, and even if it did, bunzip2 +# # isn't available to run. +# def decompress(self, data): +# return bz2.decompress(data) # # class BZ2FileTest(BaseTest): # "Test BZ2File type miscellaneous methods." @@ -682,17 +653,3 @@ # def testDecompressIncomplete(self): # # "Test decompress() function with incomplete data" # self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) -# -# def test_main(): -# test_support.run_unittest( -# BZ2FileTest, -# BZ2CompressorTest, -# BZ2DecompressorTest, -# FuncTest -# ) -# test_support.reap_children() -# -# if __name__ == '__main__': -# test_main() -# -# # vim:ts=4:sw=4 From rhymes at codespeak.net Sun Aug 6 01:13:50 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 01:13:50 +0200 (CEST) Subject: [pypy-svn] r31058 - pypy/dist/pypy/module/bz2/test Message-ID: <20060805231350.46BC310072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 01:13:48 2006 New Revision: 31058 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: one more test for BZ2File.write() Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 01:13:48 2006 @@ -511,6 +511,37 @@ assert decompress(f.read()) == TEXT f.close() + def test_write_chunks_10(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import BZ2File + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo", 'w') + n = 0 + while True: + data = TEXT[n * 10:(n + 1) * 10] + if not data: + break + + bz2f.write(data) + n += 1 + bz2f.close() + + f = open("foo", "rb") + assert decompress(f.read()) == TEXT + f.close() + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -532,31 +563,6 @@ # # class BZ2FileTest(BaseTest): # "Test BZ2File type miscellaneous methods." -# def testWrite(self): -# # "Test BZ2File.write()" -# bz2f = BZ2File(self.filename, "w") -# self.assertRaises(TypeError, bz2f.write) -# bz2f.write(self.TEXT) -# bz2f.close() -# f = open(self.filename, 'rb') -# self.assertEqual(self.decompress(f.read()), self.TEXT) -# f.close() -# -# def testWriteChunks10(self): -# # "Test BZ2File.write() with chunks of 10 bytes" -# bz2f = BZ2File(self.filename, "w") -# n = 0 -# while 1: -# str = self.TEXT[n*10:(n+1)*10] -# if not str: -# break -# bz2f.write(str) -# n += 1 -# bz2f.close() -# f = open(self.filename, 'rb') -# self.assertEqual(self.decompress(f.read()), self.TEXT) -# f.close() -# # def testWriteLines(self): # # "Test BZ2File.writelines()" # bz2f = BZ2File(self.filename, "w") From rhymes at codespeak.net Sun Aug 6 01:55:13 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 01:55:13 +0200 (CEST) Subject: [pypy-svn] r31059 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060805235513.5454A10072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 01:55:08 2006 New Revision: 31059 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: BZ2File.writelines() Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 01:55:08 2006 @@ -665,6 +665,16 @@ _catch_bz2_error(self.space, bzerror) write.unwrap_spec = ['self', str] + def writelines(self, w_sequence_of_strings): + self._check_if_closed() + + seq_w = self.space.unpackiterable(w_sequence_of_strings) + + for w_line in seq_w: + line = self.space.str_w(w_line) + self.write(line) + writelines.unwrap_spec = ['self', W_Root] + # accessors for properties def fget_newlines(space, self): if self.f_newlinetypes == NEWLINE_UNKNOWN: @@ -728,6 +738,8 @@ xreadlines = interp2app(_BZ2File.xreadlines, unwrap_spec=_BZ2File.xreadlines.unwrap_spec), write = interp2app(_BZ2File.write, unwrap_spec=_BZ2File.write.unwrap_spec), + writelines = interp2app(_BZ2File.writelines, + unwrap_spec=_BZ2File.writelines.unwrap_spec), newlines = get_newlines, closed = get_closed, name = interp_attrproperty("filename", _BZ2File), Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 01:55:08 2006 @@ -542,6 +542,32 @@ assert decompress(f.read()) == TEXT f.close() + def test_writelines(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import BZ2File + from cStringIO import StringIO + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2f = BZ2File("foo", 'w') + raises(TypeError, bz2f.writelines) + sio = StringIO(TEXT) + bz2f.writelines(sio.readlines()) + bz2f.close() + f = open("foo", "rb") + assert decompress(f.read()) == TEXT + f.close() + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -560,22 +586,7 @@ # # isn't available to run. # def decompress(self, data): # return bz2.decompress(data) -# -# class BZ2FileTest(BaseTest): -# "Test BZ2File type miscellaneous methods." -# def testWriteLines(self): -# # "Test BZ2File.writelines()" -# bz2f = BZ2File(self.filename, "w") -# self.assertRaises(TypeError, bz2f.writelines) -# sio = StringIO(self.TEXT) -# bz2f.writelines(sio.readlines()) -# bz2f.close() -# f = open(self.filename, 'rb') -# self.assertEqual(self.decompress(f.read()), self.TEXT) -# f.close() # -# -# # class BZ2CompressorTest(BaseTest): # def testCompress(self): # # "Test BZ2Compressor.compress()/flush()" From rhymes at codespeak.net Sun Aug 6 02:16:27 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 02:16:27 +0200 (CEST) Subject: [pypy-svn] r31060 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060806001627.29B0710072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 02:16:24 2006 New Revision: 31060 Modified: pypy/dist/pypy/module/bz2/__init__.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: BZ2Compressor initial stuff Modified: pypy/dist/pypy/module/bz2/__init__.py ============================================================================== --- pypy/dist/pypy/module/bz2/__init__.py (original) +++ pypy/dist/pypy/module/bz2/__init__.py Sun Aug 6 02:16:24 2006 @@ -3,6 +3,7 @@ class Module(MixedModule): interpleveldefs = { 'BZ2File': 'interp_bz2.BZ2File', + 'BZ2Compressor': 'interp_bz2.BZ2Compressor', } appleveldefs = { Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 02:16:24 2006 @@ -111,6 +111,9 @@ libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] libbz2.BZ2_bzWrite.restype = c_void +libbz2.BZ2_bzCompressInit.argtypes = [POINTER(bz_stream), c_int, c_int, c_int] +libbz2.BZ2_bzCompressInit.restype = c_int + libc.strerror.restype = c_char_p libc.strerror.argtypes = [c_int] libc.fclose.argtypes = [POINTER(FILE)] @@ -747,6 +750,39 @@ softspace = interp_attrproperty("f_softspace", _BZ2File), ) +class _BZ2Comp(Wrappable): + def __init__(self, space, compresslevel): + self.space = space + self.bzs = bz_stream() + self.running = False + + self._init_bz2comp(compresslevel) + + def _init_bz2comp(self, compresslevel): + if compresslevel < 1 or compresslevel > 9: + raise OperationError(self.space.w_ValueError, + self.space.wrap("compresslevel must be between 1 and 9")) + + bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.running = True + + +_BZ2Comp.typedef = TypeDef("_BZ2File") + +def BZ2Compressor(space, compresslevel=9): + """BZ2Compressor([compresslevel=9]) -> compressor object + + Create a new compressor object. This object may be used to compress + data sequentially. If you want to compress data in one shot, use the + compress() function instead. The compresslevel parameter, if given, + must be a number between 1 and 9.""" + + return _BZ2Comp(space, compresslevel) +BZ2Compressor.unwrap_spec = [ObjSpace, int] + def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object @@ -762,6 +798,7 @@ for this attribute is one of None (no newline read yet), '\\r', '\\n', '\\r\\n' or a tuple containing all the newline types seen. Universal newlines are available only when reading.""" + return _BZ2File(space, filename, mode, buffering, compresslevel) BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 02:16:24 2006 @@ -9,7 +9,7 @@ if os.path.exists("foo"): os.unlink("foo") -class AppTestBz2File: +class AppTestBZ2File: def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space @@ -568,6 +568,21 @@ assert decompress(f.read()) == TEXT f.close() + +class AppTestBZ2Compressor: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + + def test_creation(self): + from bz2 import BZ2Compressor + + raises(TypeError, BZ2Compressor, "foo") + raises(ValueError, BZ2Compressor, 10) + + BZ2Compressor(1) + BZ2Compressor(9) + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: From rhymes at codespeak.net Sun Aug 6 02:20:03 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 02:20:03 +0200 (CEST) Subject: [pypy-svn] r31061 - pypy/dist/pypy/module/bz2 Message-ID: <20060806002003.09C9E10072@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 02:20:01 2006 New Revision: 31061 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: BZ2Compressor destructor Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 02:20:01 2006 @@ -113,6 +113,8 @@ libbz2.BZ2_bzCompressInit.argtypes = [POINTER(bz_stream), c_int, c_int, c_int] libbz2.BZ2_bzCompressInit.restype = c_int +libbz2.BZ2_bzCompressEnd.argtypes = [POINTER(bz_stream)] +libbz2.BZ2_bzCompressEnd.restype = c_int libc.strerror.restype = c_char_p libc.strerror.argtypes = [c_int] @@ -768,6 +770,9 @@ _catch_bz2_error(self.space, bzerror) self.running = True + + def __del__(self): + libbz2.BZ2_bzCompressEnd(byref(self.bzs)) _BZ2Comp.typedef = TypeDef("_BZ2File") From ac at codespeak.net Sun Aug 6 11:48:10 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Sun, 6 Aug 2006 11:48:10 +0200 (CEST) Subject: [pypy-svn] r31074 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060806094810.060DB10071@code0.codespeak.net> Author: ac Date: Sun Aug 6 11:48:09 2006 New Revision: 31074 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: Add my and Samueles accomodation. Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Sun Aug 6 11:48:09 2006 @@ -8,8 +8,8 @@ ==================== ============== ======================== Name Arrive/Depart Accomodation ==================== ============== ======================== -Anders Chrigstroem 20-28/8 ? -Samuele Pedroni 20-28/8 ? +Anders Chrigstroem 20-28/8 on-campus +Samuele Pedroni 20-28/8 on-campus Michael Hudson 20-27 Brookfield Hall, Castletroy Armin Rigo ? ? Holger Krekel 18-28 v/dublin on-campus (faxed form) From rhymes at codespeak.net Sun Aug 6 17:02:40 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 6 Aug 2006 17:02:40 +0200 (CEST) Subject: [pypy-svn] r31077 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060806150240.9D40A1006C@code0.codespeak.net> Author: rhymes Date: Sun Aug 6 17:02:36 2006 New Revision: 31077 Modified: pypy/dist/pypy/module/bz2/bzlib.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: compress() and flush(). Modified: pypy/dist/pypy/module/bz2/bzlib.py ============================================================================== --- pypy/dist/pypy/module/bz2/bzlib.py (original) +++ pypy/dist/pypy/module/bz2/bzlib.py Sun Aug 6 17:02:36 2006 @@ -8,11 +8,11 @@ class bz_stream(Structure): pass bz_stream._fields_ = [ - ('next_in', STRING), + ('next_in', POINTER(c_char)), ('avail_in', c_uint), ('total_in_lo32', c_uint), ('total_in_hi32', c_uint), - ('next_out', STRING), + ('next_out', POINTER(c_char)), ('avail_out', c_uint), ('total_out_lo32', c_uint), ('total_out_hi32', c_uint), Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sun Aug 6 17:02:36 2006 @@ -86,6 +86,17 @@ MAXINT = sys.maxint +if BZ_CONFIG_ERROR: + if sizeof(c_long) >= 8 or sizeof(c_longlong) >= 8: + def _bzs_total_out(bzs): + return long(bzs.total_out_hi32 << 32) + bzs.total_out_lo32 + else: + def _bzs_total_out(bzs): + return bzs.total_out_lo32 +else: + def _bzs_total_out(bzs): + return bzs.total_out + pythonapi.PyFile_FromString.argtypes = [c_char_p, c_char_p] pythonapi.PyFile_FromString.restype = POINTER(PyFileObject) pythonapi.PyFile_SetBufSize.argtypes = [POINTER(PyFileObject), c_int] @@ -764,7 +775,7 @@ if compresslevel < 1 or compresslevel > 9: raise OperationError(self.space.w_ValueError, self.space.wrap("compresslevel must be between 1 and 9")) - + bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(self.space, bzerror) @@ -773,9 +784,79 @@ def __del__(self): libbz2.BZ2_bzCompressEnd(byref(self.bzs)) + + def compress(self, data): + """compress(data) -> string + Provide more data to the compressor object. It will return chunks of + compressed data whenever possible. When you've finished providing data + to compress, call the flush() method to finish the compression process, + and return what is left in the internal buffers.""" + + datasize = len(data) + + if datasize == 0: + return self.space.wrap("") + + if not self.running: + raise OperationError(self.space.w_ValueError, + self.space.wrap("this object was already flushed")) + + out_bufsize = SMALLCHUNK + out_buf = create_string_buffer(out_bufsize) + + in_bufsize = datasize + in_buf = create_string_buffer(in_bufsize) + in_buf.value = data + + self.bzs.next_in = in_buf + self.bzs.avail_in = in_bufsize + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + while True: + bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_RUN) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) -_BZ2Comp.typedef = TypeDef("_BZ2File") + if self.bzs.avail_in == 0: + break + + total_out = _bzs_total_out(self.bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + return self.space.wrap(res) + compress.unwrap_spec = ['self', str] + + def flush(self): + if not self.running: + raise OperationError(self.space.w_ValueError, + self.space.wrap("this object was already flushed")) + self.running = False + + out_bufsize = SMALLCHUNK + out_buf = create_string_buffer(out_bufsize) + + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + while True: + bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_FINISH_OK: + _catch_bz2_error(self.space, bzerror) + + total_out = _bzs_total_out(self.bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + return self.space.wrap(res) + flush.unwrap_spec = ['self'] + + +_BZ2Comp.typedef = TypeDef("_BZ2Comp", + compress = interp2app(_BZ2Comp.compress, + unwrap_spec=_BZ2Comp.compress.unwrap_spec), + flush = interp2app(_BZ2Comp.flush, unwrap_spec=_BZ2Comp.flush.unwrap_spec), +) def BZ2Compressor(space, compresslevel=9): """BZ2Compressor([compresslevel=9]) -> compressor object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Sun Aug 6 17:02:36 2006 @@ -582,7 +582,29 @@ BZ2Compressor(1) BZ2Compressor(9) + + def test_compress(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + from bz2 import BZ2Compressor + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2c = BZ2Compressor() + raises(TypeError, bz2c.compress) + data = bz2c.compress(TEXT) + data = "%s%s" % (data, bz2c.flush()) + assert decompress(data) == TEXT + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -603,14 +625,6 @@ # return bz2.decompress(data) # # class BZ2CompressorTest(BaseTest): -# def testCompress(self): -# # "Test BZ2Compressor.compress()/flush()" -# bz2c = BZ2Compressor() -# self.assertRaises(TypeError, bz2c.compress) -# data = bz2c.compress(self.TEXT) -# data += bz2c.flush() -# self.assertEqual(self.decompress(data), self.TEXT) -# # def testCompressChunks10(self): # # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" # bz2c = BZ2Compressor() From mwh at codespeak.net Mon Aug 7 10:30:37 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 7 Aug 2006 10:30:37 +0200 (CEST) Subject: [pypy-svn] r31093 - pypy/dist/pypy/interpreter Message-ID: <20060807083037.ED84810069@code0.codespeak.net> Author: mwh Date: Mon Aug 7 10:30:35 2006 New Revision: 31093 Modified: pypy/dist/pypy/interpreter/pyframe.py Log: remove a non zero (mod 4) indent Modified: pypy/dist/pypy/interpreter/pyframe.py ============================================================================== --- pypy/dist/pypy/interpreter/pyframe.py (original) +++ pypy/dist/pypy/interpreter/pyframe.py Mon Aug 7 10:30:35 2006 @@ -219,9 +219,9 @@ try: try: if we_are_translated(): - # always raising, put the resume point just before! - rstack.resume_point("eval", self, executioncontext) - self.dispatch_translated(executioncontext) + # always raising, put the resume point just before! + rstack.resume_point("eval", self, executioncontext) + self.dispatch_translated(executioncontext) else: self.dispatch(executioncontext) # catch asynchronous exceptions and turn them From antocuni at codespeak.net Mon Aug 7 10:41:27 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 7 Aug 2006 10:41:27 +0200 (CEST) Subject: [pypy-svn] r31094 - in pypy/dist/pypy/translator/cli: . src Message-ID: <20060807084127.0D63310069@code0.codespeak.net> Author: antocuni Date: Mon Aug 7 10:41:21 2006 New Revision: 31094 Modified: pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: Do not longer use the WeakRef workaround because the mono bug has been solved. Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Mon Aug 7 10:41:21 2006 @@ -16,8 +16,7 @@ log = py.log.Producer("cli") py.log.setconsumer("cli", ansi_log) -#WEAKREF = '[mscorlib]System.WeakReference' -WEAKREF = '[pypylib]pypy.runtime.WeakReference' +WEAKREF = '[mscorlib]System.WeakReference' PYPY_LIST = '[pypylib]pypy.runtime.List`1<%s>' PYPY_LIST_OF_VOID = '[pypylib]pypy.runtime.ListOfVoid' PYPY_DICT = '[pypylib]pypy.runtime.Dict`2<%s, %s>' Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Mon Aug 7 10:41:21 2006 @@ -117,28 +117,6 @@ } } - // this is a simple wrapper around System.WeakReference, whose - // interface is the same as here. This is a workaround for a Mono - // bug that causes a crash when calling set_Target. - public class WeakReference - { - System.WeakReference weakref; - public WeakReference(object obj) - { - set_Target(obj); - } - - public virtual void set_Target(object obj) - { - this.weakref = new System.WeakReference(obj); - } - - public virtual object get_Target() - { - return this.weakref.Target; - } - } - public class StringBuilder { System.Text.StringBuilder builder = new System.Text.StringBuilder(); From rhymes at codespeak.net Mon Aug 7 11:51:22 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 11:51:22 +0200 (CEST) Subject: [pypy-svn] r31096 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807095122.4B60F1006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 11:51:16 2006 New Revision: 31096 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: fix compression in BZ2Compressor to work with *huge* data too Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 11:51:16 2006 @@ -814,13 +814,29 @@ self.bzs.next_out = out_buf self.bzs.avail_out = out_bufsize + temp = [] while True: bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_RUN) - if bzerror != BZ_OK: + if bzerror != BZ_RUN_OK: _catch_bz2_error(self.space, bzerror) if self.bzs.avail_in == 0: break + elif self.bzs.avail_out == 0: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = create_string_buffer(out_bufsize) + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + if temp: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) + temp.append(data) + return self.space.wrap("".join(temp)) total_out = _bzs_total_out(self.bzs) res = "".join([out_buf[i] for i in range(total_out)]) @@ -835,17 +851,37 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - + self.bzs.next_out = out_buf self.bzs.avail_out = out_bufsize + total_out = _bzs_total_out(self.bzs) + + temp = [] while True: bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) if bzerror == BZ_STREAM_END: break elif bzerror != BZ_FINISH_OK: _catch_bz2_error(self.space, bzerror) - + + if self.bzs.avail_out == 0: + data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = create_string_buffer(out_bufsize) + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + if temp: + return self.space.wrap("".join(temp)) + + if self.bzs.avail_out: + size = _bzs_total_out(self.bzs) - total_out + res = "".join([out_buf[i] for i in range(size)]) + return self.space.wrap(res) + total_out = _bzs_total_out(self.bzs) res = "".join([out_buf[i] for i in range(total_out)]) return self.space.wrap(res) Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 11:51:16 2006 @@ -605,6 +605,29 @@ data = "%s%s" % (data, bz2c.flush()) assert decompress(data) == TEXT + def test_compress_huge_data(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import BZ2Compressor + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + HUGE_DATA = TEXT * 10000 + bz2c = BZ2Compressor() + raises(TypeError, bz2c.compress) + data = bz2c.compress(HUGE_DATA) + data = "%s%s" % (data, bz2c.flush()) + assert decompress(data) == HUGE_DATA + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: From rhymes at codespeak.net Mon Aug 7 12:05:00 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 12:05:00 +0200 (CEST) Subject: [pypy-svn] r31097 - pypy/dist/pypy/module/bz2/test Message-ID: <20060807100500.125F51006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 12:04:57 2006 New Revision: 31097 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: one more test for BZ2Compressor Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 12:04:57 2006 @@ -627,7 +627,35 @@ data = bz2c.compress(HUGE_DATA) data = "%s%s" % (data, bz2c.flush()) assert decompress(data) == HUGE_DATA + + def test_compress_chunks_10(self): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import BZ2Compressor + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + bz2c = BZ2Compressor() + n = 0 + data = "" + while True: + temp = TEXT[n * 10:(n + 1) * 10] + if not temp: + break + data = "%s%s" % (data, bz2c.compress(temp)) + n += 1 + data = "%s%s" % (data, bz2c.flush()) + assert decompress(data) == TEXT + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -647,21 +675,6 @@ # def decompress(self, data): # return bz2.decompress(data) # -# class BZ2CompressorTest(BaseTest): -# def testCompressChunks10(self): -# # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" -# bz2c = BZ2Compressor() -# n = 0 -# data = '' -# while 1: -# str = self.TEXT[n*10:(n+1)*10] -# if not str: -# break -# data += bz2c.compress(str) -# n += 1 -# data += bz2c.flush() -# self.assertEqual(self.decompress(data), self.TEXT) -# # class BZ2DecompressorTest(BaseTest): # def test_Constructor(self): # self.assertRaises(TypeError, BZ2Decompressor, 42) From rhymes at codespeak.net Mon Aug 7 12:19:34 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 12:19:34 +0200 (CEST) Subject: [pypy-svn] r31098 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807101934.027FE1006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 12:19:30 2006 New Revision: 31098 Modified: pypy/dist/pypy/module/bz2/__init__.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: BZ2Decompressor initial stuff Modified: pypy/dist/pypy/module/bz2/__init__.py ============================================================================== --- pypy/dist/pypy/module/bz2/__init__.py (original) +++ pypy/dist/pypy/module/bz2/__init__.py Mon Aug 7 12:19:30 2006 @@ -4,6 +4,7 @@ interpleveldefs = { 'BZ2File': 'interp_bz2.BZ2File', 'BZ2Compressor': 'interp_bz2.BZ2Compressor', + 'BZ2Decompressor': 'interp_bz2.BZ2Decompressor', } appleveldefs = { Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 12:19:30 2006 @@ -126,6 +126,13 @@ libbz2.BZ2_bzCompressInit.restype = c_int libbz2.BZ2_bzCompressEnd.argtypes = [POINTER(bz_stream)] libbz2.BZ2_bzCompressEnd.restype = c_int +libbz2.BZ2_bzCompress.argtypes = [POINTER(bz_stream), c_int] +libbz2.BZ2_bzCompress.restype = c_int + +libbz2.BZ2_bzDecompressInit.argtypes = [POINTER(bz_stream), c_int, c_int] +libbz2.BZ2_bzDecompressInit.restype = c_int +libbz2.BZ2_bzDecompressEnd.argtypes = [POINTER(bz_stream)] +libbz2.BZ2_bzDecompressEnd.restype = c_int libc.strerror.restype = c_char_p libc.strerror.argtypes = [c_int] @@ -894,6 +901,28 @@ flush = interp2app(_BZ2Comp.flush, unwrap_spec=_BZ2Comp.flush.unwrap_spec), ) + +class _BZ2Decomp(Wrappable): + def __init__(self, space): + self.space = space + + self.bzs = bz_stream() + self.running = False + self.unused_data = "" + + self._init_bz2decomp() + + def _init_bz2decomp(self): + bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + self.running = True + + def __del__(self): + libbz2.BZ2_bzDecompressEnd(byref(self.bzs)) + + def BZ2Compressor(space, compresslevel=9): """BZ2Compressor([compresslevel=9]) -> compressor object @@ -905,6 +934,16 @@ return _BZ2Comp(space, compresslevel) BZ2Compressor.unwrap_spec = [ObjSpace, int] +def BZ2Decompressor(space): + """BZ2Decompressor() -> decompressor object + + Create a new decompressor object. This object may be used to decompress + data sequentially. If you want to decompress data in one shot, use the + decompress() function instead.""" + + return _BZ2Decomp(space) +BZ2Decompressor.unwrap_spec = [ObjSpace] + def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): """BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 12:19:30 2006 @@ -656,6 +656,18 @@ data = "%s%s" % (data, bz2c.flush()) assert decompress(data) == TEXT +class AppTestBZ2Decompressor: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + + def test_creation(self): + from bz2 import BZ2Decompressor + + raises(TypeError, BZ2Decompressor, "foo") + + BZ2Decompressor() + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: From rhymes at codespeak.net Mon Aug 7 12:26:45 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 12:26:45 +0200 (CEST) Subject: [pypy-svn] r31100 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807102645.8929B1006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 12:26:42 2006 New Revision: 31100 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: exposed unused_data attribute Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 12:26:42 2006 @@ -921,7 +921,12 @@ def __del__(self): libbz2.BZ2_bzDecompressEnd(byref(self.bzs)) - + + +_BZ2Decomp.typedef = TypeDef("_BZ2Decomp", + unused_data = interp_attrproperty("unused_data", _BZ2Decomp), +) + def BZ2Compressor(space, compresslevel=9): """BZ2Compressor([compresslevel=9]) -> compressor object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 12:26:42 2006 @@ -667,6 +667,12 @@ raises(TypeError, BZ2Decompressor, "foo") BZ2Decompressor() + + def test_attribute(self): + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + assert bz2d.unused_data == "" # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # From rhymes at codespeak.net Mon Aug 7 12:49:35 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 12:49:35 +0200 (CEST) Subject: [pypy-svn] r31101 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807104935.5183B1006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 12:49:31 2006 New Revision: 31101 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: implemented BZ2Decompressor.decompress() Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 12:49:31 2006 @@ -133,6 +133,8 @@ libbz2.BZ2_bzDecompressInit.restype = c_int libbz2.BZ2_bzDecompressEnd.argtypes = [POINTER(bz_stream)] libbz2.BZ2_bzDecompressEnd.restype = c_int +libbz2.BZ2_bzDecompress.argtypes = [POINTER(bz_stream)] +libbz2.BZ2_bzDecompress.restype = c_int libc.strerror.restype = c_char_p libc.strerror.argtypes = [c_int] @@ -922,9 +924,71 @@ def __del__(self): libbz2.BZ2_bzDecompressEnd(byref(self.bzs)) + def decompress(self, data): + """"decompress(data) -> string + + Provide more data to the decompressor object. It will return chunks + of decompressed data whenever possible. If you try to decompress data + after the end of stream is found, EOFError will be raised. If any data + was found after the end of stream, it'll be ignored and saved in + unused_data attribute.""" + + if not self.running: + raise OperationError(self.space.w_EOFError, + self.space.wrap("end of stream was already found")) + + in_bufsize = len(data) + in_buf = create_string_buffer(in_bufsize) + in_buf.value = data + + out_bufsize = SMALLCHUNK + out_buf = create_string_buffer(out_bufsize) + + self.bzs.next_in = in_buf + self.bzs.avail_in = in_bufsize + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + temp = [] + while True: + bzerror = libbz2.BZ2_bzDecompress(byref(self.bzs)) + if bzerror == BZ_STREAM_END: + if self.bzs.avail_in != 0: + unused = [self.bzs.next_in[i] for i in range(self.bzs.avail_in)] + self.unused_data = "".join(unused) + self.running = False + break + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + if self.bzs.avail_in == 0: + break + elif self.bzs.avail_out == 0: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = create_string_buffer(out_bufsize) + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + if temp: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) + temp.append(data) + return self.space.wrap("".join(temp)) + + total_out = _bzs_total_out(self.bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + return self.space.wrap(res) + decompress.unwrap_spec = ['self', str] + _BZ2Decomp.typedef = TypeDef("_BZ2Decomp", unused_data = interp_attrproperty("unused_data", _BZ2Decomp), + decompress = interp2app(_BZ2Decomp.decompress, + unwrap_spec=_BZ2Decomp.decompress.unwrap_spec), ) Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 12:49:31 2006 @@ -674,6 +674,17 @@ bz2d = BZ2Decompressor() assert bz2d.unused_data == "" + def test_decompress(self): + from bz2 import BZ2Decompressor + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2d = BZ2Decompressor() + raises(TypeError, bz2d.decompress) + decompressed_data = bz2d.decompress(DATA) + assert decompressed_data == TEXT + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -693,16 +704,6 @@ # def decompress(self, data): # return bz2.decompress(data) # -# class BZ2DecompressorTest(BaseTest): -# def test_Constructor(self): -# self.assertRaises(TypeError, BZ2Decompressor, 42) -# -# def testDecompress(self): -# # "Test BZ2Decompressor.decompress()" -# bz2d = BZ2Decompressor() -# self.assertRaises(TypeError, bz2d.decompress) -# text = bz2d.decompress(self.DATA) -# self.assertEqual(text, self.TEXT) # # def testDecompressChunks10(self): # # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" From rhymes at codespeak.net Mon Aug 7 13:53:22 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 13:53:22 +0200 (CEST) Subject: [pypy-svn] r31103 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807115322.E3D231006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 13:53:18 2006 New Revision: 31103 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: fix to decompress in really small chunks Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 13:53:18 2006 @@ -980,7 +980,7 @@ return self.space.wrap("".join(temp)) total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out)]) + res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) return self.space.wrap(res) decompress.unwrap_spec = ['self', str] Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 13:53:18 2006 @@ -685,6 +685,24 @@ decompressed_data = bz2d.decompress(DATA) assert decompressed_data == TEXT + def test_decompress_chunks_10(self): + from bz2 import BZ2Decompressor + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2d = BZ2Decompressor() + decompressed_data = "" + n = 0 + while True: + temp = DATA[n * 10:(n + 1) * 10] + if not temp: + break + decompressed_data = "%s%s" % (decompressed_data, bz2d.decompress(temp)) + n += 1 + + assert decompressed_data == TEXT + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -703,20 +721,6 @@ # # isn't available to run. # def decompress(self, data): # return bz2.decompress(data) -# -# -# def testDecompressChunks10(self): -# # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" -# bz2d = BZ2Decompressor() -# text = '' -# n = 0 -# while 1: -# str = self.DATA[n*10:(n+1)*10] -# if not str: -# break -# text += bz2d.decompress(str) -# n += 1 -# self.assertEqual(text, self.TEXT) # # def testDecompressUnusedData(self): # # "Test BZ2Decompressor.decompress() with unused data" From auc at codespeak.net Mon Aug 7 13:55:20 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Mon, 7 Aug 2006 13:55:20 +0200 (CEST) Subject: [pypy-svn] r31104 - pypy/dist/pypy/objspace/test Message-ID: <20060807115520.AB7251006E@code0.codespeak.net> Author: auc Date: Mon Aug 7 13:55:18 2006 New Revision: 31104 Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: a couple of new tests Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Mon Aug 7 13:55:18 2006 @@ -15,8 +15,6 @@ assert not is_free(X) assert is_bound(X) assert is_bound(1) - # FIXME : propagate proper - # FailureException raises(RebindingError, bind, X, 2) def test_bind_to_self(self): @@ -224,7 +222,6 @@ bind(X, 42); schedule() # gc ... def test_one_future_exception(self): - print "one future exception", sched_info() class FooException(Exception): pass def poop(X): @@ -241,7 +238,6 @@ assert False def test_exception_in_chain(self): - print "exception in chain", sched_info() class FooException(Exception): pass def raise_foo(): @@ -265,7 +261,6 @@ assert False def test_exception_in_group(self): - print "exception in groups", sched_info() class FooException(Exception): pass def loop_or_raise(Canary, crit, Bomb_signal): @@ -297,7 +292,6 @@ """check that a wait nested in a tree of threads works correctly """ - print "nested threads", sched_info() def sleep(X): wait(X) return X @@ -313,7 +307,6 @@ assert v == 42 def test_wait_needed(self): - print "wait_needed", sched_info() X = newvar() def binder(V): @@ -331,7 +324,6 @@ schedule() # gc help def test_eager_producer_consummer(self): - print "eager_producer_consummer", sched_info() def generate(n, limit, R): if n < limit: @@ -357,7 +349,6 @@ def test_lazy_producer_consummer(self): - print "lazy_producer_consummer", sched_info() def lgenerate(n, L): """wait-needed version of generate""" @@ -462,8 +453,150 @@ assert count[0] == max_spawn + erring try: wait(Failed) - except Exception, e: # Unification Failure + except RebindingError, e: assert len(sched_info()['threads']) == 1 return assert False + def test_nd_append(self): + skip("non determnistic choice: yet to come") + #from CTM p.639 + """ + def append(A, B, C): + choice: + unify(A, None) + unify(B, C) + or: + As, Bs, X = newvar(), newvar(), newvar() + unify(A, (X, As)) + unify(C, (X, Cs)) + append(As, B, Cs) + """ + from solver import solve + X, Y, S = newvar(), newvar(), newvar() + unify((X, Y), S) + + for sol in solve(lambda : append(X, Y, [1, 2, 3])): + assert sol in ((None, [1, 2, 3]), + ([1], [2, 3]), + ([1, 2], [3]), + ([1, 2, 3], None)) + + + def test_stream_merger(self): + """this is a little cheesy, due to threads not + being preemptively scheduled + """ + + def _sleep(X, Barrier): + wait(X) + unify(Barrier, True) + + def wait_two(X, Y): + Barrier = newvar() + stacklet(_sleep, X, Barrier) + stacklet(_sleep, Y, Barrier) + wait(Barrier) + if is_free(Y): + return 1 + return 2 + + def stream_merger(S1, S2): + F = newvar() + unify(F, wait_two(S1, S2)) + if F==1: + if S1 is None: + return S2 + else: + M, NewS1 = newvar(), newvar() + unify((M, NewS1), S1) + return (M, stream_merger(S2, NewS1)) + elif F==2: + if S2 is None: + return S1 + else: + M, NewS2 = newvar(), newvar() + unify((M, NewS2), S2) + return (M, stream_merger(NewS2, S1)) + + + def feed_stream(S, values): + for v in values: + N = newvar() + bind(S, (v, N)) + S = N # yeah, tail recursion is cooler + schedule() + bind(S, None) + + S1, S2 = newvar(), newvar() + + O = future(stream_merger, S1, S2) + + stacklet(feed_stream, S2, ['foo', 'bar', 'quux', 'spam', 'eggs']) + stacklet(feed_stream, S1, range(10)) + + assert O == ('foo', (0, ('bar', (1, ('quux', (2, ('spam', + (3, ('eggs', (4, (5, (6, (7, (8, (9, None))))))))))))))) + + + def test_digital_logic(self): + + def print_stream(S): + elts = [] + while is_bound(S): + if S is None: + break + elts.append(str(S[0])) + S = S[1] + print '[%s]' % ','.join(elts) + + def bound_part(S): + elts = [] + while is_bound(S): + if S is None: + break + elts.append(S[0]) + S = S[1] + return elts + + + def gatemaker(Fun): + def _(X, Y): + def gateloop(X, Y, R): + Xs, Ys, Xr, Yr = newvar(), newvar(), newvar(), newvar() + unify((X, Y), + ((Xs, Xr), (Ys, Yr))) + Rs = newvar() + unify(R, (Fun(Xs, Ys), Rs)) + gateloop(Xr, Yr, Rs) + R = newvar() + stacklet(gateloop, X, Y, R) + return R + return _ + + andg = gatemaker(lambda X, Y: X*Y,) + org = gatemaker(lambda X, Y: X+Y-X*Y) + xorg = gatemaker(lambda X, Y: X+Y-2*X*Y) + + def full_adder(X, Y, Z, C, S): + K, L, M = newvar(), newvar(), newvar() + unify(K, andg(X, Y)) + unify(L, andg(Y, Z)) + unify(M, andg(X, Z)) + unify(C, org(K, org(L, M))) + unify(S, xorg(Z, xorg(X, Y))) + + X, Y, Z, C, S = newvar(), newvar(), newvar(), newvar(), newvar() + + unify(X, (1, (1, (0, newvar())))) + unify(Y, (0, (1, (0, newvar())))) + unify(Z, (1, (1, (1, newvar())))) + + + stacklet(full_adder, X, Y, Z, C, S) + wait(C); wait(S) + assert bound_part(C) == [1, 1, 0] + assert bound_part(S) == [0, 1, 1] + schedule() + + reset_scheduler() # free all the hanging threads From rhymes at codespeak.net Mon Aug 7 13:59:15 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 13:59:15 +0200 (CEST) Subject: [pypy-svn] r31105 - pypy/dist/pypy/module/bz2/test Message-ID: <20060807115915.BBA5B10071@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 13:59:13 2006 New Revision: 31105 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: more tests for BZ2Decompressor Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 13:59:13 2006 @@ -702,6 +702,28 @@ n += 1 assert decompressed_data == TEXT + + def test_decompress_unused_data(self): + # test with unused data. (data after EOF) + from bz2 import BZ2Decompressor + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + bz2d = BZ2Decompressor() + unused_data = "this is unused data" + decompressed_data = bz2d.decompress(DATA + unused_data) + assert decompressed_data == TEXT + assert bz2d.unused_data == unused_data + + def test_EOF_error(self): + from bz2 import BZ2Decompressor + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + + bz2d = BZ2Decompressor() + bz2d.decompress(DATA) + raises(EOFError, bz2d.decompress, "foo") # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # @@ -721,22 +743,7 @@ # # isn't available to run. # def decompress(self, data): # return bz2.decompress(data) -# -# def testDecompressUnusedData(self): -# # "Test BZ2Decompressor.decompress() with unused data" -# bz2d = BZ2Decompressor() -# unused_data = "this is unused data" -# text = bz2d.decompress(self.DATA+unused_data) -# self.assertEqual(text, self.TEXT) -# self.assertEqual(bz2d.unused_data, unused_data) -# -# def testEOFError(self): -# # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError" -# bz2d = BZ2Decompressor() -# text = bz2d.decompress(self.DATA) -# self.assertRaises(EOFError, bz2d.decompress, "anything") -# -# +# # class FuncTest(BaseTest): # "Test module functions" # From rhymes at codespeak.net Mon Aug 7 15:23:08 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 15:23:08 +0200 (CEST) Subject: [pypy-svn] r31106 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807132308.AB9871006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 15:23:04 2006 New Revision: 31106 Modified: pypy/dist/pypy/module/bz2/__init__.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: one-shot module level compress() function implemented. Modified: pypy/dist/pypy/module/bz2/__init__.py ============================================================================== --- pypy/dist/pypy/module/bz2/__init__.py (original) +++ pypy/dist/pypy/module/bz2/__init__.py Mon Aug 7 15:23:04 2006 @@ -5,6 +5,7 @@ 'BZ2File': 'interp_bz2.BZ2File', 'BZ2Compressor': 'interp_bz2.BZ2Compressor', 'BZ2Decompressor': 'interp_bz2.BZ2Decompressor', + 'compress': 'interp_bz2.compress', } appleveldefs = { Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 15:23:04 2006 @@ -992,6 +992,69 @@ ) +def compress(space, data, compresslevel=9): + """compress(data [, compresslevel=9]) -> string + + Compress data in one shot. If you want to compress data sequentially, + use an instance of BZ2Compressor instead. The compresslevel parameter, if + given, must be a number between 1 and 9.""" + + if compresslevel < 1 or compresslevel > 9: + raise OperationError(self.space.w_ValueError, + self.space.wrap("compresslevel must be between 1 and 9")) + + bzs = bz_stream() + + in_bufsize = len(data) + # conforming to bz2 manual, this is large enough to fit compressed + # data in one shot. We will check it later anyway. + out_bufsize = in_bufsize + (in_bufsize / 100 + 1) + 600 + + out_buf = create_string_buffer(out_bufsize) + in_buf = create_string_buffer(in_bufsize) + in_buf.value = data + + self.bzs.next_in = in_buf + self.bzs.avail_in = in_bufsize + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + while True: + bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) + if bzerror == BZ_STREAM_END: + break + elif bzerror != BZ_FINISH_OK: + libbz2.BZ2_bzCompressEnd(byref(self.bzs)) + _catch_bz2_error(self.space, bzerror) + + if self.bzs.avail_out == 0: + data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = create_string_buffer(out_bufsize) + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + if temp: + res = "".join(temp) + + if self.bzs.avail_out: + size = _bzs_total_out(self.bzs) - total_out + res = "".join([out_buf[i] for i in range(size)]) + else: + total_out = _bzs_total_out(self.bzs) + res = "".join([out_buf[i] for i in range(total_out)]) + + libbz2.BZ2_bzCompressEnd(byref(self.bzs)) + return self.space.wrap(res) +compress.unwrap_spec = [ObjSpace, str, int] + + def BZ2Compressor(space, compresslevel=9): """BZ2Compressor([compresslevel=9]) -> compressor object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 15:23:04 2006 @@ -724,6 +724,30 @@ bz2d = BZ2Decompressor() bz2d.decompress(DATA) raises(EOFError, bz2d.decompress, "foo") + +def test_compress_function(): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import compress + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + raises(TypeError, compress, 123) + raises(ValueError, compress, "foo", 10) + raises(TypeError, compress, "foo", "foo") + + data = compress(TEXT) + assert decompress(data) == TEXT # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # @@ -744,14 +768,6 @@ # def decompress(self, data): # return bz2.decompress(data) # -# class FuncTest(BaseTest): -# "Test module functions" -# -# def testCompress(self): -# # "Test compress() function" -# data = bz2.compress(self.TEXT) -# self.assertEqual(self.decompress(data), self.TEXT) -# # def testDecompress(self): # # "Test decompress() function" # text = bz2.decompress(self.DATA) From rhymes at codespeak.net Mon Aug 7 15:29:21 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 15:29:21 +0200 (CEST) Subject: [pypy-svn] r31107 - pypy/dist/pypy/module/bz2/test Message-ID: <20060807132921.655791006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 15:29:18 2006 New Revision: 31107 Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: one more test for compress() function Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 15:29:18 2006 @@ -724,7 +724,8 @@ bz2d = BZ2Decompressor() bz2d.decompress(DATA) raises(EOFError, bz2d.decompress, "foo") - + + def test_compress_function(): def decompress(data): import popen2 @@ -749,6 +750,27 @@ data = compress(TEXT) assert decompress(data) == TEXT +def test_compress_function_huge_data(): + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + from bz2 import compress + + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + HUGE_DATA = TEXT * 10000 + + data = compress(HUGE_DATA) + assert decompress(data) == HUGE_DATA + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: From rhymes at codespeak.net Mon Aug 7 15:49:17 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 15:49:17 +0200 (CEST) Subject: [pypy-svn] r31108 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060807134917.DF4B11006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 15:49:14 2006 New Revision: 31108 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: decompress() function. this is the last functionality of bz2 module :-) Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 15:49:14 2006 @@ -1054,6 +1054,67 @@ return self.space.wrap(res) compress.unwrap_spec = [ObjSpace, str, int] +def decompress(space, data): + """decompress(data) -> decompressed data + + Decompress data in one shot. If you want to decompress data sequentially, + use an instance of BZ2Decompressor instead.""" + + in_bufsize = len(data) + if in_bufsize == 0: + return self.space.wrap("") + + bzs = bz_stream() + + in_buf = create_string_buffer(in_bufsize) + in_buf.value = data + + out_bufsize = SMALLCHUNK + out_buf = create_string_buffer(out_bufsize) + + self.bzs.next_in = in_buf + self.bzs.avail_in = in_bufsize + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0) + if bzerror != BZ_OK: + _catch_bz2_error(self.space, bzerror) + + temp = [] + while True: + bzerror = libbz2.BZ2_bzDecompress(byref(bzs)) + if bzerror == BZ_STREAM_END: + break + if bzerror != BZ_OK: + libbz2.BZ2_bzDecompressEnd(byref(bzs)) + _catch_bz2_error(self.space, bzerror) + + if self.bzs.avail_in == 0: + libbz2.BZ2_bzDecompressEnd(byref(bzs)) + raise OperationError(space.w_ValueError, + space.wrap("couldn't find end of stream")) + elif self.bzs.avail_out == 0: + total_out = _bzs_total_out(self.bzs) + data = "".join([out_buf[i] for i in range(total_out)]) + temp.append(data) + + out_bufsize = _new_buffer_size(out_bufsize) + out_buf = create_string_buffer(out_bufsize) + self.bzs.next_out = out_buf + self.bzs.avail_out = out_bufsize + + total_out = _bzs_total_out(self.bzs) + if temp: + data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) + temp.append(data) + res = "".join(temp) + else: + res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) + + libbz2.BZ2_bzDecompressEnd(byref(bzs)) + return self.space.wrap(res) +decompress.unwrap_spec = [ObjSpace, str] def BZ2Compressor(space, compresslevel=9): """BZ2Compressor([compresslevel=9]) -> compressor object Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 15:49:14 2006 @@ -771,6 +771,25 @@ data = compress(HUGE_DATA) assert decompress(data) == HUGE_DATA +def test_decompress_function(): + from bz2 import decompress + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + + raises(TypeError, decompress) + assert decompress("") == "" + decompressed_data = decompress(DATA) + assert decompressed_data == TEXT + +def test_decompress_function_incomplete_data(): + from bz2 import decompress + + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + + raises(ValueError, decompress, DATA[:-10]) + + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: @@ -789,17 +808,3 @@ # # isn't available to run. # def decompress(self, data): # return bz2.decompress(data) -# -# def testDecompress(self): -# # "Test decompress() function" -# text = bz2.decompress(self.DATA) -# self.assertEqual(text, self.TEXT) -# -# def testDecompressEmpty(self): -# # "Test decompress() function with empty string" -# text = bz2.decompress("") -# self.assertEqual(text, "") -# -# def testDecompressIncomplete(self): -# # "Test decompress() function with incomplete data" -# self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) From rhymes at codespeak.net Mon Aug 7 16:10:02 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 16:10:02 +0200 (CEST) Subject: [pypy-svn] r31109 - pypy/dist/pypy/module/bz2 Message-ID: <20060807141002.D30C21006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 16:10:00 2006 New Revision: 31109 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: rip out left out 'self'. I'm still wondering why test passed with misplaced self anyway... Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 16:10:00 2006 @@ -1000,8 +1000,8 @@ given, must be a number between 1 and 9.""" if compresslevel < 1 or compresslevel > 9: - raise OperationError(self.space.w_ValueError, - self.space.wrap("compresslevel must be between 1 and 9")) + raise OperationError(space.w_ValueError, + space.wrap("compresslevel must be between 1 and 9")) bzs = bz_stream() @@ -1014,44 +1014,44 @@ in_buf = create_string_buffer(in_bufsize) in_buf.value = data - self.bzs.next_in = in_buf - self.bzs.avail_in = in_bufsize - self.bzs.next_out = out_buf - self.bzs.avail_out = out_bufsize + bzs.next_in = in_buf + bzs.avail_in = in_bufsize + bzs.next_out = out_buf + bzs.avail_out = out_bufsize - bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0) + bzerror = libbz2.BZ2_bzCompressInit(byref(bzs), compresslevel, 0, 0) if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + _catch_bz2_error(space, bzerror) while True: - bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH) + bzerror = libbz2.BZ2_bzCompress(byref(bzs), BZ_FINISH) if bzerror == BZ_STREAM_END: break elif bzerror != BZ_FINISH_OK: - libbz2.BZ2_bzCompressEnd(byref(self.bzs)) - _catch_bz2_error(self.space, bzerror) + libbz2.BZ2_bzCompressEnd(byref(bzs)) + _catch_bz2_error(space, bzerror) - if self.bzs.avail_out == 0: - data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) + if bzs.avail_out == 0: + data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))]) temp.append(data) out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf - self.bzs.avail_out = out_bufsize + bzs.next_out = out_buf + bzs.avail_out = out_bufsize if temp: res = "".join(temp) - if self.bzs.avail_out: - size = _bzs_total_out(self.bzs) - total_out + if bzs.avail_out: + size = _bzs_total_out(bzs) - total_out res = "".join([out_buf[i] for i in range(size)]) else: - total_out = _bzs_total_out(self.bzs) + total_out = _bzs_total_out(bzs) res = "".join([out_buf[i] for i in range(total_out)]) - libbz2.BZ2_bzCompressEnd(byref(self.bzs)) - return self.space.wrap(res) + libbz2.BZ2_bzCompressEnd(byref(bzs)) + return space.wrap(res) compress.unwrap_spec = [ObjSpace, str, int] def decompress(space, data): @@ -1062,7 +1062,7 @@ in_bufsize = len(data) if in_bufsize == 0: - return self.space.wrap("") + return space.wrap("") bzs = bz_stream() @@ -1072,14 +1072,14 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - self.bzs.next_in = in_buf - self.bzs.avail_in = in_bufsize - self.bzs.next_out = out_buf - self.bzs.avail_out = out_bufsize + bzs.next_in = in_buf + bzs.avail_in = in_bufsize + bzs.next_out = out_buf + bzs.avail_out = out_bufsize - bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0) + bzerror = libbz2.BZ2_bzDecompressInit(byref(bzs), 0, 0) if bzerror != BZ_OK: - _catch_bz2_error(self.space, bzerror) + _catch_bz2_error(space, bzerror) temp = [] while True: @@ -1088,23 +1088,23 @@ break if bzerror != BZ_OK: libbz2.BZ2_bzDecompressEnd(byref(bzs)) - _catch_bz2_error(self.space, bzerror) + _catch_bz2_error(space, bzerror) - if self.bzs.avail_in == 0: + if bzs.avail_in == 0: libbz2.BZ2_bzDecompressEnd(byref(bzs)) raise OperationError(space.w_ValueError, space.wrap("couldn't find end of stream")) - elif self.bzs.avail_out == 0: - total_out = _bzs_total_out(self.bzs) + elif bzs.avail_out == 0: + total_out = _bzs_total_out(bzs) data = "".join([out_buf[i] for i in range(total_out)]) temp.append(data) out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf - self.bzs.avail_out = out_bufsize + bzs.next_out = out_buf + bzs.avail_out = out_bufsize - total_out = _bzs_total_out(self.bzs) + total_out = _bzs_total_out(bzs) if temp: data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) temp.append(data) @@ -1113,7 +1113,7 @@ res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) libbz2.BZ2_bzDecompressEnd(byref(bzs)) - return self.space.wrap(res) + return space.wrap(res) decompress.unwrap_spec = [ObjSpace, str] def BZ2Compressor(space, compresslevel=9): From rhymes at codespeak.net Mon Aug 7 16:21:10 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 16:21:10 +0200 (CEST) Subject: [pypy-svn] r31110 - pypy/dist/pypy/module/bz2 Message-ID: <20060807142110.25CDA1006E@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 16:21:07 2006 New Revision: 31110 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: small fixes Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Mon Aug 7 16:21:07 2006 @@ -1023,6 +1023,8 @@ if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) + total_out = _bzs_total_out(bzs) + temp = [] while True: bzerror = libbz2.BZ2_bzCompress(byref(bzs), BZ_FINISH) if bzerror == BZ_STREAM_END: From auc at codespeak.net Mon Aug 7 16:49:14 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Mon, 7 Aug 2006 16:49:14 +0200 (CEST) Subject: [pypy-svn] r31112 - in pypy/dist/pypy/objspace: . cclp test Message-ID: <20060807144914.582A010075@code0.codespeak.net> Author: auc Date: Mon Aug 7 16:49:09 2006 New Revision: 31112 Added: pypy/dist/pypy/objspace/cclp/global_state.py Modified: pypy/dist/pypy/objspace/cclp/misc.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: comp. space reintroduced, ask() Added: pypy/dist/pypy/objspace/cclp/global_state.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/global_state.py Mon Aug 7 16:49:09 2006 @@ -0,0 +1,3 @@ + +scheduler = [] + Modified: pypy/dist/pypy/objspace/cclp/misc.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/misc.py (original) +++ pypy/dist/pypy/objspace/cclp/misc.py Mon Aug 7 16:49:09 2006 @@ -5,7 +5,7 @@ import os -NO_DEBUG_INFO = [False] +NO_DEBUG_INFO = [True] def w(*msgs): """writeln""" if NO_DEBUG_INFO[0]: return Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Mon Aug 7 16:49:09 2006 @@ -4,26 +4,25 @@ from pypy.objspace.cclp.types import W_Var, W_FailedValue, aliases from pypy.objspace.cclp.misc import w, v, ClonableCoroutine -#from pypy.objspace.cclp.space import CSpace - -scheduler = [] +from pypy.objspace.cclp.space import W_CSpace +from pypy.objspace.cclp.global_state import scheduler #-- Singleton scheduler ------------------------------------------------ +class FunnyBoat: pass + class Scheduler(object): def __init__(self, space): self.space = space self._main = ClonableCoroutine.w_getcurrent(space) - # link top_level space to main coroutine - #self.top_space = CSpace(space, self._main) - #self._main.cspace = self.top_space - # ... self._init_head(self._main) self._init_blocked() self._switch_count = 0 - self._traced = {} - w ("MAIN THREAD = ", str(id(self._main))) + # more accounting + self._per_space_live_threads = {} # space -> nb runnable threads + self._traced = {} # thread -> vars + w("MAIN THREAD = ", str(id(self._main))) def get_threads(self): threads = [self._head] @@ -35,8 +34,10 @@ def _init_blocked(self): self._blocked = {} # thread set + # variables suspension lists self._blocked_on = {} # var -> threads self._blocked_byneed = {} # var -> threads + self._asking = {} # thread -> cspace def _init_head(self, thread): assert isinstance(thread, ClonableCoroutine) @@ -81,13 +82,37 @@ r = thread._next l._next = r r._prev = l + self._head = r if r == thread: #XXX write a test for me ! if not we_are_translated(): import traceback traceback.print_exc() self.display_head() - thread._next = thread._next = None - return thread + thread._next = thread._prev = FunnyBoat + # cspace/threads account mgmt + if thread._cspace is not None: + count = self._per_space_live_threads[thread._cspace] + assert count > 0 + self._per_space_live_threads[thread._cspace] = count - 1 + if count == 1: + del self._per_space_live_threads[thread._cspace] + +# return thread + + #-- cspace helper + + def is_stable(self, cspace): + if not self._per_space_live_threads.has_key(cspace): + return True + return self._per_space_live_threads[cspace] == 0 + + def wait_stable(self, cspace): + if self.is_stable(cspace): + return + curr = ClonableCoroutine.w_getcurrent(self.space) + self._asking[curr] = cspace + self._blocked[curr] = True + self.schedule() #-- to be used by logic objspace @@ -117,6 +142,7 @@ or to_be_run.is_dead() \ or (to_be_run == current): to_be_run = to_be_run._next + assert isinstance(to_be_run, ClonableCoroutine) if to_be_run == sentinel: if not dont_pass: return ClonableCoroutine.w_getcurrent(self.space) @@ -127,6 +153,11 @@ w(".. SCHEDULER reinitialized") raise OperationError(self.space.w_AllBlockedError, self.space.wrap("can't schedule, possible deadlock in sight")) + if to_be_run in self._asking: + if self.is_stable(self._asking[to_be_run]): + del self._asking[to_be_run] + del self._blocked[to_be_run] + self._head = to_be_run return to_be_run #XXX call me directly for this to work translated @@ -154,6 +185,11 @@ def add_new_thread(self, thread): "insert 'thread' at end of running queue" assert isinstance(thread, ClonableCoroutine) + if thread._cspace != None: + print "Yeeeep" + count = self._per_space_live_threads.get(thread._cspace, 0) + self._per_space_live_threads[thread._cspace] = count + 1 + assert len(self._per_space_live_threads) self._chain_insert(thread) def add_to_blocked_on(self, w_var, thread): @@ -168,6 +204,9 @@ self._blocked_on[w_var] = blocked blocked.append(thread) self._blocked[thread] = True + # cspace accounting + if thread._cspace is not None: + self._per_space_live_threads[thread._cspace] -= 1 def unblock_on(self, w_var): v(".. we UNBLOCK threads dependants of var", str(w_var)) @@ -177,7 +216,11 @@ blocked = self._blocked_on[w_var] del self._blocked_on[w_var] w(str([id(thr) for thr in blocked])) - for thr in blocked: del self._blocked[thr] + for thr in blocked: + del self._blocked[thr] + # cspace accounting + if thr._cspace is not None: + self._per_space_live_threads[thr._cspace] += 1 def add_to_blocked_byneed(self, w_var, thread): w(".. we BLOCK BYNEED thread", str(id(thread)), "on var", str(w_var)) @@ -190,6 +233,9 @@ self._blocked_byneed[w_var] = blocked blocked.append(thread) self._blocked[thread] = True + # cspace accounting + if thread._cspace is not None: + self._per_space_live_threads[thread._cspace] += 1 def unblock_byneed_on(self, w_var): v(".. we UNBLOCK BYNEED dependants of var", str(w_var)) @@ -201,7 +247,12 @@ del self._blocked_byneed[w_alias] w_alias.needed = True w(str([id(thr) for thr in blocked])) - for thr in blocked: del self._blocked[thr] + for thr in blocked: + del self._blocked[thr] + # cspace accounting + if thr._cspace is not None: + self._per_space_live_threads[thr._cspace] -= 1 + # Logic Variables tracing, helps exception propagation # amongst threads @@ -244,6 +295,31 @@ return w_ret app_sched_info = gateway.interp2app(sched_info) +def sched_all(space): + s = scheduler[0] + si = space.setitem + sw = space.wrap + w_ret = space.newdict([]) + if not we_are_translated(): + si(w_ret, sw('threads'), + sw([id(th) for th in s.get_threads()])) + si(w_ret, sw('blocked_on'), + sw([(id(th), [id(th) for th in thl]) + for var, thl in s._blocked_on.items()])) + si(w_ret, sw('blocked_byneed'), + sw([(id(var), [id(th) for th in thl]) + for var, thl in s._blocked_byneed.items()])) + si(w_ret, sw('traced'), + sw([(id(th), [id(var) for var in lvar]) + for th, lvar in s._traced.items()])) + si(w_ret, sw('space_accounting'), + sw([(id(spc), count) + for spc, count in s._per_space_live_threads.items()])) + si(w_ret, sw('asking'), + sw([(id(th), id(spc)) + for th, spc in s._asking.items()])) + return w_ret +app_sched_all = gateway.interp2app(sched_all) def schedule(space): "useful til we get preemtive scheduling deep into the vm" Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Mon Aug 7 16:49:09 2006 @@ -1,25 +1,52 @@ -from pypy.interpreter import baseobjspace +from pypy.rpython.objectmodel import we_are_translated +from pypy.interpreter import baseobjspace, gateway, argument, typedef from pypy.interpreter.error import OperationError -from pypy.objspace.cclp.misc import ClonableCoroutine +from pypy.objspace.cclp.misc import ClonableCoroutine, w +from pypy.objspace.cclp.thunk import FutureThunk, ProcedureThunk +from pypy.objspace.cclp.global_state import scheduler + +def newspace(space, w_callable, __args__): + try: + args = __args__.normalize() + # coro init + w_coro = ClonableCoroutine(space) + thunk = ProcedureThunk(space, w_callable, args, w_coro) + w_coro.bind(thunk) + if not we_are_translated(): + w("NEWSPACE, thread", str(id(w_coro)), "for", str(w_callable.name)) + + w_space = W_CSpace(space, w_coro, parent=w_coro._cspace) + w_coro._cspace = w_space + + scheduler[0].add_new_thread(w_coro) + scheduler[0].schedule() + except: + print "oh, uh" + + return w_space +app_newspace = gateway.interp2app(newspace, unwrap_spec=[baseobjspace.ObjSpace, + baseobjspace.W_Root, + argument.Arguments]) + +class W_CSpace(baseobjspace.Wrappable): + + def __init__(self, space, thread, parent=None): + assert isinstance(thread, ClonableCoroutine) + assert (parent is None) or isinstance(parent, CSpace) + self.space = space # the object space ;-) + self.parent = parent + self.main_thread = thread + + def w_ask(self): + scheduler[0].wait_stable(self) + return self.space.newint(0) + +W_CSpace.typedef = typedef.TypeDef("W_CSpace", + ask = gateway.interp2app(W_CSpace.w_ask)) + + -## def newspace(w_callable, __args__): -## w_coro = stacklet(w_callable, __args__) -## w_space = CSpace(coro, parent=coro.space) -## w_coro.space = space -## return w_space - -class CSpace(baseobjspace.Wrappable): - - def __init__(self, space, distributor, parent=None): - pass -## assert isinstance(distributor, ClonableCoroutine) -## assert (parent is None) or isinstance(parent, CSpace) -## self.space = space # the object space ;-) -## self.parent = parent -## self.distributor = distributor -## self.threads = {} # the eventual other threads - ## def is_top_level(self): ## return self.parent is None @@ -28,9 +55,6 @@ ## #XXX return w_getcurrent().cspace ## pass -## def newspace(): -## #XXX fork ? -## pass ## def clone(self): ## if self.is_top_level(): Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Mon Aug 7 16:49:09 2006 @@ -4,7 +4,7 @@ from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue from pypy.objspace.cclp.misc import w, v, ClonableCoroutine from pypy.objspace.cclp.thunk import FutureThunk, ProcedureThunk -from pypy.objspace.cclp.scheduler import scheduler +from pypy.objspace.cclp.global_state import scheduler #-- Future -------------------------------------------------- @@ -21,7 +21,8 @@ w_Future = W_Future(space) thunk = FutureThunk(space, w_callable, args, w_Future, coro) coro.bind(thunk) - w("FUTURE", str(id(coro))) + if not we_are_translated(): + w("FUTURE", str(id(coro)), "for", str(w_callable.name)) scheduler[0].add_new_thread(coro) return w_Future app_future = gateway.interp2app(future, unwrap_spec=[baseobjspace.ObjSpace, @@ -39,7 +40,9 @@ #coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) coro.bind(thunk) - w("STACKLET", str(id(coro))) + print we_are_translated() + if not we_are_translated(): + w("STACKLET", str(id(coro)), "for", str(w_callable.name)) scheduler[0].add_new_thread(coro) scheduler[0].schedule() return coro Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Mon Aug 7 16:49:09 2006 @@ -1,6 +1,6 @@ from pypy.module._stackless.coroutine import _AppThunk from pypy.objspace.cclp.misc import w -from pypy.objspace.cclp.scheduler import scheduler +from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue #-- Thunk ----------------------------------------- Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Mon Aug 7 16:49:09 2006 @@ -5,7 +5,7 @@ from pypy.objspace.std.dictobject import W_DictObject from pypy.objspace.cclp.misc import w, v, ClonableCoroutine -from pypy.objspace.cclp.scheduler import scheduler +from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import deref, W_Var, W_Future, W_FailedValue Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Mon Aug 7 16:49:09 2006 @@ -16,8 +16,12 @@ from pypy.objspace.cclp.thread import app_future, app_stacklet, app_this_thread -from pypy.objspace.cclp.scheduler import Scheduler, scheduler, app_sched_info, \ - app_schedule, app_reset_scheduler +from pypy.objspace.cclp.scheduler import Scheduler, app_sched_info, \ + app_schedule, app_reset_scheduler, app_sched_all + +from pypy.objspace.cclp.global_state import scheduler + +from pypy.objspace.cclp.space import app_newspace, W_CSpace #-- VARIABLE ------------------------------------------------ @@ -188,6 +192,7 @@ # multimethods hack space.model.typeorder[W_Var] = [(W_Var, None), (W_Root, None)] # None means no conversion space.model.typeorder[W_Future] = [(W_Future, None), (W_Var, None)] + space.model.typeorder[W_CSpace] = [(W_CSpace, None), (baseobjspace.Wrappable, None)] ## space.model.typeorder[W_FiniteDomain] = [(W_FiniteDomain, None), (W_Root, None)] @@ -256,12 +261,16 @@ space.wrap(app_wait_needed)) space.setitem(space.builtin.w_dict, space.wrap('sched_info'), space.wrap(app_sched_info)) + space.setitem(space.builtin.w_dict, space.wrap('sched_all'), + space.wrap(app_sched_all)) space.setitem(space.builtin.w_dict, space.wrap('schedule'), space.wrap(app_schedule)) space.setitem(space.builtin.w_dict, space.wrap('this_thread'), space.wrap(app_this_thread)) space.setitem(space.builtin.w_dict, space.wrap('reset_scheduler'), space.wrap(app_reset_scheduler)) + space.setitem(space.builtin.w_dict, space.wrap('newspace'), + space.wrap(app_newspace)) #-- misc ----- space.setitem(space.builtin.w_dict, space.wrap('interp_id'), Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Mon Aug 7 16:49:09 2006 @@ -600,3 +600,31 @@ schedule() reset_scheduler() # free all the hanging threads + + def test_newspace_ask_noop(self): + + def bar(X): return X + 42 + + X = newvar() + s = newspace(bar, X) + s.ask() + + def test_newspace_ask_wait(self): + + def quux(X): + while 1: + if is_bound(X): + break + schedule() + + def asker(cspace): + cspace.ask() + + X = newvar() + s = newspace(quux, X) + stacklet(asker, s) + unify(X, 42) + assert len(sched_all()['asking']) == 1 + schedule() # allow quux exit + schedule() # allow asker exit + assert len(sched_all()['asking']) == 0 From arigo at codespeak.net Mon Aug 7 16:49:23 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 16:49:23 +0200 (CEST) Subject: [pypy-svn] r31113 - in pypy/dist/pypy: objspace/flow translator/backendopt Message-ID: <20060807144923.1FB1C1007B@code0.codespeak.net> Author: arigo Date: Mon Aug 7 16:49:20 2006 New Revision: 31113 Modified: pypy/dist/pypy/objspace/flow/model.py pypy/dist/pypy/translator/backendopt/removenoops.py Log: A bug in removenoops, with an extra check in checkgraph() to catch it. Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Mon Aug 7 16:49:20 2006 @@ -613,6 +613,8 @@ else: assert v.value is not last_exception #assert v.value != last_exc_value + if op.opname == 'direct_call': + assert isinstance(op.args[0], Constant) definevar(op.result) exc_links = {} Modified: pypy/dist/pypy/translator/backendopt/removenoops.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/removenoops.py (original) +++ pypy/dist/pypy/translator/backendopt/removenoops.py Mon Aug 7 16:49:20 2006 @@ -23,7 +23,8 @@ for i in range(len(op.args)): if op.args[i] == op_result: op.args[i] = op_arg - if op.opname == "indirect_call" and op.args[0] == op_arg: + if (op.opname == "indirect_call" + and isinstance(op.args[0], Constant)): op.opname = "direct_call" op.args = op.args[:-1] for link in block.exits: From arigo at codespeak.net Mon Aug 7 16:51:13 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 16:51:13 +0200 (CEST) Subject: [pypy-svn] r31114 - in pypy/dist/pypy/translator/c: . src test Message-ID: <20060807145113.224B910053@code0.codespeak.net> Author: arigo Date: Mon Aug 7 16:51:11 2006 New Revision: 31114 Modified: pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/test/test_exception.py Log: OP_MALLOC_VARSIZE bug, generates segfaults under some conditions when trying to allocate very large arrays (just too large to fit in 32-bit). Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Mon Aug 7 16:51:11 2006 @@ -525,12 +525,14 @@ eresult = self.expr(op.result) if VARPART.OF is Void: # strange esize = 'sizeof(%s)' % (cdecl(typename, ''),) - result = '' + result = '{\n' else: itemtype = cdecl(itemtypename, '') - result = 'OP_MAX_VARSIZE(%s, %s);\n' % ( + result = 'IF_VARSIZE_OVERFLOW(%s, %s, %s)\nelse {\n' % ( elength, - itemtype) + itemtype, + eresult) + tail = '\n}' esize = 'sizeof(%s)-sizeof(%s)+%s*sizeof(%s)' % ( cdecl(typename, ''), itemtype, @@ -541,6 +543,7 @@ # ctypes Arrays have no length field if not VARPART._hints.get('nolength', False): result += '\nif(%s) %s->%s = %s;' % (eresult, eresult, lenfld, elength) + result += '\n}' return result def OP_RAW_MALLOC(self, op): Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Mon Aug 7 16:51:11 2006 @@ -27,11 +27,13 @@ to compute the largest allowed number of items in the array. */ #define MAXIMUM_MALLOCABLE_SIZE (LONG_MAX-4096) -#define OP_MAX_VARSIZE(numitems, itemtype) { \ +#define IF_VARSIZE_OVERFLOW(numitems, itemtype, r) \ if (((unsigned long)(numitems)) > \ - (MAXIMUM_MALLOCABLE_SIZE / sizeof(itemtype))) \ + (MAXIMUM_MALLOCABLE_SIZE / sizeof(itemtype))) { \ FAIL_EXCEPTION(PyExc_MemoryError, "addr space overflow"); \ - } + r = NULL; \ + } +/* else { ...} -- generated by funcgen.py */ /* XXX hack to initialize the refcount of global structures: officially, Modified: pypy/dist/pypy/translator/c/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_exception.py (original) +++ pypy/dist/pypy/translator/c/test/test_exception.py Mon Aug 7 16:51:11 2006 @@ -1,5 +1,7 @@ import py +import sys from pypy.translator.c.test import test_typed +from pypy.rpython.lltypesystem import lltype getcompiled = test_typed.TestTypedTestCase().getcompiled @@ -76,6 +78,43 @@ else: py.test.fail("f1(1) did not raise anything") +def test_memoryerror(): + # in rev 30717 this test causes a segfault on some Linux, but usually + # only after the test is run. It is caused by the following sequence + # of events in lltype.malloc(S, n): there is an OP_MAX_VARSIZE macro + # which figures out that the size asked for is too large and would + # cause a wrap-around, so it sets a MemoryError; but execution continues + # nevertheless and the next line is an OP_MALLOC instruction, which + # because of the wrap-around allocates for 's' an amount of bytes which + # falls precisely between 0 and offsetof(s, tail). It succeeds. Then + # the length field of s.tail is initialized - this overwrites random + # memory! And only then is the exception check performed, and the + # MemoryError is noticed. + A = lltype.Array(lltype.Signed) + S = lltype.GcStruct('S', ('a', lltype.Signed), + ('b', lltype.Signed), + ('c', lltype.Signed), + ('tail', A)) + def g(n, tag): + s = lltype.malloc(S, n) + tag.a = 42 + return s + def testfn(n=int): + tag = lltype.malloc(S, 0) + try: + s = g(n, tag) + result = s.tail[n//2] + except MemoryError: + result = 1000 + return result + tag.a + f1 = getcompiled(testfn) + assert f1(10) == 42 + assert f1(sys.maxint) == 1000 + for i in range(20): + assert f1((sys.maxint+1) // 2 - i) == 1000 + assert f1(sys.maxint // 2 - 16384) == 1000 + assert f1(sys.maxint // 2 + 16384) == 1000 + def test_assert(): def testfn(n=int): assert n >= 0 From arigo at codespeak.net Mon Aug 7 16:52:31 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 16:52:31 +0200 (CEST) Subject: [pypy-svn] r31115 - pypy/dist/pypy/rpython/rctypes/socketmodule/test Message-ID: <20060807145231.9CC3310053@code0.codespeak.net> Author: arigo Date: Mon Aug 7 16:52:30 2006 New Revision: 31115 Modified: pypy/dist/pypy/rpython/rctypes/socketmodule/test/test__socket.py Log: Minor fix. Modified: pypy/dist/pypy/rpython/rctypes/socketmodule/test/test__socket.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/socketmodule/test/test__socket.py (original) +++ pypy/dist/pypy/rpython/rctypes/socketmodule/test/test__socket.py Mon Aug 7 16:52:30 2006 @@ -60,9 +60,10 @@ s.connect(addr) self.client_addr = s.getsockname() s.send("@") - self.received = s.recv(1024) + self.received = data = s.recv(1024) while data: - self.received += s.recv(1024) + data = s.recv(1024) + self.received += data s.close() def test_getsockname(self): @@ -83,4 +84,4 @@ assert res == "@" assert clientaddr == self.client_addr client.close() - + s.close() From arigo at codespeak.net Mon Aug 7 16:54:10 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 16:54:10 +0200 (CEST) Subject: [pypy-svn] r31116 - in pypy/dist/pypy: annotation rpython/rctypes rpython/rctypes/test Message-ID: <20060807145410.D15D010053@code0.codespeak.net> Author: arigo Date: Mon Aug 7 16:54:08 2006 New Revision: 31116 Added: pypy/dist/pypy/rpython/rctypes/rfunc.py (contents, props changed) Modified: pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/rpython/rctypes/afunc.py pypy/dist/pypy/rpython/rctypes/avoid_p.py pypy/dist/pypy/rpython/rctypes/rmodel.py pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py pypy/dist/pypy/rpython/rctypes/test/test_rfunc.py pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py Log: rctypes support for passing ctypes function pointers around, calling them generically and casting between them and 'void*'. Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Mon Aug 7 16:54:08 2006 @@ -701,6 +701,11 @@ def is_true(cto): return SomeBool() + def simple_call(cto, *args_s): + # for variables containing ctypes function pointers + entry = extregistry.lookup_type(cto.knowntype) + return entry.compute_result_annotation(*args_s) + #_________________________________________ # memory addresses Modified: pypy/dist/pypy/rpython/rctypes/afunc.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/afunc.py (original) +++ pypy/dist/pypy/rpython/rctypes/afunc.py Mon Aug 7 16:54:08 2006 @@ -1,5 +1,6 @@ from pypy.annotation.model import SomeCTypesObject from pypy.annotation import model as annmodel +from pypy.annotation.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rpython.rctypes.implementation import CTypesEntry from pypy.rpython.lltypesystem import lltype @@ -10,17 +11,75 @@ CFuncPtrType = type(ctypes.CFUNCTYPE(None)) +class SomeCTypesFunc(annmodel.SomeBuiltin): + """Stands for a known constant ctypes function. Variables containing + potentially multiple ctypes functions are regular SomeCTypesObjects. + This is a separate annotation because some features are only supported + for calls to constant functions, like _rctypes_pyerrchecker_ and + functions with no declared argtypes. It also produces better code: + a direct_call instead of an indirect_call. + """ + def normalized(self): + ctype = normalized_func_ctype(self.const) + return cto_union(ctype, ctype) # -> SomeCTypesObject + +class __extend__(pairtype(SomeCTypesFunc, SomeCTypesFunc)): + def union((ctf1, ctf2)): + ctype1 = normalized_func_ctype(ctf1.const) + ctype2 = normalized_func_ctype(ctf2.const) + return cto_union(ctype1, ctype2) + +class __extend__(pairtype(SomeCTypesFunc, SomeCTypesObject)): + def union((ctf1, cto2)): + ctype1 = normalized_func_ctype(ctf1.const) + return cto_union(ctype1, cto2.knowntype) + +class __extend__(pairtype(SomeCTypesObject, SomeCTypesFunc)): + def union((cto1, ctf2)): + ctype2 = normalized_func_ctype(ctf2.const) + return cto_union(cto1.knowntype, ctype2) + + +def normalized_func_ctype(cfuncptr): + if getattr(cfuncptr, 'argtypes', None) is None: + raise annmodel.UnionError("cannot merge two ctypes functions " + "without declared argtypes") + return ctypes.CFUNCTYPE(cfuncptr.restype, + *cfuncptr.argtypes) + +def cto_union(ctype1, ctype2): + if ctype1 != ctype2: + raise annmodel.UnionError("a ctypes function object can only be " + "merged with another function with the same " + "signature") + return SomeCTypesObject(ctype1, ownsmemory=True) + + class CallEntry(CTypesEntry): """Annotation and rtyping of calls to external functions declared with ctypes. """ _metatype_ = CFuncPtrType + def compute_annotation(self): + #self.ctype_object_discovered() + func = self.instance + analyser = self.compute_result_annotation + methodname = getattr(func, '__name__', None) + return SomeCTypesFunc(analyser, methodname=methodname) + + def get_instance_sample(self): + if self.instance is not None: + return self.instance + else: + return self.type() # a sample NULL function object + def compute_result_annotation(self, *args_s): """ Answer the annotation of the external function's result """ - result_ctype = self.instance.restype + cfuncptr = self.get_instance_sample() + result_ctype = cfuncptr.restype if result_ctype is None: return None if result_ctype is ctypes.py_object: @@ -61,98 +120,15 @@ ## s_res)) def specialize_call(self, hop): - from pypy.rpython.rctypes.rmodel import CTypesValueRepr + from pypy.rpython.rctypes.rfunc import get_funcptr_constant + from pypy.rpython.rctypes.rfunc import rtype_funcptr_call cfuncptr = self.instance - fnname = cfuncptr.__name__ - - def repr_for_ctype(ctype): - s = SomeCTypesObject(ctype, ownsmemory=False) - r = hop.rtyper.getrepr(s) - return r - - args_r = [] - if getattr(cfuncptr, 'argtypes', None) is not None: - for ctype in cfuncptr.argtypes: - args_r.append(repr_for_ctype(ctype)) - else: - # unspecified argtypes: use ctypes rules for arguments - for s_arg, r_arg in zip(hop.args_s, hop.args_r): - if not isinstance(s_arg, SomeCTypesObject): - # accept integers, strings, or None - if isinstance(s_arg, annmodel.SomeInteger): - r_arg = repr_for_ctype(ctypes.c_long) - elif (isinstance(s_arg, annmodel.SomeString) - or s_arg == annmodel.s_None): - r_arg = repr_for_ctype(ctypes.c_char_p) - else: - raise TyperError("call with no argtypes: don't know " - "how to convert argument %r"%(s_arg,)) - args_r.append(r_arg) - - hop.rtyper.call_all_setups() - vlist = hop.inputargs(*args_r) - unwrapped_args_v = [] - ARGTYPES = [] - for r_arg, v in zip(args_r, vlist): - if isinstance(r_arg, CTypesValueRepr): - # ValueRepr case - unwrapped_args_v.append(r_arg.getvalue(hop.llops, v)) - ARGTYPES.append(r_arg.ll_type) - else: - # RefRepr case -- i.e. the function argument that we pass by - # value is e.g. a complete struct; we pass a pointer to it - # in the low-level graphs and it's up to the back-end to - # generate the correct dereferencing - unwrapped_args_v.append(r_arg.get_c_data(hop.llops, v)) - ARGTYPES.append(r_arg.c_data_type) - if cfuncptr.restype is not None: - s_res = SomeCTypesObject(cfuncptr.restype, ownsmemory=True) - r_res = hop.rtyper.getrepr(s_res) - RESTYPE = r_res.ll_type - else: - RESTYPE = lltype.Void - - kwds = {} - if hasattr(cfuncptr, 'llinterp_friendly_version'): - kwds['_callable'] = cfuncptr.llinterp_friendly_version - suppress_pyerr_occurred = False - if (cfuncptr._flags_ & ctypes._FUNCFLAG_PYTHONAPI) == 0: - suppress_pyerr_occurred = True - if hasattr(cfuncptr, '_rctypes_pyerrchecker_'): - suppress_pyerr_occurred = True - if suppress_pyerr_occurred: - kwds['includes'] = getattr(cfuncptr, 'includes', ()) - kwds['libraries'] = getattr(cfuncptr, 'libraries', ()) - #else: - # no 'includes': hack to trigger in GenC a PyErr_Occurred() check - - hop.exception_cannot_occur() - v_result = hop.llops.gencapicall(fnname, unwrapped_args_v, - resulttype = RESTYPE, - **kwds) - # XXX hack! hack! temporary! I promize! - FUNCTYPE = lltype.FuncType(ARGTYPES, RESTYPE) - last_op = hop.llops[-1] - assert last_op.opname == 'direct_call' - last_op.args[0].concretetype = lltype.Ptr(FUNCTYPE) - last_op.args[0].value._set_TYPE(last_op.args[0].concretetype) - last_op.args[0].value._set_T(FUNCTYPE) - last_op.args[0].value._obj._TYPE = FUNCTYPE - - if getattr(cfuncptr, '_rctypes_pyerrchecker_', None): - # special extension to support the CPyObjSpace - # XXX hackish: someone else -- like the annotator policy -- - # must ensure that this extra function has been annotated - from pypy.translator.translator import graphof - func = cfuncptr._rctypes_pyerrchecker_ - graph = graphof(hop.rtyper.annotator.translator, func) - hop.llops.record_extra_call(graph) - # build the 'direct_call' operation - f = hop.rtyper.getcallable(graph) - c = hop.inputconst(lltype.typeOf(f), f) - hop.genop('direct_call', [c]) - - if RESTYPE is lltype.Void: - return None - else: - return r_res.return_value(hop.llops, v_result) + v_funcptr, args_r, r_res = get_funcptr_constant(hop.rtyper, cfuncptr, + hop.args_s) + pyerrchecker = getattr(cfuncptr, '_rctypes_pyerrchecker_', None) + return rtype_funcptr_call(hop, v_funcptr, args_r, r_res, pyerrchecker) + + def get_repr(self, rtyper, s_funcptr): + # for variables containing ctypes function pointers + from pypy.rpython.rctypes.rfunc import CFuncPtrRepr + return CFuncPtrRepr(rtyper, s_funcptr) Modified: pypy/dist/pypy/rpython/rctypes/avoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/avoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/avoid_p.py Mon Aug 7 16:54:08 2006 @@ -4,6 +4,7 @@ from ctypes import c_void_p, c_int, POINTER, cast, c_char, c_char_p from pypy.rpython.rctypes.astringbuf import StringBufferType +from pypy.rpython.rctypes.afunc import CFuncPtrType, SomeCTypesFunc PointerType = type(POINTER(c_int)) @@ -34,7 +35,8 @@ _about_ = cast def checkptr(self, ctype): - assert isinstance(ctype, PointerType) or ctype == c_void_p, ( + assert (isinstance(ctype, PointerType) or ctype == c_void_p or + isinstance(ctype, CFuncPtrType)), ( "cast(): can only cast between pointers so far, not %r" % (ctype,)) def compute_result_annotation(self, s_arg, s_type): @@ -42,7 +44,8 @@ "cast(p, %r): argument 2 must be constant" % (s_type,)) type = s_type.const self.checkptr(type) - if s_arg.knowntype == StringBufferType: + if (s_arg.knowntype == StringBufferType or + isinstance(s_arg, SomeCTypesFunc)): pass else: self.checkptr(s_arg.knowntype) @@ -52,16 +55,23 @@ from pypy.rpython.rctypes.rpointer import PointerRepr from pypy.rpython.rctypes.rvoid_p import CVoidPRepr from pypy.rpython.rctypes.rstringbuf import StringBufRepr + from pypy.rpython.rctypes.rfunc import CFuncPtrRepr from pypy.rpython.lltypesystem import lltype, llmemory - assert isinstance(hop.args_r[0], (PointerRepr, CVoidPRepr, - StringBufRepr)) + r_arg = hop.args_r[0] + if isinstance(hop.args_s[0], SomeCTypesFunc): + # cast(const_cfuncptr, c_void_p): force the const_cfuncptr + # to become a general non-constant SomeCTypesObject + s_arg = hop.args_s[0].normalized() + r_arg = hop.rtyper.getrepr(s_arg) + assert isinstance(r_arg, (PointerRepr, CVoidPRepr, + StringBufRepr, CFuncPtrRepr)) targetctype = hop.args_s[1].const - v_box, c_targetctype = hop.inputargs(hop.args_r[0], lltype.Void) - if isinstance(hop.args_r[0], StringBufRepr): + v_box, c_targetctype = hop.inputargs(r_arg, lltype.Void) + if isinstance(r_arg, StringBufRepr): v_index = hop.inputconst(lltype.Signed, 0) - v_adr = hop.args_r[0].get_c_data_of_item(hop.llops, v_box, v_index) + v_adr = r_arg.get_c_data_of_item(hop.llops, v_box, v_index) else: - v_adr = hop.args_r[0].getvalue(hop.llops, v_box) + v_adr = r_arg.getvalue(hop.llops, v_box) if v_adr.concretetype != llmemory.Address: v_adr = hop.genop('cast_ptr_to_adr', [v_adr], resulttype = llmemory.Address) Added: pypy/dist/pypy/rpython/rctypes/rfunc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/rctypes/rfunc.py Mon Aug 7 16:54:08 2006 @@ -0,0 +1,194 @@ +from pypy.rpython.rtyper import inputconst +from pypy.rpython.rctypes.rmodel import CTypesValueRepr, CTypesRefRepr +from pypy.rpython.rctypes.afunc import CFuncPtrType +from pypy.rpython.error import TyperError +from pypy.rpython.lltypesystem import lltype +from pypy.annotation import model as annmodel +from pypy.annotation.model import SomeCTypesObject +from pypy.objspace.flow.model import Constant + +import ctypes + + +class CFuncPtrRepr(CTypesValueRepr): + + def __init__(self, rtyper, s_funcptr): + # For recursive types, getting the args_r and r_result is delayed + # until _setup_repr(). + ll_contents = lltype.Ptr(lltype.ForwardReference()) + super(CFuncPtrRepr, self).__init__(rtyper, s_funcptr, ll_contents) + self.sample = self.ctype() + self.argtypes = self.sample.argtypes + self.restype = self.sample.restype + if self.argtypes is None: + raise TyperError("cannot handle yet function pointers with " + "unspecified argument types") + + def _setup_repr(self): + # Find the repr and low-level type of the arguments and return value + rtyper = self.rtyper + args_r = [] + for arg_ctype in self.argtypes: + r = rtyper.getrepr(SomeCTypesObject(arg_ctype, + ownsmemory=False)) + args_r.append(r) + r_result = rtyper.getrepr(SomeCTypesObject(self.restype, + ownsmemory=True)) + if isinstance(self.ll_type.TO, lltype.ForwardReference): + FUNCTYPE = get_funcptr_type(args_r, r_result) + self.ll_type.TO.become(FUNCTYPE) + self.args_r = args_r + self.r_result = r_result + + def ctypecheck(self, value): + return (isinstance(value.__class__, CFuncPtrType) and + list(value.argtypes) == list(self.argtypes) and + value.restype == self.restype) + + def initialize_const(self, p, cfuncptr): + if not cfuncptr: # passed as arg to functions expecting func pointers + return + c, args_r, r_res = get_funcptr_constant(self.rtyper, cfuncptr, None) + p.c_data[0] = c.value + + def rtype_simple_call(self, hop): + v_box = hop.inputarg(self, arg=0) + v_funcptr = self.getvalue(hop.llops, v_box) + hop2 = hop.copy() + hop2.r_s_popfirstarg() + return rtype_funcptr_call(hop2, v_funcptr, self.args_r, self.r_result) + + +# ____________________________________________________________ + + +def get_funcptr_constant(rtyper, cfuncptr, args_s): + """Get a Constant ll function pointer from a ctypes function object. + """ + fnname = cfuncptr.__name__ + args_r, r_res = get_arg_res_repr(rtyper, cfuncptr, args_s) + FUNCTYPE = get_funcptr_type(args_r, r_res) + flags = get_funcptr_flags(cfuncptr) + f = lltype.functionptr(FUNCTYPE, fnname, **flags) + return inputconst(lltype.typeOf(f), f), args_r, r_res + + +def get_arg_res_repr(rtyper, cfuncptr, args_s): + """Get the reprs to use for the arguments and the return value of a + ctypes function call. The args_s annotations are used to guess the + argument types if they are not specified by cfuncptr.argtypes. + """ + def repr_for_ctype(ctype): + s = SomeCTypesObject(ctype, ownsmemory=False) + r = rtyper.getrepr(s) + return r + + args_r = [] + if getattr(cfuncptr, 'argtypes', None) is not None: + for ctype in cfuncptr.argtypes: + args_r.append(repr_for_ctype(ctype)) + else: + # unspecified argtypes: use ctypes rules for arguments, + # accepting integers, strings, or None + for s_arg in args_s: + if isinstance(s_arg, SomeCTypesObject): + r_arg = rtyper.getrepr(s_arg) + elif isinstance(s_arg, annmodel.SomeInteger): + r_arg = repr_for_ctype(ctypes.c_long) + elif (isinstance(s_arg, annmodel.SomeString) + or s_arg == annmodel.s_None): + r_arg = repr_for_ctype(ctypes.c_char_p) + else: + raise TyperError("call with no argtypes: don't know " + "how to convert argument %r" % (s_arg,)) + args_r.append(r_arg) + if cfuncptr.restype is not None: + s_res = SomeCTypesObject(cfuncptr.restype, ownsmemory=True) + r_res = rtyper.getrepr(s_res) + else: + r_res = None + return args_r, r_res + + +def get_funcptr_type(args_r, r_res): + """Get the lltype FUNCTYPE to use for a ctypes function call. + """ + ARGTYPES = [] + for r_arg in args_r: + if isinstance(r_arg, CTypesValueRepr): + # ValueRepr case + ARGTYPES.append(r_arg.ll_type) + else: + # RefRepr case -- i.e. the function argument that we pass by + # value is e.g. a complete struct + ARGTYPES.append(r_arg.c_data_type) + if r_res is not None: + RESTYPE = r_res.ll_type + else: + RESTYPE = lltype.Void + return lltype.FuncType(ARGTYPES, RESTYPE) + + +def get_funcptr_flags(cfuncptr): + """Get the fnptr flags to use for the given concrete ctypes function. + """ + kwds = {'external': 'C'} + if hasattr(cfuncptr, 'llinterp_friendly_version'): + kwds['_callable'] = cfuncptr.llinterp_friendly_version + suppress_pyerr_occurred = False + if (cfuncptr._flags_ & ctypes._FUNCFLAG_PYTHONAPI) == 0: + suppress_pyerr_occurred = True + if hasattr(cfuncptr, '_rctypes_pyerrchecker_'): + suppress_pyerr_occurred = True + if suppress_pyerr_occurred: + kwds['includes'] = getattr(cfuncptr, 'includes', ()) + kwds['libraries'] = getattr(cfuncptr, 'libraries', ()) + #else: + # no 'includes': hack to trigger in GenC a PyErr_Occurred() check + return kwds + + +def rtype_funcptr_call(hop, v_funcptr, args_r, r_res, pyerrchecker=None): + """Generate a call to the given ll function pointer. + """ + hop.rtyper.call_all_setups() + vlist = hop.inputargs(*args_r) + unwrapped_args_v = [] + for r_arg, v in zip(args_r, vlist): + if isinstance(r_arg, CTypesValueRepr): + # ValueRepr case + unwrapped_args_v.append(r_arg.getvalue(hop.llops, v)) + elif isinstance(r_arg, CTypesRefRepr): + # RefRepr case -- i.e. the function argument that we pass by + # value is e.g. a complete struct; we pass a pointer to it + # in the low-level graphs and it's up to the back-end to + # generate the correct dereferencing + unwrapped_args_v.append(r_arg.get_c_data(hop.llops, v)) + else: + assert 0, "ctypes func call got a non-ctypes arg repr" + + FUNCTYPE = v_funcptr.concretetype.TO + hop.exception_cannot_occur() + if isinstance(v_funcptr, Constant): + opname = 'direct_call' + else: + unwrapped_args_v.append(inputconst(lltype.Void, None)) + opname = 'indirect_call' + v_result = hop.genop(opname, [v_funcptr]+unwrapped_args_v, + resulttype = FUNCTYPE.RESULT) + + if pyerrchecker is not None: + # special extension to support the CPyObjSpace + # XXX hackish: someone else -- like the annotator policy -- + # must ensure that this extra function has been annotated + from pypy.translator.translator import graphof + graph = graphof(hop.rtyper.annotator.translator, pyerrchecker) + hop.llops.record_extra_call(graph) + # build the 'direct_call' operation + f = hop.rtyper.getcallable(graph) + c = hop.inputconst(lltype.typeOf(f), f) + hop.genop('direct_call', [c]) + + if r_res is not None: + v_result = r_res.return_value(hop.llops, v_result) + return v_result Modified: pypy/dist/pypy/rpython/rctypes/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rmodel.py (original) +++ pypy/dist/pypy/rpython/rctypes/rmodel.py Mon Aug 7 16:54:08 2006 @@ -69,8 +69,11 @@ of this object.""" return None + def ctypecheck(self, value): + return isinstance(value, self.ctype) + def convert_const(self, value): - if isinstance(value, self.ctype): + if self.ctypecheck(value): key = "by_id", id(value) keepalive = value else: @@ -82,6 +85,7 @@ try: return self.const_cache[key][0] except KeyError: + self.setup() p = lltype.malloc(self.r_memoryowner.lowleveltype.TO) self.initialize_const(p, value) if self.ownsmemory: @@ -229,7 +233,7 @@ get_c_data_or_value = getvalue def initialize_const(self, p, value): - if isinstance(value, self.ctype): + if self.ctypecheck(value): value = value.value p.c_data[0] = value Modified: pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c (original) +++ pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c Mon Aug 7 16:54:08 2006 @@ -61,6 +61,13 @@ return (void *)&p; } +typedef int (*fnptr_t)(point); + +EXPORT(fnptr_t) _testfunc_get_func(void) +{ + return _testfunc_struct; +} + DL_EXPORT(void) init_rctypes_test(void) { Modified: pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_ctypes.py Mon Aug 7 16:54:08 2006 @@ -227,3 +227,9 @@ for i, c in enumerate(struct.pack("l", 12345678)): p.contents[i] = ord(c) assert x.value == 12345678 + +def test_cfunctype_inspection(): + T = CFUNCTYPE(c_int, c_ubyte) + # T.argtypes and T.restype don't work, must use a dummy instance + assert list(T().argtypes) == [c_ubyte] + assert T().restype == c_int Modified: pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py Mon Aug 7 16:54:08 2006 @@ -17,6 +17,7 @@ from ctypes import cdll from ctypes import POINTER, Structure, c_int, byref, pointer, c_void_p +from ctypes import CFUNCTYPE # __________ compile and load our local test C file __________ @@ -102,6 +103,12 @@ testfunc_erase_type.restype = c_void_p testfunc_erase_type.argtypes = [] +# _testfunc_get_func +testfunc_get_func = _rctypes_test._testfunc_get_func +testfunc_get_func.restype = CFUNCTYPE(c_int, tagpoint) +testfunc_get_func.argtypes = [] +testfunc_get_func.includes = includes + def test_testfunc_struct(): in_point = tagpoint() @@ -142,6 +149,15 @@ assert pt._z == 100 return pt.x - pt.y # this test function is reused below +def test_testfunc_get_func(): + in_point = tagpoint() + in_point.x = -9171831 + in_point.y = 9171873 + fn = testfunc_get_func() + res = fn(in_point) + assert res == 42 + return res # this test function is reused below + class Test_annotation: def test_annotate_struct(self): t = TranslationContext() @@ -195,3 +211,7 @@ def test_compile_swap(self): fn = compile(test_testfunc_swap, []) assert fn() == 4 + + def test_compile_get_func(self): + fn = compile(test_testfunc_get_func, []) + assert fn() == 42 Modified: pypy/dist/pypy/rpython/rctypes/test/test_rfunc.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rfunc.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rfunc.py Mon Aug 7 16:54:08 2006 @@ -5,6 +5,7 @@ import py import sys import pypy.rpython.rctypes.implementation +from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.test_llinterp import interpret @@ -48,6 +49,15 @@ return result atoi.llinterp_friendly_version = ll_atoi +atol = mylib.atol +atol.restype = c_long +atol.argtypes = [c_char_p] +atol.llinterp_friendly_version = ll_atoi + +strlen = mylib.strlen +strlen.restype = c_long +strlen.argtypes = [c_char_p] + time_ = mylib.time time_.restype = c_long # should rather use ctypes_platform.getsimpletype() time_.argtypes = [POINTER(c_long)] @@ -218,6 +228,23 @@ ## assert a.binding(v1).knowntype == int ## assert a.binding(v2).knowntype == int + def test_annotate_indirect_call(self): + s = '442' + def f(n): + if n > 0: + f = strlen + else: + f = atol + return f + a = RPythonAnnotator() + s = a.build_types(f, [int]) + if conftest.option.view: + a.translator.view() + assert isinstance(s, annmodel.SomeCTypesObject) + sample = s.knowntype() + assert list(sample.argtypes) == [c_char_p] + assert sample.restype == c_long + class Test_specialization: def test_specialize_labs(self): res = interpret(test_labs, [-11]) @@ -397,3 +424,14 @@ fn = compile(f, []) assert fn() == string + def test_compile_indirect_call(self): + s = '442' + def f(n): + if n > 0: + f = strlen + else: + f = atol + return f(s) + fn = compile(f, [int]) + assert fn(1) == 3 + assert fn(0) == 442 Modified: pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py Mon Aug 7 16:54:08 2006 @@ -11,7 +11,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.test.test_llinterp import interpret -from ctypes import c_void_p, c_int, cast, pointer, POINTER +from ctypes import c_void_p, c_int, c_long, cast, pointer, POINTER from ctypes import c_char, c_byte, c_char_p, create_string_buffer, CFUNCTYPE class Test_annotation: @@ -92,3 +92,21 @@ fn = compile(func, []) assert fn() == 12 + + def test_compile_funcptr_as_void_p(self): + from pypy.rpython.rctypes.test.test_rfunc import labs + UNARYFN = CFUNCTYPE(c_long, c_long) + def func(n): + if n < -100: + p1 = c_void_p() # NULL void pointer - don't try! + else: + p1 = cast(labs, c_void_p) + p2 = cast(p1, UNARYFN) + return p2(n) + + assert func(-41) == 41 + assert func(72) == 72 + + fn = compile(func, [int]) + assert fn(-42) == 42 + assert fn(71) == 71 From arigo at codespeak.net Mon Aug 7 16:54:42 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 16:54:42 +0200 (CEST) Subject: [pypy-svn] r31117 - pypy/dist/pypy/module/mmap Message-ID: <20060807145442.7F0EC10053@code0.codespeak.net> Author: arigo Date: Mon Aug 7 16:54:41 2006 New Revision: 31117 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: munmap() takes a void* argument, not really a char*. Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Mon Aug 7 16:54:41 2006 @@ -65,7 +65,7 @@ libc.mmap.restype = c_void_p libc.close.argtypes = [c_int] libc.close.restype = c_int -libc.munmap.argtypes = [POINTER(c_char), size_t] +libc.munmap.argtypes = [c_void_p, size_t] libc.munmap.restype = c_int libc.msync.argtypes = [c_char_p, size_t, c_int] libc.msync.restype = c_int From arigo at codespeak.net Mon Aug 7 17:00:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 17:00:59 +0200 (CEST) Subject: [pypy-svn] r31119 - pypy/dist/pypy/jit/codegen/i386 user/arigo/hack/pypy-hack/asm Message-ID: <20060807150059.A64201006F@code0.codespeak.net> Author: arigo Date: Mon Aug 7 17:00:57 2006 New Revision: 31119 Added: pypy/dist/pypy/jit/codegen/i386/ - copied from r31118, user/arigo/hack/pypy-hack/asm/ Removed: user/arigo/hack/pypy-hack/asm/ Log: An RPython module that generates Intel 386 machine code. Moved from my user directory; to be streamlined and adapted to the directory structure of PyPy. From arigo at codespeak.net Mon Aug 7 17:17:18 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 17:17:18 +0200 (CEST) Subject: [pypy-svn] r31121 - in pypy/dist/pypy/jit/codegen: . i386 i386/test Message-ID: <20060807151718.D2BAA10072@code0.codespeak.net> Author: arigo Date: Mon Aug 7 17:17:14 2006 New Revision: 31121 Added: pypy/dist/pypy/jit/codegen/detect_cpu.py (contents, props changed) pypy/dist/pypy/jit/codegen/i386/__init__.py (contents, props changed) pypy/dist/pypy/jit/codegen/i386/conftest.py (contents, props changed) pypy/dist/pypy/jit/codegen/i386/test/ (props changed) pypy/dist/pypy/jit/codegen/i386/test/__init__.py (contents, props changed) pypy/dist/pypy/jit/codegen/i386/test/test_assembler.py - copied, changed from r31119, pypy/dist/pypy/jit/codegen/i386/test_assembler.py pypy/dist/pypy/jit/codegen/i386/test/test_i386.py - copied, changed from r31119, pypy/dist/pypy/jit/codegen/i386/test_i386.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386.py - copied, changed from r31119, pypy/dist/pypy/jit/codegen/i386/test_ri386.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py - copied, changed from r31119, pypy/dist/pypy/jit/codegen/i386/test_ri386genop.py Removed: pypy/dist/pypy/jit/codegen/i386/test_assembler.py pypy/dist/pypy/jit/codegen/i386/test_i386.py pypy/dist/pypy/jit/codegen/i386/test_ri386.py pypy/dist/pypy/jit/codegen/i386/test_ri386genop.py Modified: pypy/dist/pypy/jit/codegen/i386/autotest.py Log: * Copied simple CPU detection code from Psyco's setup.py. * Skip tests on non-386 processors. * Added __init__s, moved the tests in a test/ subdirectory. Added: pypy/dist/pypy/jit/codegen/detect_cpu.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/detect_cpu.py Mon Aug 7 17:17:14 2006 @@ -0,0 +1,27 @@ +""" +Processor auto-detection +""" +import sys, os + + +class ProcessorAutodetectError(Exception): + pass + +def autodetect(): + platform = sys.platform.lower() + if platform.startswith('win'): # assume an Intel Windows + return 'i386' + # assume we have 'uname' + mach = os.popen('uname -m', 'r').read().strip() + if not mach: + raise ProcessorAutodetectError, "cannot run 'uname -m'" + try: + return {'i386': 'i386', + 'i486': 'i386', + 'i586': 'i386', + 'i686': 'i386', + 'i86pc': 'i386', # Solaris/Intel + 'x86': 'i386', # Apple + }[mach] + except KeyError: + raise ProcessorAutodetectError, "unsupported processor '%s'" % mach Added: pypy/dist/pypy/jit/codegen/i386/__init__.py ============================================================================== Modified: pypy/dist/pypy/jit/codegen/i386/autotest.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/autotest.py (original) +++ pypy/dist/pypy/jit/codegen/i386/autotest.py Mon Aug 7 17:17:14 2006 @@ -2,8 +2,8 @@ import i386 FILENAME = 'checkfile.tmp' -BEGIN_TAG = '<<>>' -END_TAG = '<<>>' +BEGIN_TAG = '<<>>' +END_TAG = '<<>>' COUNT1 = 15 COUNT2 = 25 Added: pypy/dist/pypy/jit/codegen/i386/conftest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/i386/conftest.py Mon Aug 7 17:17:14 2006 @@ -0,0 +1,16 @@ +import py +from pypy.jit.codegen import detect_cpu + + +class Directory(py.test.collect.Directory): + + def run(self): + try: + processor = detect_cpu.autodetect() + except detect_cpu.ProcessorAutodetectError, e: + py.test.skip(str(e)) + else: + if processor != 'i386': + py.test.skip('detected a %r CPU' % (processor,)) + + return super(Directory, self).run() Added: pypy/dist/pypy/jit/codegen/i386/test/__init__.py ============================================================================== From arigo at codespeak.net Mon Aug 7 18:25:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 18:25:59 +0200 (CEST) Subject: [pypy-svn] r31130 - pypy/dist/pypy/translator/c/test Message-ID: <20060807162559.B8D4510071@code0.codespeak.net> Author: arigo Date: Mon Aug 7 18:25:57 2006 New Revision: 31130 Modified: pypy/dist/pypy/translator/c/test/test_wrapping.py Log: test_wrapping still occasionally segfaults Modified: pypy/dist/pypy/translator/c/test/test_wrapping.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_wrapping.py (original) +++ pypy/dist/pypy/translator/c/test/test_wrapping.py Mon Aug 7 18:25:57 2006 @@ -1,3 +1,5 @@ +import py; py.test.skip("XXX occasionally segfaults") + from pypy.translator.tool.raymond import (get_compiled_module, get_compiled, wrap, unwrap, __init__, ExtCompiler) From arigo at codespeak.net Mon Aug 7 18:28:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 18:28:59 +0200 (CEST) Subject: [pypy-svn] r31131 - in pypy/dist/pypy/rpython: . test Message-ID: <20060807162859.10D4D10074@code0.codespeak.net> Author: arigo Date: Mon Aug 7 18:28:57 2006 New Revision: 31131 Modified: pypy/dist/pypy/rpython/extregistry.py pypy/dist/pypy/rpython/test/test_extregistry.py Log: Fix test_extregistry failures - my fault. Modified: pypy/dist/pypy/rpython/extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extregistry.py (original) +++ pypy/dist/pypy/rpython/extregistry.py Mon Aug 7 18:28:57 2006 @@ -19,7 +19,7 @@ del selfcls._metatype_ def _register_value(selfcls, key): - if isinstance(key, (tuple, list)): + if isinstance(key, tuple): for k in key: selfcls._register_value(k) else: @@ -27,7 +27,7 @@ EXT_REGISTRY_BY_VALUE[key] = selfcls def _register_type(selfcls, key): - if isinstance(key, (tuple, list)): + if isinstance(key, tuple): for k in key: selfcls._register_type(k) else: @@ -35,7 +35,7 @@ EXT_REGISTRY_BY_TYPE[key] = selfcls def _register_metatype(selfcls, key): - if isinstance(key, (tuple, list)): + if isinstance(key, tuple): for k in key: selfcls._register_metatype(k) else: Modified: pypy/dist/pypy/rpython/test/test_extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_extregistry.py (original) +++ pypy/dist/pypy/rpython/test/test_extregistry.py Mon Aug 7 18:28:57 2006 @@ -1,6 +1,4 @@ -import py - -##py.test.skip('In progress at PyCon') +import py, sys from pypy.rpython import extregistry from pypy.rpython.extregistry import ExtRegistryEntry @@ -172,10 +170,10 @@ assert isinstance(extregistry.lookup(lst1), Entry) py.test.raises(KeyError, "extregistry.lookup(lst2)") -def test_register_non_weakly_refable(n=6): - tup1 = (5, 6) - tup2 = (5, n) - class Entry(ExtRegistryEntry): - _about_ = tup1 - assert isinstance(extregistry.lookup(tup1), Entry) - assert isinstance(extregistry.lookup(tup2), Entry) +def test_register_non_weakly_refable(): + n1 = sys.maxint // 2 + n2 = sys.maxint // 2 + class Entry(ExtRegistryEntry): + _about_ = n1 + assert isinstance(extregistry.lookup(n1), Entry) + assert isinstance(extregistry.lookup(n2), Entry) From arigo at codespeak.net Mon Aug 7 19:10:01 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 19:10:01 +0200 (CEST) Subject: [pypy-svn] r31135 - pypy/dist/pypy/translator/backendopt Message-ID: <20060807171001.744F810078@code0.codespeak.net> Author: arigo Date: Mon Aug 7 19:10:00 2006 New Revision: 31135 Modified: pypy/dist/pypy/translator/backendopt/inline.py Log: Fix for the failing test_auto_inlining_small_call_big(). Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Mon Aug 7 19:10:00 2006 @@ -106,6 +106,8 @@ """ this function checks, whether graph contains operations which can raise and which are not exception guarded """ for block in graph.iterblocks(): + if block is graph.exceptblock: + return True # the except block is reachable if block.exitswitch == c_last_exception: consider_ops_to = -1 else: @@ -584,7 +586,6 @@ for parentgraph in callers[graph]: if parentgraph == graph: continue - sys.stdout.flush() try: res = bool(inline_function(translator, graph, parentgraph, lltype_to_classdef, raise_analyzer)) From arigo at codespeak.net Mon Aug 7 19:43:47 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Aug 2006 19:43:47 +0200 (CEST) Subject: [pypy-svn] r31141 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20060807174347.6958310072@code0.codespeak.net> Author: arigo Date: Mon Aug 7 19:43:46 2006 New Revision: 31141 Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py Log: Change the test to be what I guess was its intended meaning. Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_inline.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_inline.py Mon Aug 7 19:43:46 2006 @@ -223,22 +223,19 @@ try: return h(x) except: - raise + return 87 def g(x): try: - f(x) + return f(x) except CustomError1: return 2 - except CustomError2: - return 3 - return 1 eval_func = check_inline(f, g, [int], inline_guarded_calls=True) result = eval_func([0]) assert result == 1 result = eval_func([1]) - assert result == 2 + assert result == 87 result = eval_func([2]) - assert result == 3 + assert result == 87 def test_inline_var_exception(): def f(x): From rhymes at codespeak.net Mon Aug 7 20:46:44 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 7 Aug 2006 20:46:44 +0200 (CEST) Subject: [pypy-svn] r31142 - in pypy/dist/pypy/module: _ssl/test bz2/test fcntl/test mmap/test rctime/test Message-ID: <20060807184644.0A32D10072@code0.codespeak.net> Author: rhymes Date: Mon Aug 7 20:46:35 2006 New Revision: 31142 Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py pypy/dist/pypy/module/bz2/test/test_bz2.py pypy/dist/pypy/module/fcntl/test/test_fcntl.py pypy/dist/pypy/module/mmap/test/test_mmap.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: useless imports Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/test/test_ssl.py (original) +++ pypy/dist/pypy/module/_ssl/test/test_ssl.py Mon Aug 7 20:46:35 2006 @@ -1,4 +1,3 @@ -from py.test import raises, skip from pypy.conftest import gettestobjspace import os Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Mon Aug 7 20:46:35 2006 @@ -1,4 +1,3 @@ -from py.test import raises, skip from pypy.conftest import gettestobjspace import os Modified: pypy/dist/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/test/test_fcntl.py Mon Aug 7 20:46:35 2006 @@ -1,4 +1,3 @@ -from py.test import raises, skip from pypy.conftest import gettestobjspace import os Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Mon Aug 7 20:46:35 2006 @@ -1,4 +1,3 @@ -from py.test import raises, skip from pypy.conftest import gettestobjspace import os Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Mon Aug 7 20:46:35 2006 @@ -1,4 +1,3 @@ -from py.test import raises, skip from pypy.conftest import gettestobjspace import os From cfbolz at codespeak.net Mon Aug 7 23:59:23 2006 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 Aug 2006 23:59:23 +0200 (CEST) Subject: [pypy-svn] r31144 - pypy/dist/pypy/tool Message-ID: <20060807215923.B06DC10063@code0.codespeak.net> Author: cfbolz Date: Mon Aug 7 23:59:21 2006 New Revision: 31144 Modified: pypy/dist/pypy/tool/option.py Log: typo that made --oldstyle stop working Modified: pypy/dist/pypy/tool/option.py ============================================================================== --- pypy/dist/pypy/tool/option.py (original) +++ pypy/dist/pypy/tool/option.py Mon Aug 7 23:59:21 2006 @@ -103,7 +103,7 @@ kwds.get("uselibfile", False)): conf.objspace.uselibfile = True if getattr(cmdlineopt, "oldstyle", False) or kwds.get("oldstyle", False): - conf.objspace.oldstyle = True + conf.objspace.std.oldstyle = True if hasattr(cmdlineopt, "parser") and cmdlineopt.parser is not None: conf.objspace.parser = cmdlineopt.parser if kwds.get("compiler") is not None: From antocuni at codespeak.net Tue Aug 8 00:30:54 2006 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 8 Aug 2006 00:30:54 +0200 (CEST) Subject: [pypy-svn] r31146 - pypy/dist/pypy/translator/cli Message-ID: <20060807223054.E62DE10063@code0.codespeak.net> Author: antocuni Date: Tue Aug 8 00:30:48 2006 New Revision: 31146 Modified: pypy/dist/pypy/translator/cli/database.py pypy/dist/pypy/translator/cli/ilgenerator.py Log: Added support for inline constants. Inline constants are not stored as static fields of the Constant class, but they are computed every time they are needed. By now null instances and constants of type Class are inlined. Moreover, add the beforefieldinit flag to the Constants class because it saves some seconds in pypy startup time. Modified: pypy/dist/pypy/translator/cli/database.py ============================================================================== --- pypy/dist/pypy/translator/cli/database.py (original) +++ pypy/dist/pypy/translator/cli/database.py Tue Aug 8 00:30:48 2006 @@ -168,7 +168,7 @@ self.consts[value] = const self.pending_node(const) - return '%s.%s::%s' % (CONST_NAMESPACE, CONST_CLASS, const.name), const.get_type() + return const def record_delegate(self, TYPE): try: @@ -194,34 +194,31 @@ def gen_constants(self, ilasm): self.locked = True # new pending nodes are not allowed here ilasm.begin_namespace(CONST_NAMESPACE) - ilasm.begin_class(CONST_CLASS) + ilasm.begin_class(CONST_CLASS, beforefieldinit=True) + + const_list = [const for const in self.consts.itervalues() if not const.is_inline()] + const_list.sort(key=operator.attrgetter('PRIORITY')) + num_const = len(const_list) # render field definitions - for const in self.consts.itervalues(): + for const in const_list: + assert not const.is_null() ilasm.field(const.name, const.get_type(), static=True) # this point we have collected all constant we # need. Instantiate&initialize them. self.step = 0 - for i, const in enumerate(self.consts.itervalues()): + for i, const in enumerate(const_list): if i % MAX_CONST_PER_STEP == 0: self.__new_step(ilasm) type_ = const.get_type() - if const.is_null(): - ilasm.opcode('ldnull') - else: - const.instantiate(ilasm) + const.instantiate(ilasm) ilasm.store_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) - const_list = self.consts.values() - num_const = len(const_list) - const_list.sort(key=operator.attrgetter('PRIORITY')) for i, const in enumerate(const_list): if i % MAX_CONST_PER_STEP == 0: self.__new_step(ilasm) ilasm.stderr('CONST: initializing #%d' % i, DEBUG_CONST_INIT_VERBOSE) - if const.is_null(): - continue type_ = const.get_type() ilasm.load_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) const.init(ilasm) @@ -308,11 +305,11 @@ ilasm.opcode("ldstr", string_literal(value._str)) else: assert TYPE not in cls.PRIMITIVE_TYPES - cts = CTS(db) - name, cts_static_type = db.record_const(value) - ilasm.opcode('ldsfld %s %s' % (cts_static_type, name)) - if cts_static_type != cts.lltype_to_cts(TYPE): - ilasm.opcode('castclass', cts.lltype_to_cts(TYPE, include_class=False)) + const = db.record_const(value) + if const.is_null(): + ilasm.opcode('ldnull') + else: + const._load(ilasm, TYPE) load = classmethod(load) def __hash__(self): @@ -336,6 +333,26 @@ def is_null(self): return self.value is ootype.null(self.value._TYPE) + def is_inline(self): + """ + Inline constants are not stored as static fields in the + Constant class, but they are newly created on the stack every + time they are used. Classes overriding is_inline should + override _load too. + """ + return self.is_null() # by default only null constants are inlined + + def _load(self, ilasm, EXPECTED_TYPE): + """ + Load the constant onto the stack. + """ + cts = CTS(self.db) + full_name = '%s.%s::%s' % (CONST_NAMESPACE, CONST_CLASS, self.name) + cts_static_type = self.get_type() + ilasm.opcode('ldsfld %s %s' % (cts_static_type, full_name)) + if cts_static_type != cts.lltype_to_cts(EXPECTED_TYPE): + ilasm.opcode('castclass', cts.lltype_to_cts(EXPECTED_TYPE, include_class=False)) + def record_const_maybe(self, TYPE, value): if AbstractConst.is_primitive(TYPE): return @@ -444,12 +461,14 @@ def is_null(self): return self.value._INSTANCE is None - def instantiate(self, ilasm): + def is_inline(self): + return True + + def _load(self, ilasm, EXPECTED_TYPE): assert not self.is_null() INSTANCE = self.value._INSTANCE ilasm.opcode('ldtoken', self.db.class_name(INSTANCE)) ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') - self.db.const_count.inc('Class') class ListConst(AbstractConst): def __init__(self, db, list_, count): Modified: pypy/dist/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/dist/pypy/translator/cli/ilgenerator.py (original) +++ pypy/dist/pypy/translator/cli/ilgenerator.py Tue Aug 8 00:30:48 2006 @@ -55,7 +55,8 @@ def end_namespace(self): self.code.closeblock() - def begin_class(self, name, base=None, sealed=False, interfaces=(), abstract=False): + def begin_class(self, name, base=None, sealed=False, interfaces=(), abstract=False, + beforefieldinit=False): if base is None: base = '[mscorlib]System.Object' s = '' @@ -63,6 +64,8 @@ s += 'abstract ' if sealed: s += 'sealed ' + if beforefieldinit: + s += 'beforefieldinit ' self.code.writeline('.class public %s %s extends %s' % (s, name, base)) if interfaces: From mwh at codespeak.net Tue Aug 8 00:33:44 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 00:33:44 +0200 (CEST) Subject: [pypy-svn] r31147 - in pypy/dist/pypy: config objspace/std Message-ID: <20060807223344.5D7E710063@code0.codespeak.net> Author: mwh Date: Tue Aug 8 00:33:41 2006 New Revision: 31147 Added: pypy/dist/pypy/objspace/std/dictmultiobject.py - copied, changed from r30977, pypy/dist/pypy/objspace/std/dictobject.py Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/objspace/std/model.py pypy/dist/pypy/objspace/std/objspace.py Log: yet another kind of dictionary, this one designed to allow many different internal implementations. translatable, and seems to be somehow slightly faster than the default, which is surprising but i'll take it :-) Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Tue Aug 8 00:33:41 2006 @@ -65,6 +65,10 @@ "use dictionaries optimized for string keys", default=False), + BoolOption("withmultidict", + "use dictionaries optimized for flexibility", + default=False), + BoolOption("oldstyle", "specify whether the default metaclass should be classobj", default=False), Modified: pypy/dist/pypy/objspace/std/model.py ============================================================================== --- pypy/dist/pypy/objspace/std/model.py (original) +++ pypy/dist/pypy/objspace/std/model.py Tue Aug 8 00:33:41 2006 @@ -14,6 +14,8 @@ "withstrjoin" : ["strjoinobject.W_StringJoinObject"], "withstrdict" : ["dictstrobject.W_DictStrObject", "dictstrobject.W_DictStrIterObject"], + "withmultidict" : ["dictmultiobject.W_DictMultiObject", + "dictmultiobject.W_DictMultiIterObject"], } class StdTypeModel: @@ -58,6 +60,7 @@ from pypy.objspace.std import listobject from pypy.objspace.std import dictobject from pypy.objspace.std import dictstrobject + from pypy.objspace.std import dictmultiobject from pypy.objspace.std import stringobject from pypy.objspace.std import strsliceobject from pypy.objspace.std import strjoinobject Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Tue Aug 8 00:33:41 2006 @@ -58,6 +58,9 @@ if self.config.objspace.std.withstrdict: from pypy.objspace.std import dictstrobject self.DictObjectCls = dictstrobject.W_DictStrObject + elif self.config.objspace.std.withmultidict: + from pypy.objspace.std import dictmultiobject + self.DictObjectCls = dictmultiobject.W_DictMultiObject else: from pypy.objspace.std import dictobject self.DictObjectCls = dictobject.W_DictObject @@ -469,7 +472,7 @@ def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) if type(w_obj) is self.DictObjectCls: - if not self.config.objspace.std.withstrdict: + if not self.config.objspace.std.withstrdict and not self.config.objspace.std.withmultidict: return w_obj.content.get(w_key, None) else: return w_obj.get(w_key, None) From mwh at codespeak.net Tue Aug 8 00:37:03 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 00:37:03 +0200 (CEST) Subject: [pypy-svn] r31148 - pypy/dist/pypy/objspace/std Message-ID: <20060807223703.E968910063@code0.codespeak.net> Author: mwh Date: Tue Aug 8 00:37:02 2006 New Revision: 31148 Modified: pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/objspace.py Log: add W_DictObject.get, remove some ugly code. Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Tue Aug 8 00:37:02 2006 @@ -38,6 +38,9 @@ def len(w_self): return len(w_self.content) + def get(w_dict, w_lookup, w_default): + return w_dict.content.get(w_lookup, w_default) + def set_str_keyed_item(w_dict, w_key, w_value): w_dict.content[w_key] = w_value Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Tue Aug 8 00:37:02 2006 @@ -472,10 +472,7 @@ def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) if type(w_obj) is self.DictObjectCls: - if not self.config.objspace.std.withstrdict and not self.config.objspace.std.withmultidict: - return w_obj.content.get(w_key, None) - else: - return w_obj.get(w_key, None) + return w_obj.get(w_key, None) return ObjSpace.finditem(self, w_obj, w_key) def set_str_keyed_item(self, w_obj, w_key, w_value): From mwh at codespeak.net Tue Aug 8 00:40:14 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 00:40:14 +0200 (CEST) Subject: [pypy-svn] r31149 - pypy/dist/pypy/objspace/std Message-ID: <20060807224014.1B0DD10063@code0.codespeak.net> Author: mwh Date: Tue Aug 8 00:40:13 2006 New Revision: 31149 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/dictstrobject.py Log: remove unused 'return_entry' methods Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 00:40:13 2006 @@ -303,8 +303,6 @@ w_self.pos = 0 w_self.setup_iterator() - def return_entry(w_self, w_key, w_value): - raise NotImplementedError registerimplementation(W_DictMultiIterObject) Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Tue Aug 8 00:40:13 2006 @@ -229,8 +229,6 @@ w_self.pos = 0 w_self.setup_iterator() - def return_entry(w_self, w_key, w_value): - raise NotImplementedError registerimplementation(W_DictIterObject) Modified: pypy/dist/pypy/objspace/std/dictstrobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictstrobject.py (original) +++ pypy/dist/pypy/objspace/std/dictstrobject.py Tue Aug 8 00:40:13 2006 @@ -363,9 +363,6 @@ w_self.pos = 0 w_self.setup_iterator() - def return_entry(w_self, w_key, w_value): - raise NotImplementedError - def handleMutation(w_self): space = w_self.space w_self.len = -1 # Make this error state sticky From cfbolz at codespeak.net Tue Aug 8 00:40:23 2006 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 8 Aug 2006 00:40:23 +0200 (CEST) Subject: [pypy-svn] r31150 - pypy/dist/pypy/objspace/std Message-ID: <20060807224023.27A841006E@code0.codespeak.net> Author: cfbolz Date: Tue Aug 8 00:40:20 2006 New Revision: 31150 Modified: pypy/dist/pypy/objspace/std/objspace.py Log: _really_ fix --oldstyle. Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Tue Aug 8 00:40:20 2006 @@ -44,8 +44,9 @@ PACKAGE_PATH = 'objspace.std' - def setoptions(self, oldstyle=False): - self.config.objspace.std.oldstyle = oldstyle + def setoptions(self, **kwds): + if "oldstyle" in kwds: + self.config.objspace.std.oldstyle = kwds["oldstyle"] def initialize(self): "NOT_RPYTHON: only for initializing the space." From mwh at codespeak.net Tue Aug 8 03:06:29 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 03:06:29 +0200 (CEST) Subject: [pypy-svn] r31151 - pypy/dist/pypy/objspace/std Message-ID: <20060808010629.4FD9F1005A@code0.codespeak.net> Author: mwh Date: Tue Aug 8 03:06:26 2006 New Revision: 31151 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: silly example: a specialized representation for empty dicts. uses a hack that may or may not be horrible: the mutating implementation methods don't have a return value, so return the implementation (usually just 'self') from them. keeping the iterator code translatable is going to be annoying, i can tell already. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 03:06:26 2006 @@ -35,22 +35,36 @@ def items(self): return [(w_key, w_value) or w_key, w_value in self.iteritems()] -## class EmptyDictImplementation(DictImplementation): -## def getitem(self, w_key, w_value): -## raise KeyError -## def setitem(self, w_dict, w_key, w_value): -## assert False -## def delitem(self, w_dict, w_key, w_value): -## raise KeyError -## def length(self): -## return 0 -## def clear(self, w_dict): -## pass -## def has_key(self, w_lookup): -## return False +class EmptyDictImplementation(DictImplementation): + def __init__(self, space): + self.space = space + + def getitem(self, w_key): + raise KeyError + def setitem(self, w_key, w_value): + return RDictImplementation(self.space).setitem(w_key, w_value) + def delitem(self, w_key): + raise KeyError + + def length(self): + return 0 + def clear(self): + return self + def has_key(self, w_lookup): + return False + def get(self, w_lookup, w_default): + return w_default + + def iteritems(self): + return RDictImplementation(self.space).iteritems() + def iterkeys(self): + return RDictImplementation(self.space).iterkeys() + def itervalues(self): + return RDictImplementation(self.space).itervalues() class RDictImplementation(DictImplementation): def __init__(self, space): + self.space = space self.content = r_dict(space.eq_w, space.hash_w) def __repr__(self): @@ -60,13 +74,18 @@ return self.content[w_key] def setitem(self, w_key, w_value): self.content[w_key] = w_value + return self def delitem(self, w_key): del self.content[w_key] + if self.content: + return self + else: + return EmptyDictImplementation(self.space) def length(self): return len(self.content) def clear(self): - self.content.clear() + return EmptyDictImplementation(self.space) def has_key(self, w_lookup): return w_lookup in self.content def get(self, w_lookup, w_default): @@ -90,14 +109,16 @@ from pypy.objspace.std.dicttype import dict_typedef as typedef def __init__(w_self, space, w_otherdict=None): - w_self.implementation = RDictImplementation(space) + w_self.implementation = EmptyDictImplementation(space) if w_otherdict is not None: from pypy.objspace.std.dicttype import dict_update__ANY_ANY dict_update__ANY_ANY(space, w_self, w_otherdict) def initialize_content(w_self, list_pairs_w): + impl = w_self.implementation for w_k, w_v in list_pairs_w: - w_self.implementation.setitem(w_k, w_v) + impl = impl.setitem(w_k, w_v) + w_self.implementation = impl def __repr__(w_self): """ representation for debugging purposes """ @@ -117,7 +138,7 @@ return w_dict.implementation.get(w_key, w_default) def set_str_keyed_item(w_dict, w_key, w_value): - w_dict.implementation.setitem(w_key, w_value) + w_dict.implementation = w_dict.implementation.setitem(w_key, w_value) registerimplementation(W_DictMultiObject) @@ -126,7 +147,7 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictMultiObject(space)]) # default argument - w_dict.implementation.clear() + w_dict.implementation = w_dict.implementation.clear() try: space.getattr(w_src, space.wrap("keys")) except OperationError: @@ -137,7 +158,7 @@ raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) w_k, w_v = pair - w_dict.implementation.setitem(w_k, w_v) + w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) else: if space.is_true(w_src): from pypy.objspace.std.dicttype import dict_update__ANY_ANY @@ -153,11 +174,11 @@ raise OperationError(space.w_KeyError, w_lookup) def setitem__DictMulti_ANY_ANY(space, w_dict, w_newkey, w_newvalue): - w_dict.implementation.setitem(w_newkey, w_newvalue) + w_dict.implementation = w_dict.implementation.setitem(w_newkey, w_newvalue) def delitem__DictMulti_ANY(space, w_dict, w_lookup): try: - w_dict.implementation.delitem(w_lookup) + w_dict.implementation = w_dict.implementation.delitem(w_lookup) except KeyError: raise OperationError(space.w_KeyError, w_lookup) @@ -178,7 +199,7 @@ if w_left.implementation.length() != w_right.implementation.length(): return space.w_False - for w_key, w_val in w_left.implementation.content.iteritems(): + for w_key, w_val in w_left.implementation.iteritems(): try: w_rightval = w_right.implementation.getitem(w_key) except KeyError: @@ -192,7 +213,7 @@ returns the smallest key in acontent for which b's value is different or absent and this value """ w_smallest_diff_a_key = None w_its_value = None - for w_key, w_val in aimpl.content.iteritems(): + for w_key, w_val in aimpl.iteritems(): if w_smallest_diff_a_key is None or space.is_true(space.lt(w_key, w_smallest_diff_a_key)): try: w_bvalue = bimpl.getitem(w_key) @@ -251,7 +272,7 @@ return W_DictMultiIter_Values(space, w_self.implementation) def dict_clear__DictMulti(space, w_self): - w_self.implementation.clear() + w_self.implementation = w_self.implementation.clear() def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): return w_dict.implementation.get(w_lookup, w_default) From auc at codespeak.net Tue Aug 8 10:49:10 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 8 Aug 2006 10:49:10 +0200 (CEST) Subject: [pypy-svn] r31153 - pypy/dist/pypy/module/_stackless Message-ID: <20060808084910.5D49C10053@code0.codespeak.net> Author: auc Date: Tue Aug 8 10:49:08 2006 New Revision: 31153 Modified: pypy/dist/pypy/module/_stackless/clonable.py Log: oops, forgoten in last commit Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Tue Aug 8 10:49:08 2006 @@ -2,6 +2,7 @@ from pypy.module._stackless.interp_clonable import InterpClonableCoroutine from pypy.rpython.objectmodel import we_are_translated +from pypy.rpython.rgc import gc_swap_pool, gc_clone from pypy.module._stackless.stackless_flags import StacklessFlags from pypy.interpreter.function import StaticMethod @@ -12,21 +13,26 @@ from pypy.rpython import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode -class ClonableCoroutine(InterpClonableCoroutine): +class ClonableCoroutine(InterpClonableCoroutine): + local_pool = None #XXX cut'n'pasted from AppCoroutine # so, watch changes in coroutine.py - def __init__(self, space, is_main=False): + def __init__(self, space, is_main=False, costate=None): + assert isinstance(is_main, bool) self.space = space self.is_main = is_main state = self._get_state(space) Coroutine.__init__(self, state) + if costate is not None: + self.costate = costate self.flags = 0 self.framestack = None if not is_main: space.getexecutioncontext().subcontext_new(self) self._dead = False - self._next = self._prev = self # help the annotator with the scheduling ring + self._next = self._prev = self + self._cspace = None def hello(self): if we_are_translated(): @@ -42,27 +48,23 @@ ec = self.space.getexecutioncontext() ec.subcontext_leave(self) - def from_interp(self, interp_coro): - assert isinstance(interp_coro, InterpClonableCoroutine) - # get parent, frame, local_pool - new = ClonableCoroutine(self.space, self.is_main) - new.parent = interp_coro.parent - new.frame = interp_coro.frame - new.local_pool = interp_coro.local_pool - return new - def w_clone(self): - if self.is_main: - raise OperationError(self.space.w_NotImplementedError, - self.space.wrap("The main coroutine can't be cloned")) - try: - interp_coro = InterpClonableCoroutine.clone(self) - app_coro = self.from_interp(interp_coro) - return app_coro - except NotImplementedError: - raise OperationError(self.space.w_NotImplementedError, - self.space.wrap("clone() is only available in translated PyPy")) + if not we_are_translated(): + raise NotImplementedError + if ClonableCoroutine.w_getcurrent(self.space) is self: + raise RuntimeError("clone() cannot clone the current coroutine; " + "use fork() instead") + if self.local_pool is None: # force it now + self.local_pool = gc_swap_pool(gc_swap_pool(None)) + # cannot gc_clone() directly self, because it is not in its own + # local_pool. Moreover, it has a __del__, which cloning doesn't + # support properly at the moment. + copy = ClonableCoroutine(self.space, costate=self.costate) + copy.parent = self.parent + copy.frame, copy.local_pool = gc_clone(self.frame, self.local_pool) + return copy + def w_getcurrent(space): return space.wrap(ClonableCoroutine._get_state(space).current) w_getcurrent = staticmethod(w_getcurrent) @@ -250,3 +252,4 @@ ) + From auc at codespeak.net Tue Aug 8 10:57:32 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 8 Aug 2006 10:57:32 +0200 (CEST) Subject: [pypy-svn] r31154 - in pypy/dist/pypy: lib objspace objspace/cclp objspace/test Message-ID: <20060808085732.AFF3D10069@code0.codespeak.net> Author: auc Date: Tue Aug 8 10:57:24 2006 New Revision: 31154 Modified: pypy/dist/pypy/lib/_exceptions.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: ask, waitstable, commit, choose ... Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Tue Aug 8 10:57:24 2006 @@ -445,7 +445,7 @@ class LOError(Exception): pass class UnificationError(LOError): pass -class RebindingError(LOError): pass +class RebindingError(UnificationError): pass class FutureBindingError(LOError): pass class AllBlockedError(LOError): pass Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 8 10:57:24 2006 @@ -55,6 +55,7 @@ assert self._head not in self._blocked assert self._head not in self._blocked_on assert self._head not in self._blocked_byneed + assert self._head not in self._asking except: #XXX give sched_info maybe raise OperationError(self.space.w_RuntimeError, @@ -91,18 +92,15 @@ thread._next = thread._prev = FunnyBoat # cspace/threads account mgmt if thread._cspace is not None: - count = self._per_space_live_threads[thread._cspace] - assert count > 0 - self._per_space_live_threads[thread._cspace] = count - 1 - if count == 1: + count = self.dec_live_thread_count(thread._cspace) + if count == 0: del self._per_space_live_threads[thread._cspace] - -# return thread #-- cspace helper def is_stable(self, cspace): if not self._per_space_live_threads.has_key(cspace): + #XXX meaning ? return True return self._per_space_live_threads[cspace] == 0 @@ -112,7 +110,26 @@ curr = ClonableCoroutine.w_getcurrent(self.space) self._asking[curr] = cspace self._blocked[curr] = True - self.schedule() + # either we choose() from inside + if curr._cspace == cspace: + self.dec_live_thread_count(cspace) + self.schedule() + self.inc_live_thread_count(cspace) + else: # or we ask() from outside + self.schedule() + + #-- cspace -> thread_count helpers + def inc_live_thread_count(self, cspace): + count = self._per_space_live_threads.get(cspace, 0) + 1 + self._per_space_live_threads[cspace] = count + return count + + def dec_live_thread_count(self, cspace): + count = self._per_space_live_threads[cspace] -1 + assert count >= 0 + self._per_space_live_threads[cspace] = count + return count + #-- / #-- to be used by logic objspace @@ -141,6 +158,7 @@ while (to_be_run in self._blocked) \ or to_be_run.is_dead() \ or (to_be_run == current): + to_be_run = to_be_run._next assert isinstance(to_be_run, ClonableCoroutine) if to_be_run == sentinel: @@ -153,10 +171,12 @@ w(".. SCHEDULER reinitialized") raise OperationError(self.space.w_AllBlockedError, self.space.wrap("can't schedule, possible deadlock in sight")) - if to_be_run in self._asking: + # asking threads + if to_be_run in self._asking.keys(): if self.is_stable(self._asking[to_be_run]): del self._asking[to_be_run] del self._blocked[to_be_run] + break self._head = to_be_run return to_be_run @@ -185,11 +205,10 @@ def add_new_thread(self, thread): "insert 'thread' at end of running queue" assert isinstance(thread, ClonableCoroutine) + # cspace account mgmt if thread._cspace != None: - print "Yeeeep" - count = self._per_space_live_threads.get(thread._cspace, 0) - self._per_space_live_threads[thread._cspace] = count + 1 - assert len(self._per_space_live_threads) + self._per_space_live_threads.get(thread._cspace, 0) + self.inc_live_thread_count(thread._cspace) self._chain_insert(thread) def add_to_blocked_on(self, w_var, thread): @@ -206,7 +225,7 @@ self._blocked[thread] = True # cspace accounting if thread._cspace is not None: - self._per_space_live_threads[thread._cspace] -= 1 + self.dec_live_thread_count(thread._cspace) def unblock_on(self, w_var): v(".. we UNBLOCK threads dependants of var", str(w_var)) @@ -220,7 +239,7 @@ del self._blocked[thr] # cspace accounting if thr._cspace is not None: - self._per_space_live_threads[thr._cspace] += 1 + self.inc_live_thread_count(thr._cspace) def add_to_blocked_byneed(self, w_var, thread): w(".. we BLOCK BYNEED thread", str(id(thread)), "on var", str(w_var)) @@ -235,7 +254,7 @@ self._blocked[thread] = True # cspace accounting if thread._cspace is not None: - self._per_space_live_threads[thread._cspace] += 1 + self.dec_live_thread_count(thread._cspace) def unblock_byneed_on(self, w_var): v(".. we UNBLOCK BYNEED dependants of var", str(w_var)) @@ -251,7 +270,7 @@ del self._blocked[thr] # cspace accounting if thr._cspace is not None: - self._per_space_live_threads[thr._cspace] -= 1 + self.inc_live_thread_count(thr._cspace) # Logic Variables tracing, helps exception propagation @@ -304,7 +323,7 @@ si(w_ret, sw('threads'), sw([id(th) for th in s.get_threads()])) si(w_ret, sw('blocked_on'), - sw([(id(th), [id(th) for th in thl]) + sw([(id(var), [id(th) for th in thl]) for var, thl in s._blocked_on.items()])) si(w_ret, sw('blocked_byneed'), sw([(id(var), [id(th) for th in thl]) Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Tue Aug 8 10:57:24 2006 @@ -2,33 +2,47 @@ from pypy.interpreter import baseobjspace, gateway, argument, typedef from pypy.interpreter.error import OperationError +from pypy.objspace.std.intobject import W_IntObject + from pypy.objspace.cclp.misc import ClonableCoroutine, w -from pypy.objspace.cclp.thunk import FutureThunk, ProcedureThunk +from pypy.objspace.cclp.thunk import CSpaceThunk from pypy.objspace.cclp.global_state import scheduler +from pypy.objspace.cclp.variable import newvar + def newspace(space, w_callable, __args__): - try: - args = __args__.normalize() - # coro init - w_coro = ClonableCoroutine(space) - thunk = ProcedureThunk(space, w_callable, args, w_coro) - w_coro.bind(thunk) - if not we_are_translated(): - w("NEWSPACE, thread", str(id(w_coro)), "for", str(w_callable.name)) - - w_space = W_CSpace(space, w_coro, parent=w_coro._cspace) - w_coro._cspace = w_space - - scheduler[0].add_new_thread(w_coro) - scheduler[0].schedule() - except: - print "oh, uh" + args = __args__.normalize() + # coro init + w_coro = ClonableCoroutine(space) + thunk = CSpaceThunk(space, w_callable, args, w_coro) + w_coro.bind(thunk) + if not we_are_translated(): + w("NEWSPACE, thread", str(id(w_coro)), "for", str(w_callable.name)) + w_space = W_CSpace(space, w_coro, parent=w_coro._cspace) + w_coro._cspace = w_space + + scheduler[0].add_new_thread(w_coro) + scheduler[0].schedule() return w_space app_newspace = gateway.interp2app(newspace, unwrap_spec=[baseobjspace.ObjSpace, baseobjspace.W_Root, argument.Arguments]) + +def choose(space, w_n): + assert isinstance(w_n, W_IntObject) + n = space.int_w(w_n) + cspace = ClonableCoroutine.w_getcurrent(space)._cspace + if cspace != None: + assert isinstance(cspace, W_CSpace) + return cspace.choose(n) + raise OperationError(space.w_RuntimeError, + space.wrap("choose is forbidden from the top-level space")) +app_choose = gateway.interp2app(choose) + + + class W_CSpace(baseobjspace.Wrappable): def __init__(self, space, thread, parent=None): @@ -37,13 +51,37 @@ self.space = space # the object space ;-) self.parent = parent self.main_thread = thread + # choice mgmt + self._choice = newvar(space) + self._committed = newvar(space) def w_ask(self): scheduler[0].wait_stable(self) - return self.space.newint(0) + self.space.wait(self._choice) + return self.space.newint(self._choice) + + def choose(self, n): + assert n > 1 + scheduler[0].wait_stable(self) + assert self.space.is_true(self.space.is_free(self._choice)) + assert self.space.is_true(self.space.is_free(self._committed)) + self.space.bind(self._choice, self.space.wrap(n)) + self.space.wait(self._committed) + committed = self._committed + self._committed = newvar(self.space) + return committed + + def w_commit(self, w_n): + assert self.space.is_true(self.space.is_bound(self._choice)) + assert 0 < self.space.int_w(w_n) + assert self.space.int_w(w_n) <= self._choice + self.space.bind(self._committed, w_n) + self._choice = newvar(self.space) + W_CSpace.typedef = typedef.TypeDef("W_CSpace", - ask = gateway.interp2app(W_CSpace.w_ask)) + ask = gateway.interp2app(W_CSpace.w_ask), + commit = gateway.interp2app(W_CSpace.w_commit)) Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Tue Aug 8 10:57:24 2006 @@ -40,7 +40,6 @@ #coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) coro.bind(thunk) - print we_are_translated() if not we_are_translated(): w("STACKLET", str(id(coro)), "for", str(w_callable.name)) scheduler[0].add_new_thread(coro) Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Tue Aug 8 10:57:24 2006 @@ -21,9 +21,7 @@ scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True else: - w(".! clean (valueless) EXIT of", str(id(self._coro)), - "-- setting future result", str(self.w_Result), "to", - str(self.costate.w_tempval)) + w(".! clean (valueless) EXIT of", str(id(self._coro))) finally: scheduler[0].remove_thread(self._coro) scheduler[0].schedule() @@ -56,6 +54,35 @@ scheduler[0].remove_thread(self._coro) scheduler[0].schedule() +SPACE_FAILURE = 0 +SPACE_SOLUTION = 1 + +class CSpaceThunk(_AppThunk): + def __init__(self, space, w_callable, args, coro): + _AppThunk.__init__(self, space, coro.costate, w_callable, args) + self._coro = coro + + def call(self): + w(". initial (returnless) thunk CALL in", str(id(self._coro))) + scheduler[0].trace_vars(self._coro, logic_args(self.args.unpack())) + cspace = self._coro._cspace + try: + try: + _AppThunk.call(self) + except Exception, exc: + w(".% exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) + self._coro._dead = True + self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) + else: + w(".% clean (valueless) EXIT of", str(id(self._coro))) + self.space.bind(cspace._choice, self.space.wrap(SPACE_SOLUTION)) + finally: + scheduler[0].remove_thread(self._coro) + scheduler[0].schedule() + + + def logic_args(args): "returns logic vars found in unpacked normalized args" Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Tue Aug 8 10:57:24 2006 @@ -21,7 +21,7 @@ from pypy.objspace.cclp.global_state import scheduler -from pypy.objspace.cclp.space import app_newspace, W_CSpace +from pypy.objspace.cclp.space import app_newspace, app_choose, W_CSpace #-- VARIABLE ------------------------------------------------ @@ -269,8 +269,11 @@ space.wrap(app_this_thread)) space.setitem(space.builtin.w_dict, space.wrap('reset_scheduler'), space.wrap(app_reset_scheduler)) + #-- comp. spaces -- space.setitem(space.builtin.w_dict, space.wrap('newspace'), space.wrap(app_newspace)) + space.setitem(space.builtin.w_dict, space.wrap('choose'), + space.wrap(app_choose)) #-- misc ----- space.setitem(space.builtin.w_dict, space.wrap('interp_id'), Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Tue Aug 8 10:57:24 2006 @@ -603,11 +603,24 @@ def test_newspace_ask_noop(self): - def bar(X): return X + 42 + def in_space(X): return X + 42 + + def asker(): + ask() X = newvar() - s = newspace(bar, X) - s.ask() + s = newspace(in_space, X) + + assert sched_all()['space_accounting'][0][1] == 0 # live threads + assert len(sched_all()['blocked_on']) == 1 + + stacklet(asker) + + unify(X, 42) + schedule() + assert len(sched_all()['threads']) == 1 + + def test_newspace_ask_wait(self): @@ -628,3 +641,38 @@ schedule() # allow quux exit schedule() # allow asker exit assert len(sched_all()['asking']) == 0 + + def test_ask_choose(self): + + def chooser(X): + choice = choose(3) + unify(X, choice) + + def asker(cspace): + choices = cspace.ask() + cspace.commit(2) + + X = newvar() + + s = newspace(chooser, X) + stacklet(asker, s) + schedule() + assert X == 2 + + + def test_ask_choose(self): + + def chooser(X): + choice = choose(3) + unify(X, choice) + + def asker(cspace): + choices = cspace.ask() + cspace.commit(2) + + X = newvar() + + s = newspace(chooser, X) + stacklet(asker, s) + schedule() + assert X == 2 From rhymes at codespeak.net Tue Aug 8 11:15:12 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Tue, 8 Aug 2006 11:15:12 +0200 (CEST) Subject: [pypy-svn] r31155 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060808091512.DF0E210069@code0.codespeak.net> Author: rhymes Date: Tue Aug 8 11:15:09 2006 New Revision: 31155 Modified: pypy/dist/pypy/module/bz2/__init__.py pypy/dist/pypy/module/bz2/test/test_bz2.py Log: fix and refactor Modified: pypy/dist/pypy/module/bz2/__init__.py ============================================================================== --- pypy/dist/pypy/module/bz2/__init__.py (original) +++ pypy/dist/pypy/module/bz2/__init__.py Tue Aug 8 11:15:09 2006 @@ -6,6 +6,7 @@ 'BZ2Compressor': 'interp_bz2.BZ2Compressor', 'BZ2Decompressor': 'interp_bz2.BZ2Decompressor', 'compress': 'interp_bz2.compress', + 'decompress': 'interp_bz2.decompress', } appleveldefs = { Modified: pypy/dist/pypy/module/bz2/test/test_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2.py Tue Aug 8 11:15:09 2006 @@ -2,7 +2,36 @@ import os if os.name == "nt": - skip("fcntl module is not available on Windows") + skip("bz2 module is not available on Windows") + +def setup_module(mod): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + + def create_temp_file(crlf=False): + f = open("foo", "wb") + + data = (DATA, DATA_CRLF)[crlf] + f.write(data) + f.close() + + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + mod.DATA = DATA + mod.DATA_CRLF = DATA_CRLF + mod.create_temp_file = create_temp_file + mod.decompress = decompress def teardown_module(mod): if os.path.exists("foo"): @@ -12,6 +41,11 @@ def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_DATA = space.wrap(DATA) + cls.w_DATA_CRLF = space.wrap(DATA_CRLF) + cls.w_create_temp_file = space.wrap(create_temp_file) + cls.w_decompress = space.wrap(decompress) def test_attributes(self): from bz2 import BZ2File @@ -86,18 +120,8 @@ assert bz2f.tell() == 0 def test_open_close_del(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() for i in range(10000): f = BZ2File("foo") @@ -110,220 +134,102 @@ def test_open_mode_U(self): # bug #1194181: bz2.BZ2File opened for write with mode "U" - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - def create_temp_file(crlf=False): - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() bz2f = BZ2File("foo", "U") bz2f.close() f = open("foo") f.seek(0, 2) data = f.read() - assert f.tell() == len(DATA) + assert f.tell() == len(self.DATA) f.close() def test_seek_forward(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.seek(150) # (150, 0) - assert bz2f.read() == TEXT[150:] + assert bz2f.read() == self.TEXT[150:] bz2f.close() def test_seek_backwards(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.read(500) bz2f.seek(-150, 1) - assert bz2f.read() == TEXT[500 - 150:] + assert bz2f.read() == self.TEXT[500 - 150:] bz2f.close() def test_seek_backwards_from_end(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.seek(-150, 2) - assert bz2f.read() == TEXT[len(TEXT) - 150:] + assert bz2f.read() == self.TEXT[len(self.TEXT) - 150:] bz2f.close() def test_seek_post_end(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.seek(150000) - assert bz2f.tell() == len(TEXT) + assert bz2f.tell() == len(self.TEXT) assert bz2f.read() == "" bz2f.close() def test_seek_post_end_twice(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.seek(150000) bz2f.seek(150000) - assert bz2f.tell() == len(TEXT) + assert bz2f.tell() == len(self.TEXT) assert bz2f.read() == "" bz2f.close() def test_seek_pre_start(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() + self.create_temp_file() - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2f = BZ2File("foo") bz2f.seek(-150) assert bz2f.tell() == 0 - assert bz2f.read() == TEXT + assert bz2f.read() == self.TEXT bz2f.close() def test_readline(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File from cStringIO import StringIO - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") raises(TypeError, bz2f.readline, None) - sio = StringIO(TEXT) + sio = StringIO(self.TEXT) for line in sio.readlines(): line_read = bz2f.readline() assert line_read == line bz2f.close() def test_read(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") raises(TypeError, bz2f.read, None) text_read = bz2f.read() - assert text_read == TEXT + assert text_read == self.TEXT bz2f.close() def test_read_chunk10(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") text_read = "" @@ -332,137 +238,65 @@ if not data: break text_read = "%s%s" % (text_read, data) - assert text_read == TEXT + assert text_read == self.TEXT bz2f.close() def test_read_100_bytes(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") - assert bz2f.read(100) == TEXT[:100] + assert bz2f.read(100) == self.TEXT[:100] bz2f.close() def test_universal_newlines_lf(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo", "rU") - assert bz2f.read() == TEXT + assert bz2f.read() == self.TEXT assert bz2f.newlines == "\n" bz2f.close() def test_universal_newlines_crlf(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File - create_temp_file(crlf=True) - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file(crlf=True) bz2f = BZ2File("foo", "rU") data = bz2f.read() - assert data == TEXT + assert data == self.TEXT assert bz2f.newlines == "\r\n" bz2f.close() def test_readlines(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File from cStringIO import StringIO - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") raises(TypeError, bz2f.readlines, None) - sio = StringIO(TEXT) + sio = StringIO(self.TEXT) assert bz2f.readlines() == sio.readlines() bz2f.close() def test_iterator(self): # XXX: to fix when the actual iterator protocol will be available - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File from cStringIO import StringIO - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") - sio = StringIO(TEXT) + sio = StringIO(self.TEXT) assert list(bz2f.get_iterator()) == sio.readlines() bz2f.close() def test_xreadlines(self): - def create_temp_file(crlf=False): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' - - f = open("foo", "wb") - - data = (DATA, DATA_CRLF)[crlf] - f.write(data) - f.close() - from bz2 import BZ2File from cStringIO import StringIO - create_temp_file() - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + self.create_temp_file() bz2f = BZ2File("foo") - sio = StringIO(TEXT) + sio = StringIO(self.TEXT) assert list(bz2f.xreadlines()) == sio.readlines() bz2f.close() @@ -470,9 +304,9 @@ # readlines()/xreadlines() for files containing no newline from bz2 import BZ2File - data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' + DATA = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' f = open("foo", "wb") - f.write(data) + f.write(DATA) f.close() bz2f = BZ2File("foo") @@ -485,51 +319,25 @@ bz2f.close() assert xlines == ['Test'] - def test_write(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - + def test_write(self): from bz2 import BZ2File - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' bz2f = BZ2File("foo", 'w') raises(TypeError, bz2f.write) - bz2f.write(TEXT) + bz2f.write(self.TEXT) bz2f.close() f = open("foo", "rb") - assert decompress(f.read()) == TEXT + assert self.decompress(f.read()) == self.TEXT f.close() def test_write_chunks_10(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - from bz2 import BZ2File - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' bz2f = BZ2File("foo", 'w') n = 0 while True: - data = TEXT[n * 10:(n + 1) * 10] + data = self.TEXT[n * 10:(n + 1) * 10] if not data: break @@ -538,33 +346,20 @@ bz2f.close() f = open("foo", "rb") - assert decompress(f.read()) == TEXT + assert self.decompress(f.read()) == self.TEXT f.close() def test_writelines(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - from bz2 import BZ2File from cStringIO import StringIO - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' bz2f = BZ2File("foo", 'w') raises(TypeError, bz2f.writelines) - sio = StringIO(TEXT) + sio = StringIO(self.TEXT) bz2f.writelines(sio.readlines()) bz2f.close() f = open("foo", "rb") - assert decompress(f.read()) == TEXT + assert self.decompress(f.read()) == self.TEXT f.close() @@ -572,6 +367,8 @@ def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_decompress = space.wrap(decompress) def test_creation(self): from bz2 import BZ2Compressor @@ -583,82 +380,45 @@ BZ2Compressor(9) def test_compress(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - from bz2 import BZ2Compressor - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' bz2c = BZ2Compressor() raises(TypeError, bz2c.compress) - data = bz2c.compress(TEXT) + data = bz2c.compress(self.TEXT) data = "%s%s" % (data, bz2c.flush()) - assert decompress(data) == TEXT + assert self.decompress(data) == self.TEXT def test_compress_huge_data(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - from bz2 import BZ2Compressor - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - HUGE_DATA = TEXT * 10000 + HUGE_DATA = self.TEXT * 10000 bz2c = BZ2Compressor() raises(TypeError, bz2c.compress) data = bz2c.compress(HUGE_DATA) data = "%s%s" % (data, bz2c.flush()) - assert decompress(data) == HUGE_DATA + assert self.decompress(data) == HUGE_DATA def test_compress_chunks_10(self): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - from bz2 import BZ2Compressor - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' bz2c = BZ2Compressor() n = 0 data = "" while True: - temp = TEXT[n * 10:(n + 1) * 10] + temp = self.TEXT[n * 10:(n + 1) * 10] if not temp: break data = "%s%s" % (data, bz2c.compress(temp)) n += 1 data = "%s%s" % (data, bz2c.flush()) - assert decompress(data) == TEXT + assert self.decompress(data) == self.TEXT class AppTestBZ2Decompressor: def setup_class(cls): space = gettestobjspace(usemodules=('bz2',)) cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_DATA = space.wrap(DATA) def test_creation(self): from bz2 import BZ2Decompressor @@ -676,117 +436,81 @@ def test_decompress(self): from bz2 import BZ2Decompressor - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2d = BZ2Decompressor() raises(TypeError, bz2d.decompress) - decompressed_data = bz2d.decompress(DATA) - assert decompressed_data == TEXT + decompressed_data = bz2d.decompress(self.DATA) + assert decompressed_data == self.TEXT def test_decompress_chunks_10(self): from bz2 import BZ2Decompressor - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2d = BZ2Decompressor() decompressed_data = "" n = 0 while True: - temp = DATA[n * 10:(n + 1) * 10] + temp = self.DATA[n * 10:(n + 1) * 10] if not temp: break decompressed_data = "%s%s" % (decompressed_data, bz2d.decompress(temp)) n += 1 - assert decompressed_data == TEXT + assert decompressed_data == self.TEXT def test_decompress_unused_data(self): # test with unused data. (data after EOF) from bz2 import BZ2Decompressor - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - bz2d = BZ2Decompressor() unused_data = "this is unused data" - decompressed_data = bz2d.decompress(DATA + unused_data) - assert decompressed_data == TEXT + decompressed_data = bz2d.decompress(self.DATA + unused_data) + assert decompressed_data == self.TEXT assert bz2d.unused_data == unused_data def test_EOF_error(self): from bz2 import BZ2Decompressor - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - bz2d = BZ2Decompressor() - bz2d.decompress(DATA) + bz2d.decompress(self.DATA) raises(EOFError, bz2d.decompress, "foo") +class AppTestBZ2ModuleFunctions: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_DATA = space.wrap(DATA) + cls.w_decompress = space.wrap(decompress) -def test_compress_function(): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - - from bz2 import compress - - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + def test_compress_function(self): + from bz2 import compress - raises(TypeError, compress, 123) - raises(ValueError, compress, "foo", 10) - raises(TypeError, compress, "foo", "foo") + raises(TypeError, compress, 123) + raises(ValueError, compress, "foo", 10) + raises(TypeError, compress, "foo", "foo") - data = compress(TEXT) - assert decompress(data) == TEXT + data = compress(self.TEXT) + assert self.decompress(data) == self.TEXT -def test_compress_function_huge_data(): - def decompress(data): - import popen2 - import bz2 - pop = popen2.Popen3("bunzip2", capturestderr=1) - pop.tochild.write(data) - pop.tochild.close() - res = pop.fromchild.read() - pop.fromchild.close() - if pop.wait() != 0: - res = bz2.decompress(data) - return res - - from bz2 import compress + def test_compress_function_huge_data(self): + from bz2 import compress - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' - HUGE_DATA = TEXT * 10000 + HUGE_DATA = self.TEXT * 10000 - data = compress(HUGE_DATA) - assert decompress(data) == HUGE_DATA + data = compress(HUGE_DATA) + assert self.decompress(data) == HUGE_DATA -def test_decompress_function(): - from bz2 import decompress - - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + def test_decompress_function(self): + import bz2 - raises(TypeError, decompress) - assert decompress("") == "" - decompressed_data = decompress(DATA) - assert decompressed_data == TEXT + raises(TypeError, bz2.decompress) + assert bz2.decompress("") == "" + decompressed_data = bz2.decompress(self.DATA) + assert decompressed_data == self.TEXT -def test_decompress_function_incomplete_data(): - from bz2 import decompress - - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - - raises(ValueError, decompress, DATA[:-10]) + def test_decompress_function_incomplete_data(self): + import bz2 + + raises(ValueError, bz2.decompress, self.DATA[:-10]) # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") From mwh at codespeak.net Tue Aug 8 11:15:26 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 11:15:26 +0200 (CEST) Subject: [pypy-svn] r31156 - pypy/dist/pypy/objspace/std/test Message-ID: <20060808091526.4B7D91006E@code0.codespeak.net> Author: mwh Date: Tue Aug 8 11:15:24 2006 New Revision: 31156 Added: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py - copied, changed from r31146, pypy/dist/pypy/objspace/std/test/test_dictstrobject.py Log: add automatic tests of the multidicts From mwh at codespeak.net Tue Aug 8 13:19:21 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 13:19:21 +0200 (CEST) Subject: [pypy-svn] r31158 - pypy/dist/pypy/objspace/std Message-ID: <20060808111921.CF3E210063@code0.codespeak.net> Author: mwh Date: Tue Aug 8 13:19:20 2006 New Revision: 31158 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: don't use generic, slow implementation of keys/values/items Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 13:19:20 2006 @@ -62,6 +62,13 @@ def itervalues(self): return RDictImplementation(self.space).itervalues() + def keys(self): + return [] + def values(self): + return [] + def items(self): + return [] + class RDictImplementation(DictImplementation): def __init__(self, space): self.space = space @@ -98,12 +105,12 @@ def itervalues(self): return self.content.itervalues() -## def keys(self): -## return [w_k for w_k in self.iterkeys()] -## def values(self): -## return [w_v for w_v in self.itervalues()] -## def items(self): -## return [(w_key, w_value) or w_key, w_value in self.iteritems()] + def keys(self): + return self.content.keys() + def values(self): + return self.content.values() + def items(self): + return self.content.items() class W_DictMultiObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef From rhymes at codespeak.net Tue Aug 8 14:24:08 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Tue, 8 Aug 2006 14:24:08 +0200 (CEST) Subject: [pypy-svn] r31160 - pypy/dist/pypy/module/bz2/test Message-ID: <20060808122408.E196810063@code0.codespeak.net> Author: rhymes Date: Tue Aug 8 14:24:05 2006 New Revision: 31160 Added: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/dist/pypy/module/bz2/test/test_bz2_file.py - copied, changed from r31155, pypy/dist/pypy/module/bz2/test/test_bz2.py Removed: pypy/dist/pypy/module/bz2/test/test_bz2.py Log: split tests Added: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py Tue Aug 8 14:24:05 2006 @@ -0,0 +1,178 @@ +from pypy.conftest import gettestobjspace +import os + +if os.name == "nt": + skip("bz2 module is not available on Windows") + +def setup_module(mod): + DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' + + def decompress(data): + import popen2 + import bz2 + pop = popen2.Popen3("bunzip2", capturestderr=1) + pop.tochild.write(data) + pop.tochild.close() + res = pop.fromchild.read() + pop.fromchild.close() + if pop.wait() != 0: + res = bz2.decompress(data) + return res + + mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + mod.DATA = DATA + mod.decompress = decompress + +def teardown_module(mod): + if os.path.exists("foo"): + os.unlink("foo") + + +class AppTestBZ2Compressor: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_decompress = space.wrap(decompress) + + def test_creation(self): + from bz2 import BZ2Compressor + + raises(TypeError, BZ2Compressor, "foo") + raises(ValueError, BZ2Compressor, 10) + + BZ2Compressor(1) + BZ2Compressor(9) + + def test_compress(self): + from bz2 import BZ2Compressor + + bz2c = BZ2Compressor() + raises(TypeError, bz2c.compress) + data = bz2c.compress(self.TEXT) + data = "%s%s" % (data, bz2c.flush()) + assert self.decompress(data) == self.TEXT + + def test_compress_huge_data(self): + from bz2 import BZ2Compressor + + HUGE_DATA = self.TEXT * 10000 + bz2c = BZ2Compressor() + raises(TypeError, bz2c.compress) + data = bz2c.compress(HUGE_DATA) + data = "%s%s" % (data, bz2c.flush()) + assert self.decompress(data) == HUGE_DATA + + def test_compress_chunks_10(self): + from bz2 import BZ2Compressor + + bz2c = BZ2Compressor() + n = 0 + data = "" + while True: + temp = self.TEXT[n * 10:(n + 1) * 10] + if not temp: + break + data = "%s%s" % (data, bz2c.compress(temp)) + n += 1 + data = "%s%s" % (data, bz2c.flush()) + assert self.decompress(data) == self.TEXT + +class AppTestBZ2Decompressor: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_DATA = space.wrap(DATA) + + def test_creation(self): + from bz2 import BZ2Decompressor + + raises(TypeError, BZ2Decompressor, "foo") + + BZ2Decompressor() + + def test_attribute(self): + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + assert bz2d.unused_data == "" + + def test_decompress(self): + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + raises(TypeError, bz2d.decompress) + decompressed_data = bz2d.decompress(self.DATA) + assert decompressed_data == self.TEXT + + def test_decompress_chunks_10(self): + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + decompressed_data = "" + n = 0 + while True: + temp = self.DATA[n * 10:(n + 1) * 10] + if not temp: + break + decompressed_data = "%s%s" % (decompressed_data, bz2d.decompress(temp)) + n += 1 + + assert decompressed_data == self.TEXT + + def test_decompress_unused_data(self): + # test with unused data. (data after EOF) + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + unused_data = "this is unused data" + decompressed_data = bz2d.decompress(self.DATA + unused_data) + assert decompressed_data == self.TEXT + assert bz2d.unused_data == unused_data + + def test_EOF_error(self): + from bz2 import BZ2Decompressor + + bz2d = BZ2Decompressor() + bz2d.decompress(self.DATA) + raises(EOFError, bz2d.decompress, "foo") + +class AppTestBZ2ModuleFunctions: + def setup_class(cls): + space = gettestobjspace(usemodules=('bz2',)) + cls.space = space + cls.w_TEXT = space.wrap(TEXT) + cls.w_DATA = space.wrap(DATA) + cls.w_decompress = space.wrap(decompress) + + def test_compress_function(self): + from bz2 import compress + + raises(TypeError, compress, 123) + raises(ValueError, compress, "foo", 10) + raises(TypeError, compress, "foo", "foo") + + data = compress(self.TEXT) + assert self.decompress(data) == self.TEXT + + def test_compress_function_huge_data(self): + from bz2 import compress + + HUGE_DATA = self.TEXT * 10000 + + data = compress(HUGE_DATA) + assert self.decompress(data) == HUGE_DATA + + def test_decompress_function(self): + import bz2 + + raises(TypeError, bz2.decompress) + assert bz2.decompress("") == "" + decompressed_data = bz2.decompress(self.DATA) + assert decompressed_data == self.TEXT + + def test_decompress_function_incomplete_data(self): + import bz2 + + raises(ValueError, bz2.decompress, self.DATA[:-10]) From auc at codespeak.net Tue Aug 8 15:23:34 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 8 Aug 2006 15:23:34 +0200 (CEST) Subject: [pypy-svn] r31161 - in pypy/dist/pypy/objspace: . cclp constraint constraint/test test Message-ID: <20060808132334.574C310063@code0.codespeak.net> Author: auc Date: Tue Aug 8 15:23:30 2006 New Revision: 31161 Modified: pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/constraint/domain.py pypy/dist/pypy/objspace/constraint/test/test_fd.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: many fixes, constraint vars, domain_of, .... Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 8 15:23:30 2006 @@ -9,8 +9,6 @@ #-- Singleton scheduler ------------------------------------------------ -class FunnyBoat: pass - class Scheduler(object): def __init__(self, space): @@ -89,7 +87,7 @@ import traceback traceback.print_exc() self.display_head() - thread._next = thread._prev = FunnyBoat + thread._next = thread._prev = None # cspace/threads account mgmt if thread._cspace is not None: count = self.dec_live_thread_count(thread._cspace) @@ -99,12 +97,14 @@ #-- cspace helper def is_stable(self, cspace): - if not self._per_space_live_threads.has_key(cspace): + assert isinstance(cspace, W_CSpace) + if cspace not in self._per_space_live_threads.keys(): #XXX meaning ? return True return self._per_space_live_threads[cspace] == 0 def wait_stable(self, cspace): + assert isinstance(cspace, W_CSpace) if self.is_stable(cspace): return curr = ClonableCoroutine.w_getcurrent(self.space) @@ -120,11 +120,13 @@ #-- cspace -> thread_count helpers def inc_live_thread_count(self, cspace): + assert isinstance(cspace, W_CSpace) count = self._per_space_live_threads.get(cspace, 0) + 1 self._per_space_live_threads[cspace] = count return count def dec_live_thread_count(self, cspace): + assert isinstance(cspace, W_CSpace) count = self._per_space_live_threads[cspace] -1 assert count >= 0 self._per_space_live_threads[cspace] = count Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Tue Aug 8 15:23:30 2006 @@ -47,18 +47,20 @@ def __init__(self, space, thread, parent=None): assert isinstance(thread, ClonableCoroutine) - assert (parent is None) or isinstance(parent, CSpace) + assert (parent is None) or isinstance(parent, W_CSpace) self.space = space # the object space ;-) self.parent = parent self.main_thread = thread # choice mgmt self._choice = newvar(space) self._committed = newvar(space) + # constraint store ... + def w_ask(self): scheduler[0].wait_stable(self) self.space.wait(self._choice) - return self.space.newint(self._choice) + return self._choice def choose(self, n): assert n > 1 @@ -74,10 +76,13 @@ def w_commit(self, w_n): assert self.space.is_true(self.space.is_bound(self._choice)) assert 0 < self.space.int_w(w_n) - assert self.space.int_w(w_n) <= self._choice + assert self.space.int_w(w_n) <= self._choice.w_bound_to self.space.bind(self._committed, w_n) self._choice = newvar(self.space) - + + + def tell(self, w_constraint): + pass W_CSpace.typedef = typedef.TypeDef("W_CSpace", ask = gateway.interp2app(W_CSpace.w_ask), @@ -86,13 +91,6 @@ -## def is_top_level(self): -## return self.parent is None - -## def current_space(): -## #XXX return w_getcurrent().cspace -## pass - ## def clone(self): ## if self.is_top_level(): @@ -105,20 +103,3 @@ ## tclone.cspace = new ## new.threads[tclone] = True -## def choose(self, n): -## if self.is_top_level(): -## raise OperationError(self.space.w_RuntimeError, -## self.space.wrap("Choose"+forbidden_boilerplate)) - -## def ask(self): -## if self.is_top_level(): -## raise OperationError(self.space.w_RuntimeError, -## self.space.wrap("Ask"+forbidden_boilerplate)) -## #XXX basically hang until a call to choose, then return n - -## def commit(self, n): -## if self.is_top_level(): -## raise OperationError(self.space.w_RuntimeError, -## self.space.wrap("Commit"+forbidden_boilerplate)) -## # ensure 0 < n < chosen n -## # ... Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Tue Aug 8 15:23:30 2006 @@ -1,6 +1,7 @@ -from pypy.interpreter import baseobjspace, typedef +from pypy.interpreter import baseobjspace, gateway, typedef from pypy.objspace.cclp.misc import w, ClonableCoroutine +from pypy.objspace.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root @@ -20,6 +21,7 @@ prettyfy_id(id(w_self))) __str__ = __repr__ + class W_Future(W_Var): "a read-only-by-its-consummer variant of logic. var" def __init__(w_self, space): @@ -27,6 +29,18 @@ w_self._client = ClonableCoroutine.w_getcurrent(space) w("FUT", str(w_self)) + +class W_CVar(W_Var): + def __init__(w_self, space, w_dom): + assert isinstance(w_dom, W_FiniteDomain) + W_Var.__init__(w_self, space) + w_self.w_dom = w_dom + +def domain_of(space, w_v): + assert isinstance(w_v, W_CVar) + return w_v.w_dom +app_domain_of = gateway.interp2app(domain_of) + #-- Exception types ---------------------------------------- class W_FailedValue(W_Root): @@ -36,50 +50,6 @@ def __init__(w_self, exc): w_self.exc = exc -#-- Something to hold the ring of coros -------------------------------- - -## class Triple(object): - -## def __init__(self, thread): -## assert isinstance(thread, ClonableCoroutine) -## self._thread = thread -## self._prev = self._next = self - -## def get_next(self): -## return self._next - -## def get_prev(self): -## return self._prev - -## def set_next(self, triple): -## assert isinstance(triple, Triple) -## self._next = triple - -## def set_prev(self, triple): -## assert isinstance(triple, Triple) -## self._prev = triple - -## next = property(get_next, set_next) -## prev = property(get_prev, set_prev) - -## def insert_before(self, triple): -## assert isinstance(triple, Triple) -## before = self.prev -## # ... -## before.next = triple -## triple.prev = before -## # ... -## self.prev = triple -## triple.next = self - -## def __str__(self): -## curr = self -## out = ['[', str(id(self._thread))] -## while curr != self: -## curr = self.next -## out.append(str(id(curr._thread))) -## return ''.join(out) - #-- Misc --------------------------------------------------- def deref(space, w_var): Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Tue Aug 8 15:23:30 2006 @@ -6,8 +6,9 @@ from pypy.objspace.cclp.misc import w, v, ClonableCoroutine from pypy.objspace.cclp.global_state import scheduler -from pypy.objspace.cclp.types import deref, W_Var, W_Future, W_FailedValue +from pypy.objspace.cclp.types import deref, W_Var, W_CVar, W_Future, W_FailedValue +from pypy.objspace.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root all_mms = {} @@ -18,6 +19,14 @@ return w_v app_newvar = gateway.interp2app(newvar) +def domain(space, w_values): + assert isinstance(w_values, W_ListObject) + w_dom = W_FiniteDomain(space, w_values) + w_var = W_CVar(space, w_dom) + w("CVAR", str(w_var)) + return w_var +app_domain = gateway.interp2app(domain) + #-- Wait ------------------------------------------------- def wait__Root(space, w_obj): @@ -110,7 +119,7 @@ all_mms['is_bound'] = is_bound_mm -def alias_of(space, w_var1, w_var2): # FIXME: appears to block +def alias_of(space, w_var1, w_var2): assert isinstance(w_var1, W_Var) assert isinstance(w_var2, W_Var) assert space.is_true(space.is_free(w_var1)) @@ -118,9 +127,9 @@ w_curr = w_var1 while 1: w_next = w_curr.w_bound_to - if space.is_true(space.is_nb_(w_next, w_var2)): + if w_next is w_var2: return space.newbool(True) - if space.is_true(space.is_nb_(w_next, w_var1)): + if w_next is w_var1: break w_curr = w_next return space.newbool(False) @@ -140,11 +149,11 @@ w_curr = w_next -def raise_unification_failure(space): +def raise_unification_failure(space, comment="Unification failure"): """raises a specific exception for bind/unify should fail the current comp. space at some point""" raise OperationError(space.w_UnificationError, - space.wrap("Unification failure")) + space.wrap(comment)) # to signal a future binding exception def raise_future_binding(space): @@ -170,19 +179,26 @@ #w("var val", str(id(w_var))) # 3. var and value if space.is_true(space.is_free(w_var)): - return _assign(space, w_var, w_obj) + return _assign_aliases(space, w_var, w_obj) if space.is_true(space.eq(w_var.w_bound_to, w_obj)): return raise OperationError(space.w_RebindingError, space.wrap("Cannot bind twice but two identical values")) - def bind__Future_Root(space, w_fut, w_obj): #v("future val", str(id(w_fut))) if w_fut._client == ClonableCoroutine.w_getcurrent(space): raise_future_binding(space) return bind__Var_Root(space, w_fut, w_obj) # call-next-method ? +def bind__CVar_Root(space, w_cvar, w_obj): + #XXX we should (want to) be able to test membership + # in a wrapped against wrappeds into a non-wrapped dict + if [True for elt in w_cvar.w_dom._values + if space.is_true(space.eq(w_obj, elt))]: + return bind__Var_Root(space, w_cvar, w_obj) + raise_unification_failure(space, "value not in variable domain") + def bind__Var_Var(space, w_v1, w_v2): #w("var var") if space.is_true(space.is_bound(w_v1)): @@ -192,10 +208,10 @@ deref(space, w_v1), deref(space, w_v2)) # 2. a (obj unbound, var bound) - return _assign(space, w_v2, deref(space, w_v1)) + return _assign_aliases(space, w_v2, deref(space, w_v1)) elif space.is_true(space.is_bound(w_v2)): # 2. b (var unbound, obj bound) - return _assign(space, w_v1, deref(space, w_v2)) + return _assign_aliases(space, w_v1, deref(space, w_v2)) else: # 1. both are unbound return _alias(space, w_v1, w_v2) @@ -205,6 +221,24 @@ raise_future_binding(space) return bind__Var_Var(space, w_fut, w_var) +def bind__CVar_CVar(space, w_cvar1, w_cvar2): + w_inter_dom = space.intersection(w_cvar1.w_dom, w_cvar2.w_dom) + if w_inter_dom.__len__() > 0: + if w_inter_dom.__len__() == 1: + w_value = w_inter_dom.get_values()[0] + _assign_aliases(space, w_cvar1, w_value) + _assign_aliases(space, w_cvar2, w_value) + else: + w_cvar1.w_dom = w_cvar2.w_dom = w_inter_dom + _alias(space, w_cvar1, w_cvar2) + else: + raise_unification_failure(space, "incompatible domains") + +def bind__CVar_Var(space, w_cvar, w_var): + if space.is_true(space.is_bound(w_var)): + return bind__CVar_Root(space, w_cvar, w_var) + return bind__Var_Var(space, w_cvar, w_var) + def bind__Var_Future(space, w_var, w_fut): if space.is_true(space.is_bound(w_fut)): #XXX write a test for me ! return bind__Var_Root(space, w_var, deref(space, w_fut)) @@ -218,16 +252,19 @@ bind_mm.register(bind__Future_Root, W_Future, W_Root) bind_mm.register(bind__Future_Var, W_Future, W_Var) bind_mm.register(bind__Var_Future, W_Var, W_Future) +bind_mm.register(bind__CVar_CVar, W_CVar, W_CVar) +bind_mm.register(bind__CVar_Root, W_CVar, W_Root) +bind_mm.register(bind__CVar_Var, W_CVar, W_Var) all_mms['bind'] = bind_mm -def _assign(space, w_var, w_val): +def _assign_aliases(space, w_var, w_val): w(" :assign") assert isinstance(w_var, W_Var) assert isinstance(w_val, W_Root) w_curr = w_var while 1: w_next = w_curr.w_bound_to - w_curr.w_bound_to = w_val + _assign(space, w_curr, w_val) # notify the blocked threads scheduler[0].unblock_on(w_curr) if space.is_true(space.is_nb_(w_next, w_var)): @@ -236,6 +273,14 @@ w_curr = w_next w(" :assigned") return space.w_None + +def _assign(space, w_var, w_val): + assert isinstance(w_var, W_Var) + if isinstance(w_var, W_CVar): + if not w_val in w_var.w_dom._values: + raise_unification_failure(space, "assignment out of domain") + w_var.w_bound_to = w_val + def _alias(space, w_v1, w_v2): """appends one var to the alias chain of another Modified: pypy/dist/pypy/objspace/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/constraint/domain.py Tue Aug 8 15:23:30 2006 @@ -91,6 +91,7 @@ def size(self): """computes the size of a finite domain""" return len(self._values) + __len__ = size def w_get_values(self): """return all the values in the domain @@ -136,9 +137,9 @@ def intersection__FiniteDomain_FiniteDomain(space, w_fd1, w_fd2): w_v1 = w_fd1._values - w_res = [w_v for w_v in w_fd2._values + res = [w_v for w_v in w_fd2._values if w_v in w_v1] - return make_fd(space, space.newlist(w_res)) + return make_fd(space, space.newlist(res)) intersection_mm = StdObjSpaceMultiMethod('intersection', 2) intersection_mm.register(intersection__FiniteDomain_FiniteDomain, Modified: pypy/dist/pypy/objspace/constraint/test/test_fd.py ============================================================================== --- pypy/dist/pypy/objspace/constraint/test/test_fd.py (original) +++ pypy/dist/pypy/objspace/constraint/test/test_fd.py Tue Aug 8 15:23:30 2006 @@ -2,7 +2,6 @@ from py.test import skip class AppTest_FiniteDomain(object): - skip("currently unplugged") def setup_class(cls): cls.space = gettestobjspace('logic', usemodules=('_stackless', )) Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Tue Aug 8 15:23:30 2006 @@ -27,17 +27,19 @@ from pypy.objspace.cclp.variable import app_newvar, wait, app_wait, app_wait_needed, \ app_is_aliased, app_is_free, app_is_bound, app_alias_of, alias_of, app_bind, \ - app_unify, W_Var, W_Future, all_mms as variable_mms + app_unify, W_Var, W_CVar, W_Future, app_domain, all_mms as variable_mms + +from pypy.objspace.cclp.types import app_domain_of all_mms.update(variable_mms) #-- CONSTRAINTS ---------------------------------------------- ## #------ domains ------------------ -## from pypy.objspace.constraint import domain -## all_mms.update(domain.all_mms) +from pypy.objspace.constraint import domain +all_mms.update(domain.all_mms) -## W_FiniteDomain = domain.W_FiniteDomain +W_FiniteDomain = domain.W_FiniteDomain ## #-------- computationspace -------- ## from pypy.objspace.constraint import computationspace @@ -192,8 +194,9 @@ # multimethods hack space.model.typeorder[W_Var] = [(W_Var, None), (W_Root, None)] # None means no conversion space.model.typeorder[W_Future] = [(W_Future, None), (W_Var, None)] + space.model.typeorder[W_CVar] = [(W_CVar, None), (W_Var, None)] space.model.typeorder[W_CSpace] = [(W_CSpace, None), (baseobjspace.Wrappable, None)] -## space.model.typeorder[W_FiniteDomain] = [(W_FiniteDomain, None), (W_Root, None)] + space.model.typeorder[W_FiniteDomain] = [(W_FiniteDomain, None), (W_Root, None)] for name in all_mms.keys(): @@ -211,13 +214,12 @@ setattr(space, name, boundmethod) # store into 'space' instance # /multimethods hack - # XXXprovide a UnificationError exception - # patching the table in-place? - #space.ExceptionTable.append('UnificationError') - #space.ExceptionTable.sort() # hmmm - space.setitem(space.builtin.w_dict, space.wrap('newvar'), space.wrap(app_newvar)) + space.setitem(space.builtin.w_dict, space.wrap('domain'), + space.wrap(app_domain)) + space.setitem(space.builtin.w_dict, space.wrap('domain_of'), + space.wrap(app_domain_of)) space.setitem(space.builtin.w_dict, space.wrap('is_free'), space.wrap(app_is_free)) space.setitem(space.builtin.w_dict, space.wrap('is_bound'), @@ -234,10 +236,10 @@ ## space.setitem(space.builtin.w_dict, space.wrap('newspace'), ## space.wrap(computationspace.app_newspace)) ## #-- domain ------- -## space.setitem(space.builtin.w_dict, space.wrap('FiniteDomain'), -## space.wrap(domain.app_make_fd)) -## space.setitem(space.builtin.w_dict, space.wrap('intersection'), -## space.wrap(domain.app_intersection)) + space.setitem(space.builtin.w_dict, space.wrap('FiniteDomain'), + space.wrap(domain.app_make_fd)) + space.setitem(space.builtin.w_dict, space.wrap('intersection'), + space.wrap(domain.app_intersection)) ## #-- constraint ---- ## space.setitem(space.builtin.w_dict, space.wrap('make_expression'), ## space.wrap(constraint.app_make_expression)) Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Tue Aug 8 15:23:30 2006 @@ -621,6 +621,55 @@ assert len(sched_all()['threads']) == 1 +class AppTest_CompSpace(object): + + def setup_class(cls): + cls.space = gettestobjspace('logic', usemodules=("_stackless",)) + + def test_cvar(self): + + d = domain([1, 2, 4]) + + raises(UnificationError, bind, d, 42) + bind(d, 2) + assert d == 2 + + class Foo(object): + pass + + f = Foo() + d = domain([Foo(), f, Foo()]) + raises(UnificationError, bind, d, Foo()) + bind(d, f) + assert d == f + + d1 = domain([1, 2, 3]) + d2 = domain([2, 3, 4]) + d3 = domain([5, 6]) + raises(UnificationError, unify, d1, d3) + unify(d1, d2) + assert alias_of(d1, d2) + assert domain_of(d1) == domain_of(d2) == FiniteDomain([2, 3]) + + d1 = domain([1, 2, 3]) + d4 = domain([3, 4]) + unify(d1, d4) + assert d1 == d4 == 3 + + d1 = domain([1, 2]) + x = newvar() + unify(d1, x) + assert alias_of(x, d1) + raises(UnificationError, unify, x, 42) + + d1 = domain([1, 2]) + x = newvar() + unify(d1, x) + assert alias_of(x, d1) + unify(x, 2) + assert d1 == x == 2 + #XXX and a bunch of app-level functions + #raises(TypeError, domain_of, x) def test_newspace_ask_wait(self): @@ -660,19 +709,35 @@ assert X == 2 - def test_ask_choose(self): + def test_more_ask_choose(self): - def chooser(X): - choice = choose(3) - unify(X, choice) + def chooser(vec, X): + for v in vec: + choice = choose(v) + assert choice == v + unify(X, 'done') def asker(cspace): - choices = cspace.ask() - cspace.commit(2) + while 1: + choices = cspace.ask() + if choices == 1: # success ! + break + cspace.commit(choices) + # choices >= 1 + v = range(2, 9) X = newvar() - - s = newspace(chooser, X) + s = newspace(chooser, v, X) stacklet(asker, s) + schedule() - assert X == 2 + + assert len(sched_all()['asking']) == 1 + assert sched_all()['space_accounting'][0][1] == 0 + + assert X == 'done' + schedule() + assert len(sched_all()['threads']) == 1 + + + From mwh at codespeak.net Tue Aug 8 15:36:46 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 15:36:46 +0200 (CEST) Subject: [pypy-svn] r31162 - pypy/dist/pypy/objspace/std Message-ID: <20060808133646.8D3A110069@code0.codespeak.net> Author: mwh Date: Tue Aug 8 15:36:45 2006 New Revision: 31162 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: add a "MeasuringDictImplementation" Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 15:36:45 2006 @@ -112,11 +112,131 @@ def items(self): return self.content.items() +# yeah, so this should do something clever with pypy.config +MEASURE_DICT = False + +if MEASURE_DICT: + _dict_infos = [] + + class DictInfo: + def __init__(self): + self.getitems = 0; self.setitems = 0; self.delitems = 0 + self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 + self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 + self.keys = 0; self.values = 0; self.items = 0 + + self.maxcontents = 0 + + self.reads = 0 + self.writes = 0 + self.iterations = 0 + self.listings = 0 + + _dict_infos.append(self) + + class MeasuringDictImplementation(DictImplementation): + def __init__(self, space): + self.space = space + self.content = r_dict(space.eq_w, space.hash_w) + self.info = DictInfo() + + def __repr__(self): + return "%s<%s>" % (self.__class__.__name__, self.content) + + def getitem(self, w_key): + self.info.getitems += 1 + self.info.reads += 1 + return self.content[w_key] + def setitem(self, w_key, w_value): + self.info.setitems += 1 + self.info.writes += 1 + self.info.maxcontents = max(self.info.maxcontents, len(self.content)) + self.content[w_key] = w_value + return self + def delitem(self, w_key): + self.info.delitems += 1 + self.info.writes += 1 + del self.content[w_key] + return self + + def length(self): + self.info.lengths += 1 + return len(self.content) + def clear(self): + self.info.clears += 1 + self.info.writes += 1 + self.content.clear() + return self + def has_key(self, w_lookup): + self.info.has_keys += 1 + self.info.reads += 1 + return w_lookup in self.content + def get(self, w_lookup, w_default): + self.info.gets += 1 + self.info.reads += 1 + return self.content.get(w_lookup, w_default) + + def iteritems(self): + self.info.iteritems += 1 + self.info.iterations += 1 + return self.content.iteritems() + def iterkeys(self): + self.info.iterkeys += 1 + self.info.iterations += 1 + return self.content.iterkeys() + def itervalues(self): + self.info.itervalues += 1 + self.info.iterations += 1 + return self.content.itervalues() + + def keys(self): + self.info.keys += 1 + self.info.listings += 1 + return self.content.keys() + def values(self): + self.info.values += 1 + self.info.listings += 1 + return self.content.values() + def items(self): + self.info.items += 1 + self.info.listings += 1 + return self.content.items() + + def reportDictInfo(): + d = {} + if not _dict_infos: + return + rwratios = [] + maxcontents = [] + for info in _dict_infos: + for attr in info.__dict__: + if attr == 'maxcontents': + continue + d[attr] = d.get(attr, 0) + info.__dict__[attr] + if info.writes: + rwratios.append(float(info.reads)/(info.writes)) + elif info.reads: + rwratios.append('inf') + else: + rwratios.append('nan') + maxcontents.append(info.maxcontents) + import cPickle + cPickle.dump(_dict_infos, open('dictinfos.pickle', 'wb')) + open('rwratios.txt', 'w').write(repr(rwratios)) + open('maxcontents.txt', 'w').write(repr(maxcontents)) + print d + + import atexit + atexit.register(reportDictInfo) + class W_DictMultiObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef def __init__(w_self, space, w_otherdict=None): - w_self.implementation = EmptyDictImplementation(space) + if MEASURE_DICT: + w_self.implementation = MeasuringDictImplementation(space) + else: + w_self.implementation = EmptyDictImplementation(space) if w_otherdict is not None: from pypy.objspace.std.dicttype import dict_update__ANY_ANY dict_update__ANY_ANY(space, w_self, w_otherdict) From auc at codespeak.net Tue Aug 8 16:20:11 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Tue, 8 Aug 2006 16:20:11 +0200 (CEST) Subject: [pypy-svn] r31165 - in pypy/dist/pypy: module/_stackless objspace/cclp objspace/test Message-ID: <20060808142011.4287010063@code0.codespeak.net> Author: auc Date: Tue Aug 8 16:20:07 2006 New Revision: 31165 Modified: pypy/dist/pypy/module/_stackless/clonable.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: transl. fix Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Tue Aug 8 16:20:07 2006 @@ -32,7 +32,7 @@ space.getexecutioncontext().subcontext_new(self) self._dead = False self._next = self._prev = self - self._cspace = None + self._cspace = space.w_None def hello(self): if we_are_translated(): Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Tue Aug 8 16:20:07 2006 @@ -89,7 +89,7 @@ self.display_head() thread._next = thread._prev = None # cspace/threads account mgmt - if thread._cspace is not None: + if thread._cspace is not self.space.w_None: count = self.dec_live_thread_count(thread._cspace) if count == 0: del self._per_space_live_threads[thread._cspace] @@ -208,7 +208,7 @@ "insert 'thread' at end of running queue" assert isinstance(thread, ClonableCoroutine) # cspace account mgmt - if thread._cspace != None: + if thread._cspace != self.space.w_None: self._per_space_live_threads.get(thread._cspace, 0) self.inc_live_thread_count(thread._cspace) self._chain_insert(thread) @@ -226,7 +226,7 @@ blocked.append(thread) self._blocked[thread] = True # cspace accounting - if thread._cspace is not None: + if thread._cspace is not self.space.w_None: self.dec_live_thread_count(thread._cspace) def unblock_on(self, w_var): @@ -240,7 +240,7 @@ for thr in blocked: del self._blocked[thr] # cspace accounting - if thr._cspace is not None: + if thr._cspace is not self.space.w_None: self.inc_live_thread_count(thr._cspace) def add_to_blocked_byneed(self, w_var, thread): @@ -255,7 +255,7 @@ blocked.append(thread) self._blocked[thread] = True # cspace accounting - if thread._cspace is not None: + if thread._cspace is not self.space.w_None: self.dec_live_thread_count(thread._cspace) def unblock_byneed_on(self, w_var): @@ -271,7 +271,7 @@ for thr in blocked: del self._blocked[thr] # cspace accounting - if thr._cspace is not None: + if thr._cspace is not self.space.w_None: self.inc_live_thread_count(thr._cspace) Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Tue Aug 8 16:20:07 2006 @@ -34,7 +34,7 @@ assert isinstance(w_n, W_IntObject) n = space.int_w(w_n) cspace = ClonableCoroutine.w_getcurrent(space)._cspace - if cspace != None: + if cspace != space.w_None: assert isinstance(cspace, W_CSpace) return cspace.choose(n) raise OperationError(space.w_RuntimeError, @@ -45,9 +45,9 @@ class W_CSpace(baseobjspace.Wrappable): - def __init__(self, space, thread, parent=None): + def __init__(self, space, thread, parent): assert isinstance(thread, ClonableCoroutine) - assert (parent is None) or isinstance(parent, W_CSpace) + assert (parent is space.w_None) or isinstance(parent, W_CSpace) self.space = space # the object space ;-) self.parent = parent self.main_thread = thread Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Tue Aug 8 16:20:07 2006 @@ -739,5 +739,3 @@ schedule() assert len(sched_all()['threads']) == 1 - - From rhymes at codespeak.net Tue Aug 8 16:25:19 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Tue, 8 Aug 2006 16:25:19 +0200 (CEST) Subject: [pypy-svn] r31166 - pypy/dist/pypy/module/bz2 Message-ID: <20060808142519.177E310063@code0.codespeak.net> Author: rhymes Date: Tue Aug 8 16:25:18 2006 New Revision: 31166 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: the cast is not needed Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Tue Aug 8 16:25:18 2006 @@ -89,7 +89,7 @@ if BZ_CONFIG_ERROR: if sizeof(c_long) >= 8 or sizeof(c_longlong) >= 8: def _bzs_total_out(bzs): - return long(bzs.total_out_hi32 << 32) + bzs.total_out_lo32 + return (bzs.total_out_hi32 << 32) + bzs.total_out_lo32 else: def _bzs_total_out(bzs): return bzs.total_out_lo32 From mwh at codespeak.net Tue Aug 8 17:26:37 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 17:26:37 +0200 (CEST) Subject: [pypy-svn] r31171 - pypy/dist/pypy/interpreter Message-ID: <20060808152637.518B310069@code0.codespeak.net> Author: mwh Date: Tue Aug 8 17:26:36 2006 New Revision: 31171 Modified: pypy/dist/pypy/interpreter/function.py Log: delay creating a Function's __dict__ until it's asked for. Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Tue Aug 8 17:26:36 2006 @@ -25,7 +25,7 @@ self.w_func_globals = w_globals # the globals dictionary self.closure = closure # normally, list of Cell instances or None self.defs_w = defs_w # list of w_default's - self.w_func_dict = space.newdict([]) + self.w_func_dict = None # filled out below if needed self.w_module = None def __repr__(self): @@ -120,6 +120,8 @@ stkargs.valuestack = None def getdict(self): + if self.w_func_dict is None: + self.w_func_dict = self.space.newdict([]) return self.w_func_dict def setdict(self, space, w_dict): From mwh at codespeak.net Tue Aug 8 17:36:59 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 17:36:59 +0200 (CEST) Subject: [pypy-svn] r31173 - pypy/dist/pypy/objspace/std Message-ID: <20060808153659.6ADA010069@code0.codespeak.net> Author: mwh Date: Tue Aug 8 17:36:58 2006 New Revision: 31173 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: record a guess at where the dictionary was allocated from. fix stupid, stupid bug in maxcontents calculation. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 17:36:58 2006 @@ -128,10 +128,20 @@ self.maxcontents = 0 self.reads = 0 + self.hits = self.misses = 0 self.writes = 0 self.iterations = 0 self.listings = 0 + # very probable stack from here: + # 0 - us + # 1 - MeasuringDictImplementation.__init__ + # 2 - W_DictMultiObject.__init__ + # 3 - space.newdict + # 4 - newdict's caller. let's look at that + frame = sys._getframe(4) + self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) + _dict_infos.append(self) class MeasuringDictImplementation(DictImplementation): @@ -150,8 +160,8 @@ def setitem(self, w_key, w_value): self.info.setitems += 1 self.info.writes += 1 - self.info.maxcontents = max(self.info.maxcontents, len(self.content)) self.content[w_key] = w_value + self.info.maxcontents = max(self.info.maxcontents, len(self.content)) return self def delitem(self, w_key): self.info.delitems += 1 @@ -212,7 +222,10 @@ for attr in info.__dict__: if attr == 'maxcontents': continue - d[attr] = d.get(attr, 0) + info.__dict__[attr] + v = info.__dict__[attr] + if not isinstance(v, int): + continue + d[attr] = d.get(attr, 0) + v if info.writes: rwratios.append(float(info.reads)/(info.writes)) elif info.reads: @@ -222,8 +235,7 @@ maxcontents.append(info.maxcontents) import cPickle cPickle.dump(_dict_infos, open('dictinfos.pickle', 'wb')) - open('rwratios.txt', 'w').write(repr(rwratios)) - open('maxcontents.txt', 'w').write(repr(maxcontents)) + print 'reporting on', len(_dict_infos), 'dictionaries' print d import atexit From wanja at codespeak.net Tue Aug 8 17:39:29 2006 From: wanja at codespeak.net (wanja at codespeak.net) Date: Tue, 8 Aug 2006 17:39:29 +0200 (CEST) Subject: [pypy-svn] r31174 - pypy/dist/pypy/doc/image Message-ID: <20060808153929.B49DF10069@code0.codespeak.net> Author: wanja Date: Tue Aug 8 17:39:27 2006 New Revision: 31174 Added: pypy/dist/pypy/doc/image/mallorca-trailer.jpg (contents, props changed) Log: a thumbnail image for the recently edited mallorca trailer Added: pypy/dist/pypy/doc/image/mallorca-trailer.jpg ============================================================================== Binary file. No diff available. From mwh at codespeak.net Tue Aug 8 19:01:02 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 8 Aug 2006 19:01:02 +0200 (CEST) Subject: [pypy-svn] r31178 - pypy/dist/pypy/objspace/std Message-ID: <20060808170102.9157310069@code0.codespeak.net> Author: mwh Date: Tue Aug 8 19:01:01 2006 New Revision: 31178 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: record more stuff: hits and misses, whether a non-string key was seen and how long the dictionary was alive for. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 8 19:01:01 2006 @@ -116,6 +116,7 @@ MEASURE_DICT = False if MEASURE_DICT: + import time _dict_infos = [] class DictInfo: @@ -133,6 +134,12 @@ self.iterations = 0 self.listings = 0 + self.seen_non_string = 0 + self.seen_non_string_in_read_first = 0 + + self.createtime = time.time() + self.lifetime = -1.0 + # very probable stack from here: # 0 - us # 1 - MeasuringDictImplementation.__init__ @@ -143,27 +150,60 @@ self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) _dict_infos.append(self) + def __repr__(self): + args = [] + for k in sorted(self.__dict__): + v = self.__dict__[k] + if v != 0: + args.append('%s=%r'%(k, v)) + return ''%(', '.join(args),) + + class OnTheWayOut: + def __init__(self, info): + self.info = info + def __del__(self): + self.info.lifetime = time.time() - self.info.createtime class MeasuringDictImplementation(DictImplementation): def __init__(self, space): self.space = space self.content = r_dict(space.eq_w, space.hash_w) self.info = DictInfo() + self.thing_with_del = OnTheWayOut(self.info) def __repr__(self): return "%s<%s>" % (self.__class__.__name__, self.content) + def _is_str(self, w_key): + space = self.space + return space.is_true(space.isinstance(w_key, space.w_str)) + def _read(self, w_key): + self.info.reads += 1 + if not self.info.seen_non_string and not self._is_str(w_key): + self.info.seen_non_string = True + self.info.seen_non_string_in_read_first = True + hit = w_key in self.content + if hit: + self.info.hits += 1 + else: + self.info.misses += 1 + def getitem(self, w_key): self.info.getitems += 1 - self.info.reads += 1 + self._read(w_key) return self.content[w_key] def setitem(self, w_key, w_value): + if not self.info.seen_non_string and not self._is_str(w_key): + self.info.seen_non_string = True self.info.setitems += 1 self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) return self def delitem(self, w_key): + if not self.info.seen_non_string and not self._is_str(w_key): + self.info.seen_non_string_in_read_first = True + self.info.seen_non_string = True self.info.delitems += 1 self.info.writes += 1 del self.content[w_key] @@ -179,11 +219,11 @@ return self def has_key(self, w_lookup): self.info.has_keys += 1 - self.info.reads += 1 + self._read(w_lookup) return w_lookup in self.content def get(self, w_lookup, w_default): self.info.gets += 1 - self.info.reads += 1 + self._read(w_lookup) return self.content.get(w_lookup, w_default) def iteritems(self): @@ -216,8 +256,8 @@ d = {} if not _dict_infos: return - rwratios = [] - maxcontents = [] + stillAlive = 0 + totLifetime = 0.0 for info in _dict_infos: for attr in info.__dict__: if attr == 'maxcontents': @@ -226,16 +266,16 @@ if not isinstance(v, int): continue d[attr] = d.get(attr, 0) + v - if info.writes: - rwratios.append(float(info.reads)/(info.writes)) - elif info.reads: - rwratios.append('inf') + if info.lifetime != -1.0: + totLifetime += info.lifetime else: - rwratios.append('nan') - maxcontents.append(info.maxcontents) + stillAlive += 1 import cPickle cPickle.dump(_dict_infos, open('dictinfos.pickle', 'wb')) print 'reporting on', len(_dict_infos), 'dictionaries' + if stillAlive != len(_dict_infos): + print 'average lifetime', totLifetime/(len(_dict_infos) - stillAlive), + print '('+str(stillAlive), 'still alive)' print d import atexit From wanja at codespeak.net Tue Aug 8 19:15:35 2006 From: wanja at codespeak.net (wanja at codespeak.net) Date: Tue, 8 Aug 2006 19:15:35 +0200 (CEST) Subject: [pypy-svn] r31179 - pypy/dist/pypy/doc Message-ID: <20060808171535.1498B1006F@code0.codespeak.net> Author: wanja Date: Tue Aug 8 19:15:33 2006 New Revision: 31179 Modified: pypy/dist/pypy/doc/video-index.txt Log: added the mallorca trailer description, image and url / also changed the order of the videos Modified: pypy/dist/pypy/doc/video-index.txt ============================================================================== --- pypy/dist/pypy/doc/video-index.txt (original) +++ pypy/dist/pypy/doc/video-index.txt Tue Aug 8 19:15:33 2006 @@ -43,9 +43,7 @@ -------------------------- 130mb: http://codespeak.net/download/pypy/video/pycon-trailer.avi.torrent - 71mb: http://codespeak.net/download/pypy/video/pycon-trailer-medium.avi.torrent - 50mb: http://codespeak.net/download/pypy/video/pycon-trailer-320x240.avi.torrent .. image:: image/pycon-trailer.jpg @@ -63,7 +61,6 @@ ------------------------- 440mb: http://codespeak.net/download/pypy/video/interview-timpeters-v2.avi.torrent - 138mb: http://codespeak.net/download/pypy/video/interview-timpeters-320x240.avi.torrent .. image:: image/interview-timpeters.jpg @@ -83,7 +80,6 @@ --------------------------- 155mb: http://codespeak.net/download/pypy/video/interview-bobippolito-v2.avi.torrent - 50mb: http://codespeak.net/download/pypy/video/interview-bobippolito-320x240.avi.torrent .. image:: image/interview-bobippolito.jpg @@ -103,7 +99,6 @@ ------------------------- 430mb: http://codespeak.net/download/pypy/video/introductory-talk-pycon-v1.avi.torrent - 166mb: http://codespeak.net/download/pypy/video/introductory-talk-pycon-320x240.avi.torrent .. image:: image/introductory-talk-pycon.jpg @@ -126,7 +121,6 @@ ----------------------------------------------------- 395mb: http://codespeak.net/download/pypy/video/agile-talk-v1.avi.torrent - 153mb: http://codespeak.net/download/pypy/video/agile-talk-320x240.avi.torrent .. image:: image/agile-talk.jpg @@ -149,7 +143,6 @@ ------------------------- 744mb: http://codespeak.net/download/pypy/video/architecture-session-v1.avi.torrent - 288mb: http://codespeak.net/download/pypy/video/architecture-session-320x240.avi.torrent .. image:: image/architecture-session.jpg @@ -172,7 +165,6 @@ --------------- 680mb: http://codespeak.net/download/pypy/video/sprint-tutorial-v2.avi.torrent - 263mb: http://codespeak.net/download/pypy/video/sprint-tutorial-320x240.avi.torrent .. image:: image/sprint-tutorial.jpg @@ -187,12 +179,45 @@ Michael Hudson gives an in-depth, very technical introduction to a PyPy sprint. The film provides a detailed and hands-on overview about the architecture of PyPy, especially the translation toolchain. +Scripting .NET with IronPython by Jim Hugunin +--------------------------------------------- + +372mb: http://codespeak.net/download/pypy/video/ironpython-talk-v2.avi.torrent +270mb: http://codespeak.net/download/pypy/video/ironpython-talk-320x240.avi.torrent + +.. image:: image/ironpython.jpg + :scale: 100 + :alt: Jim Hugunin on IronPython + :align: left + +Talk by Jim Hugunin (Microsoft) on the IronPython implementation on the .NET framework at this years PyCon, Dallas, US. + +PAL, 44 min, DivX AVI + +Jim Hugunin talks about regression tests, the code generation and the object layout, the new-style instance and gives a CLS interop demo. + + +Trailer: PyPy sprint at the University of Palma de Mallorca +----------------------------------------------------------- + +166mb: http://codespeak.net/download/pypy/video/mallorca-trailer-v1.avi.torrent +88mb: http://codespeak.net/download/pypy/video/mallorca-trailer-medium.avi.torrent +64mb: http://codespeak.net/download/pypy/video/mallorca-trailer-320x240.avi.torrent + +.. image:: image/mallorca-trailer.jpg + :scale: 100 + :alt: Trailer PyPy sprint in Mallorca + :align: left + +This trailer shows the PyPy team at the sprint in Mallorca, a behind-the-scenes of a typical PyPy coding sprint and talk as well as everything else. + +PAL, 11 min, DivX AVI + Coding discussion of core developers Armin Rigo and Samuele Pedroni ------------------------------------------------------------------- 620mb: http://codespeak.net/download/pypy/video/coding-discussion-v1.avi.torrent - 240mb: http://codespeak.net/download/pypy/video/coding-discussion-320x240.avi.torrent .. image:: image/coding-discussion.jpg @@ -209,7 +234,6 @@ ---------------------------------------------------------- 865mb: http://codespeak.net/download/pypy/video/introductory-student-talk-v2.avi.torrent - 437mb: http://codespeak.net/download/pypy/video/introductory-student-talk-320x240.avi.torrent .. image:: image/introductory-student-talk.jpg @@ -223,22 +247,3 @@ Core developers Armin Rigo, Samuele Pedroni and Carl Friedrich Bolz are giving an overview of the PyPy architecture, the standard interpreter, the translation toolchain and the just-in-time compiler. - -Scripting .NET with IronPython by Jim Hugunin ---------------------------------------------- - -372mb: http://codespeak.net/download/pypy/video/ironpython-talk-v2.avi.torrent - -270mb: http://codespeak.net/download/pypy/video/ironpython-talk-320x240.avi.torrent - -.. image:: image/ironpython.jpg - :scale: 100 - :alt: Jim Hugunin on IronPython - :align: left - -Talk by Jim Hugunin (Microsoft) on the IronPython implementation on the .NET framework at this years PyCon, Dallas, US. - -PAL, 44 min, DivX AVI - -Jim Hugunin talks about regression tests, the code generation and the object layout, the new-style instance and gives a CLS interop demo. - From wanja at codespeak.net Tue Aug 8 19:17:40 2006 From: wanja at codespeak.net (wanja at codespeak.net) Date: Tue, 8 Aug 2006 19:17:40 +0200 (CEST) Subject: [pypy-svn] r31180 - pypy/dist/pypy/doc Message-ID: <20060808171740.8770B1006F@code0.codespeak.net> Author: wanja Date: Tue Aug 8 19:17:39 2006 New Revision: 31180 Modified: pypy/dist/pypy/doc/video-index.txt Log: added missing empty lines Modified: pypy/dist/pypy/doc/video-index.txt ============================================================================== --- pypy/dist/pypy/doc/video-index.txt (original) +++ pypy/dist/pypy/doc/video-index.txt Tue Aug 8 19:17:39 2006 @@ -43,7 +43,9 @@ -------------------------- 130mb: http://codespeak.net/download/pypy/video/pycon-trailer.avi.torrent + 71mb: http://codespeak.net/download/pypy/video/pycon-trailer-medium.avi.torrent + 50mb: http://codespeak.net/download/pypy/video/pycon-trailer-320x240.avi.torrent .. image:: image/pycon-trailer.jpg @@ -61,6 +63,7 @@ ------------------------- 440mb: http://codespeak.net/download/pypy/video/interview-timpeters-v2.avi.torrent + 138mb: http://codespeak.net/download/pypy/video/interview-timpeters-320x240.avi.torrent .. image:: image/interview-timpeters.jpg @@ -80,6 +83,7 @@ --------------------------- 155mb: http://codespeak.net/download/pypy/video/interview-bobippolito-v2.avi.torrent + 50mb: http://codespeak.net/download/pypy/video/interview-bobippolito-320x240.avi.torrent .. image:: image/interview-bobippolito.jpg @@ -99,6 +103,7 @@ ------------------------- 430mb: http://codespeak.net/download/pypy/video/introductory-talk-pycon-v1.avi.torrent + 166mb: http://codespeak.net/download/pypy/video/introductory-talk-pycon-320x240.avi.torrent .. image:: image/introductory-talk-pycon.jpg @@ -121,6 +126,7 @@ ----------------------------------------------------- 395mb: http://codespeak.net/download/pypy/video/agile-talk-v1.avi.torrent + 153mb: http://codespeak.net/download/pypy/video/agile-talk-320x240.avi.torrent .. image:: image/agile-talk.jpg @@ -143,6 +149,7 @@ ------------------------- 744mb: http://codespeak.net/download/pypy/video/architecture-session-v1.avi.torrent + 288mb: http://codespeak.net/download/pypy/video/architecture-session-320x240.avi.torrent .. image:: image/architecture-session.jpg @@ -165,6 +172,7 @@ --------------- 680mb: http://codespeak.net/download/pypy/video/sprint-tutorial-v2.avi.torrent + 263mb: http://codespeak.net/download/pypy/video/sprint-tutorial-320x240.avi.torrent .. image:: image/sprint-tutorial.jpg @@ -182,7 +190,8 @@ Scripting .NET with IronPython by Jim Hugunin --------------------------------------------- -372mb: http://codespeak.net/download/pypy/video/ironpython-talk-v2.avi.torrent +372mb: http://codespeak.net/download/pypy/video/ironpython-talk-v2.avi.torrent + 270mb: http://codespeak.net/download/pypy/video/ironpython-talk-320x240.avi.torrent .. image:: image/ironpython.jpg @@ -201,7 +210,9 @@ ----------------------------------------------------------- 166mb: http://codespeak.net/download/pypy/video/mallorca-trailer-v1.avi.torrent + 88mb: http://codespeak.net/download/pypy/video/mallorca-trailer-medium.avi.torrent + 64mb: http://codespeak.net/download/pypy/video/mallorca-trailer-320x240.avi.torrent .. image:: image/mallorca-trailer.jpg @@ -218,6 +229,7 @@ ------------------------------------------------------------------- 620mb: http://codespeak.net/download/pypy/video/coding-discussion-v1.avi.torrent + 240mb: http://codespeak.net/download/pypy/video/coding-discussion-320x240.avi.torrent .. image:: image/coding-discussion.jpg @@ -234,6 +246,7 @@ ---------------------------------------------------------- 865mb: http://codespeak.net/download/pypy/video/introductory-student-talk-v2.avi.torrent + 437mb: http://codespeak.net/download/pypy/video/introductory-student-talk-320x240.avi.torrent .. image:: image/introductory-student-talk.jpg From benyoung at codespeak.net Wed Aug 9 11:58:25 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Wed, 9 Aug 2006 11:58:25 +0200 (CEST) Subject: [pypy-svn] r31189 - in pypy/dist/pypy: bin config interpreter objspace/std/test Message-ID: <20060809095825.A723810063@code0.codespeak.net> Author: benyoung Date: Wed Aug 9 11:57:37 2006 New Revision: 31189 Added: pypy/dist/pypy/interpreter/opcodeorder.py (contents, props changed) Modified: pypy/dist/pypy/bin/py.py pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/objspace/std/test/test_complexobject.py Log: Patch to allow logging of bytecode useage, and change the order of the dispatch switch based on the info gathered Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Wed Aug 9 11:57:37 2006 @@ -57,6 +57,7 @@ mod = __import__('pypy.objspace.%s' % conf.objspace.name, None, None, ['Space']) Space = mod.Space + #conf.objspace.logbytecodes = True space = Space(conf) return space @@ -127,6 +128,13 @@ space.finish() main.run_toplevel(space, doit, verbose=Options.verbose) + if space.config.objspace.logbytecodes: + import dis + counts = space.bytecodecounts.items() + counts.sort(key = (lambda x: (-x[1], x[0]))) + print [(dis.opname[opcode], count) for opcode, count in counts] + print [opcode for opcode, count in counts] + return exit_status ##def main_(argv=None): Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Wed Aug 9 11:57:37 2006 @@ -41,6 +41,9 @@ BoolOption("geninterp", "specify whether geninterp should be used"), + BoolOption("logbytecodes", + "keep track of bytecode usage", + default=False), OptionDescription("std", [ BoolOption("withsmallint", "use tagged integers", Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Wed Aug 9 11:57:37 2006 @@ -145,6 +145,10 @@ self.config = config self.interned_strings = {} self.setoptions(**kw) + + if self.config.objspace.logbytecodes: + self.bytecodecounts = {} + self.initialize() def setoptions(self): @@ -428,7 +432,7 @@ """ If w_obj is a wrapped internal interpreter class instance unwrap to it, otherwise return None. (Can be overridden in specific spaces; you - should generally use the helper space.interp_w() instead.) + should generally use the helper space.interp_w() instead.) """ if isinstance(w_obj, Wrappable): return w_obj @@ -436,19 +440,19 @@ def interp_w(self, RequiredClass, w_obj, can_be_None=False): """ - Unwrap w_obj, checking that it is an instance of the required internal - interpreter class (a subclass of Wrappable). - """ - if can_be_None and self.is_w(w_obj, self.w_None): - return None - obj = self.interpclass_w(w_obj) - if not isinstance(obj, RequiredClass): # or obj is None - msg = "'%s' object expected, got '%s' instead" % ( - RequiredClass.typedef.name, - w_obj.getclass(self).getname(self, '?')) - raise OperationError(self.w_TypeError, self.wrap(msg)) - return obj - interp_w._annspecialcase_ = 'specialize:arg(1)' + Unwrap w_obj, checking that it is an instance of the required internal + interpreter class (a subclass of Wrappable). + """ + if can_be_None and self.is_w(w_obj, self.w_None): + return None + obj = self.interpclass_w(w_obj) + if not isinstance(obj, RequiredClass): # or obj is None + msg = "'%s' object expected, got '%s' instead" % ( + RequiredClass.typedef.name, + w_obj.getclass(self).getname(self, '?')) + raise OperationError(self.w_TypeError, self.wrap(msg)) + return obj + interp_w._annspecialcase_ = 'specialize:arg(1)' def unpackiterable(self, w_iterable, expected_length=-1): """Unpack an iterable object into a real (interpreter-level) list. Added: pypy/dist/pypy/interpreter/opcodeorder.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/opcodeorder.py Wed Aug 9 11:57:37 2006 @@ -0,0 +1 @@ +opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] \ No newline at end of file Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Wed Aug 9 11:57:37 2006 @@ -11,6 +11,7 @@ from pypy.interpreter.miscutils import InitializedClass from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack from pypy.interpreter.pycode import PyCode +from pypy.interpreter.opcodeorder import opcodeorder from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.objectmodel import we_are_translated from pypy.rpython.rarithmetic import intmask @@ -57,6 +58,8 @@ ec.bytecode_trace(self) self.next_instr = self.last_instr opcode = self.nextop() + if self.space.config.objspace.logbytecodes: + self.space.bytecodecounts[opcode] = self.space.bytecodecounts.get(opcode, 0) + 1 if opcode >= pythonopcode.HAVE_ARGUMENT: oparg = self.nextarg() while True: @@ -813,6 +816,8 @@ ec.bytecode_trace(self) self.next_instr = self.last_instr opcode = ord(code[self.next_instr]) + if self.space.config.objspace.logbytecodes: + self.space.bytecodecounts[opcode] = self.space.bytecodecounts.get(opcode, 0) + 1 self.next_instr += 1 if opcode >= %s: oparg = ord(code[self.next_instr]) | ord(code[self.next_instr + 1]) << 8 @@ -828,8 +833,16 @@ ''' % (pythonopcode.HAVE_ARGUMENT, pythonopcode.EXTENDED_ARG, pythonopcode.HAVE_ARGUMENT) + + def sortkey(opcode, opcodeorder=opcodeorder, ValueError=ValueError): + try: + index = opcodeorder.index(opcode) + except ValueError: + index = 1000000 + return index, opcode opcases = [(i, opname) for opname, i in pythonopcode.opmap.iteritems()] - opcases.sort() # for predictable results + opcases.sort(key = sortkey) # for predictable results + for i, opname in opcases: if i == pythonopcode.EXTENDED_ARG or i < pythonopcode.HAVE_ARGUMENT: continue Modified: pypy/dist/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_complexobject.py Wed Aug 9 11:57:37 2006 @@ -60,7 +60,7 @@ class AppTestAppComplexTest: def setup_class(cls): - cls.w_helper = cls.space.appexec([], "():\n import sys\n sys.path.append('%s')\n import helper\n return helper" % (py.magic.autopath().dirpath(), )) + cls.w_helper = cls.space.appexec([], "():\n import sys\n sys.path.append('%s')\n import helper\n return helper" % (str(py.magic.autopath().dirpath()).replace('\\', '\\\\'), )) def test_div(self): h = self.helper From arigo at codespeak.net Wed Aug 9 12:16:24 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 12:16:24 +0200 (CEST) Subject: [pypy-svn] r31190 - in pypy/dist/pypy: interpreter interpreter/pyparser interpreter/pyparser/test interpreter/test jit/codegen/i386 jit/timeshifter lib/pyontology/test module/__builtin__ module/_socket module/bz2 module/recparser module/recparser/test objspace/cpy rpython/lltypesystem rpython/numpy rpython/ootypesystem tool/pytest translator/backendopt translator/c translator/cli translator/goal translator/js/demo/jsdemo translator/microbench/pybench Message-ID: <20060809101624.2E3A010068@code0.codespeak.net> Author: arigo Date: Wed Aug 9 12:16:17 2006 New Revision: 31190 Modified: pypy/dist/pypy/interpreter/pycompiler.py pypy/dist/pypy/interpreter/pyparser/astbuilder.py pypy/dist/pypy/interpreter/pyparser/ebnflexer.py pypy/dist/pypy/interpreter/pyparser/ebnfparse.py pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py pypy/dist/pypy/interpreter/test/test_objspace.py pypy/dist/pypy/jit/codegen/i386/autotest.py pypy/dist/pypy/jit/timeshifter/rtyper.py pypy/dist/pypy/lib/pyontology/test/test_ontology.py pypy/dist/pypy/module/__builtin__/importing.py pypy/dist/pypy/module/_socket/interp_socket.py pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/recparser/pyparser.py pypy/dist/pypy/module/recparser/test/test_compilehooks.py pypy/dist/pypy/objspace/cpy/objspace.py pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/numpy/aarray.py pypy/dist/pypy/rpython/ootypesystem/rclass.py pypy/dist/pypy/tool/pytest/genreportdata.py pypy/dist/pypy/translator/backendopt/merge_if_blocks.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/cli/database.py pypy/dist/pypy/translator/goal/gcbench.py pypy/dist/pypy/translator/js/demo/jsdemo/servermessage.py pypy/dist/pypy/translator/microbench/pybench/pybench.py Log: Massive untabify. Modified: pypy/dist/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/dist/pypy/interpreter/pycompiler.py (original) +++ pypy/dist/pypy/interpreter/pycompiler.py Wed Aug 9 12:16:17 2006 @@ -250,5 +250,5 @@ def install_compiler_hook(space, w_callable): # if not space.get( w_callable ): -# raise OperationError( space.w_TypeError( space.wrap( "must have a callable" ) ) +# raise OperationError( space.w_TypeError( space.wrap( "must have a callable" ) ) space.default_compiler.w_compile_hook = w_callable Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Wed Aug 9 12:16:17 2006 @@ -1303,8 +1303,8 @@ names.append((name, as_name)) # move forward until next ',' # XXX: what is it supposed to do ? - while index=0 + _endpos = npos - 1 + assert _endpos>=0 return Token(TOK_STRING,inp[pos+1:_endpos]) else: npos = match_symbol( inp, pos, end) Modified: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/ebnfparse.py (original) +++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py Wed Aug 9 12:16:17 2006 @@ -70,17 +70,17 @@ def ebnf_handle_grammar(self, node): for rule in node.nodes: - rule.visit(self) + rule.visit(self) # the rules are registered already # we do a pass through the variables to detect # terminal symbols from non terminals for r in self.items: - for i,a in enumerate(r.args): - if a.codename in self.rules: - assert isinstance(a,Token) - r.args[i] = self.rules[a.codename] - if a.codename in self.terminals: - del self.terminals[a.codename] + for i,a in enumerate(r.args): + if a.codename in self.rules: + assert isinstance(a,Token) + r.args[i] = self.rules[a.codename] + if a.codename in self.terminals: + del self.terminals[a.codename] # XXX .keywords also contains punctuations self.terminals['NAME'].keywords = self.keywords @@ -91,14 +91,14 @@ alt = node.nodes[1] rule = alt.visit(self) if not isinstance(rule, Token): - rule.codename = self.symbols.add_symbol( symdef ) + rule.codename = self.symbols.add_symbol( symdef ) self.rules[rule.codename] = rule def ebnf_handle_alternative(self, node): items = [node.nodes[0].visit(self)] items += node.nodes[1].visit(self) if len(items) == 1 and not items[0].is_root(): - return items[0] + return items[0] alt = Alternative(self.new_symbol(), items) return self.new_item(alt) @@ -106,11 +106,11 @@ """ """ items = [] for n in node.nodes: - items.append( n.visit(self) ) + items.append( n.visit(self) ) if len(items)==1: - return items[0] + return items[0] elif len(items)>1: - return self.new_item( Sequence( self.new_symbol(), items) ) + return self.new_item( Sequence( self.new_symbol(), items) ) raise RuntimeError("Found empty sequence") def ebnf_handle_sequence_cont( self, node ): @@ -126,13 +126,13 @@ sym = node.nodes[0].value terminal = self.terminals.get( sym, None ) if not terminal: - tokencode = pytoken.tok_values.get( sym, None ) - if tokencode is None: - tokencode = self.symbols.add_symbol( sym ) - terminal = Token( tokencode ) - else: - terminal = Token( tokencode ) - self.terminals[sym] = terminal + tokencode = pytoken.tok_values.get( sym, None ) + if tokencode is None: + tokencode = self.symbols.add_symbol( sym ) + terminal = Token( tokencode ) + else: + terminal = Token( tokencode ) + self.terminals[sym] = terminal return self.repeat( star_opt, terminal ) @@ -148,15 +148,15 @@ value = node.value tokencode = pytoken.tok_punct.get( value, None ) if tokencode is None: - if not py_name.match( value ): - raise RuntimeError("Unknown STRING value ('%s')" % value ) - # assume a keyword - tok = Token( pytoken.NAME, value ) - if value not in self.keywords: - self.keywords.append( value ) + if not py_name.match( value ): + raise RuntimeError("Unknown STRING value ('%s')" % value ) + # assume a keyword + tok = Token( pytoken.NAME, value ) + if value not in self.keywords: + self.keywords.append( value ) else: - # punctuation - tok = Token( tokencode ) + # punctuation + tok = Token( tokencode ) return tok def ebnf_handle_sequence_alt( self, node ): @@ -169,9 +169,9 @@ ebnf_handles = {} for name, value in globals().items(): if name.startswith("ebnf_handle_"): - name = name[12:] - key = getattr(ebnfgrammar, name ) - ebnf_handles[key] = value + name = name[12:] + key = getattr(ebnfgrammar, name ) + ebnf_handles[key] = value def handle_unknown( self, node ): raise RuntimeError("Unknown Visitor for %r" % node.name) @@ -200,8 +200,8 @@ return itm def visit_syntaxnode( self, node ): - visit_func = ebnf_handles.get( node.name, handle_unknown ) - return visit_func( self, node ) + visit_func = ebnf_handles.get( node.name, handle_unknown ) + return visit_func( self, node ) def visit_tokennode( self, node ): return self.visit_syntaxnode( node ) Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py Wed Aug 9 12:16:17 2006 @@ -28,8 +28,8 @@ print "Type mismatch", repr(l), repr(r) print "l is str", repr(l), type(l)==str print "r is AssName", repr(r), isinstance(r,ast_ast.AssName) - print "left is", repr(left) - print "right is", repr(right) + print "left is", repr(left) + print "right is", repr(right) return False return True Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Wed Aug 9 12:16:17 2006 @@ -134,14 +134,14 @@ def test_interp_w(self): w = self.space.wrap - w_bltinfunction = self.space.builtin.get('len') - res = self.space.interp_w(Function, w_bltinfunction) - assert res is w_bltinfunction # with the std objspace only - self.space.raises_w(self.space.w_TypeError, self.space.interp_w, PyCode, w_bltinfunction) - self.space.raises_w(self.space.w_TypeError, self.space.interp_w, Function, w(42)) - self.space.raises_w(self.space.w_TypeError, self.space.interp_w, Function, w(None)) - res = self.space.interp_w(Function, w(None), can_be_None=True) - assert res is None + w_bltinfunction = self.space.builtin.get('len') + res = self.space.interp_w(Function, w_bltinfunction) + assert res is w_bltinfunction # with the std objspace only + self.space.raises_w(self.space.w_TypeError, self.space.interp_w, PyCode, w_bltinfunction) + self.space.raises_w(self.space.w_TypeError, self.space.interp_w, Function, w(42)) + self.space.raises_w(self.space.w_TypeError, self.space.interp_w, Function, w(None)) + res = self.space.interp_w(Function, w(None), can_be_None=True) + assert res is None class TestModuleMinimal: def test_sys_exists(self): Modified: pypy/dist/pypy/jit/codegen/i386/autotest.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/autotest.py (original) +++ pypy/dist/pypy/jit/codegen/i386/autotest.py Wed Aug 9 12:16:17 2006 @@ -304,9 +304,9 @@ items.sort() for key, value in items: if isinstance(value, i386.Instruction): - if key in FORBIDDEN: - print "Skipped", key - else: + if key in FORBIDDEN: + print "Skipped", key + else: complete_test(key,value) print "Ok." Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtyper.py Wed Aug 9 12:16:17 2006 @@ -35,7 +35,7 @@ class HintRTyper(RPythonTyper): def __init__(self, hannotator, timeshifter): - RPythonTyper.__init__(self, hannotator, + RPythonTyper.__init__(self, hannotator, type_system=HintTypeSystem.instance) self.green_reprs = PRECOMPUTED_GREEN_REPRS.copy() self.red_reprs = {} Modified: pypy/dist/pypy/lib/pyontology/test/test_ontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_ontology.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_ontology.py Wed Aug 9 12:16:17 2006 @@ -646,33 +646,33 @@ a_cls = URIRef('a') O.type(a_cls, URIRef(namespaces['owl']+'#Class')) - assert isinstance(O.variables['a_'], ClassDomain) + assert isinstance(O.variables['a_'], ClassDomain) O.type(a_cls, URIRef(namespaces['owl']+'#Restriction')) - assert isinstance(O.variables['a_'], Restriction) + assert isinstance(O.variables['a_'], Restriction) def test_class_demotion(): O = Ontology() a_cls = URIRef('a') O.type(a_cls, URIRef(namespaces['owl']+'#Restriction')) O.variables[O.make_var(None, a_cls)].property = "SomeProp" - assert isinstance(O.variables['a_'], Restriction) + assert isinstance(O.variables['a_'], Restriction) O.type(a_cls, URIRef(namespaces['owl']+'#Class')) - assert isinstance(O.variables['a_'], Restriction) + assert isinstance(O.variables['a_'], Restriction) assert O.variables[O.make_var(None, a_cls)].property == "SomeProp" def test_property_to_objectproperty(): O = Ontology() a_cls = URIRef('a') O.type(a_cls, URIRef(namespaces['rdf']+'#Property')) - assert isinstance(O.variables['a_'], Property) + assert isinstance(O.variables['a_'], Property) O.type(a_cls, URIRef(namespaces['owl']+'#ObjectProperty')) - assert isinstance(O.variables['a_'], Property) + assert isinstance(O.variables['a_'], Property) O.type(a_cls, URIRef(namespaces['rdf']+'#Property')) - assert isinstance(O.variables['a_'], ObjectProperty) + assert isinstance(O.variables['a_'], ObjectProperty) def test_individual(): # test comparison (unknown, equal, different) Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Wed Aug 9 12:16:17 2006 @@ -544,9 +544,9 @@ import sys meta_path = sys.meta_path for hook in meta_path: - loader = hook.find_module(fullname, path) - if loader: - return loader + loader = hook.find_module(fullname, path) + if loader: + return loader if path != None and type(path) == str: pass # XXX Check for frozen modules ? Modified: pypy/dist/pypy/module/_socket/interp_socket.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_socket.py (original) +++ pypy/dist/pypy/module/_socket/interp_socket.py Wed Aug 9 12:16:17 2006 @@ -133,7 +133,7 @@ addr = res.contents.ai_addr _c.freeaddrinfo(res) return addr - + def w_makesockaddr(space, caddr, caddrlen, proto): if caddrlen == 0: # No address -- may be recvfrom() from known socket Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Wed Aug 9 12:16:17 2006 @@ -1007,7 +1007,7 @@ in_bufsize = len(data) # conforming to bz2 manual, this is large enough to fit compressed - # data in one shot. We will check it later anyway. + # data in one shot. We will check it later anyway. out_bufsize = in_bufsize + (in_bufsize / 100 + 1) + 600 out_buf = create_string_buffer(out_bufsize) Modified: pypy/dist/pypy/module/recparser/pyparser.py ============================================================================== --- pypy/dist/pypy/module/recparser/pyparser.py (original) +++ pypy/dist/pypy/module/recparser/pyparser.py Wed Aug 9 12:16:17 2006 @@ -236,11 +236,11 @@ GrammarElement.descr_grammarelement_get_children = descr_grammarelement_get_children GrammarElement.typedef = TypeDef( "GrammarElement", - #__repr__ = interp2app(GrammarElement.descr_grammarelement_repr, - # unwrap_spec=['self', ObjSpace] ), - get_children = interp2app(GrammarElement.descr_grammarelement_get_children, - unwrap_spec=['self', ObjSpace] ), - ) + #__repr__ = interp2app(GrammarElement.descr_grammarelement_repr, + # unwrap_spec=['self', ObjSpace] ), + get_children = interp2app(GrammarElement.descr_grammarelement_get_children, + unwrap_spec=['self', ObjSpace] ), + ) @@ -262,7 +262,7 @@ def descr_alternative_insert(self, space, idx, w_rule ): rule = space.interp_w(GrammarElement, w_rule) if idx<0 or idx>len(self.args): - raise OperationError( space.w_IndexError, space.wrap("Invalid index") ) + raise OperationError( space.w_IndexError, space.wrap("Invalid index") ) self.args.insert( idx, rule ) Alternative.descr_alternative_append = descr_alternative_append @@ -273,17 +273,17 @@ Alternative.typedef = TypeDef("Alternative", GrammarElement.typedef, - __getitem__ = interp2app( Alternative.descr_alternative___getitem__, - unwrap_spec=['self',ObjSpace,int]), - __setitem__ = interp2app( Alternative.descr_alternative___setitem__, - unwrap_spec=['self',ObjSpace,int,W_Root]), - __delitem__ = interp2app( Alternative.descr_alternative___delitem__, - unwrap_spec=['self',ObjSpace,int]), - insert = interp2app( Alternative.descr_alternative_insert, - unwrap_spec = ['self', ObjSpace, int, W_Root ] ), - append = interp2app( Alternative.descr_alternative_append, - unwrap_spec = ['self', ObjSpace, W_Root ] ), - ) + __getitem__ = interp2app( Alternative.descr_alternative___getitem__, + unwrap_spec=['self',ObjSpace,int]), + __setitem__ = interp2app( Alternative.descr_alternative___setitem__, + unwrap_spec=['self',ObjSpace,int,W_Root]), + __delitem__ = interp2app( Alternative.descr_alternative___delitem__, + unwrap_spec=['self',ObjSpace,int]), + insert = interp2app( Alternative.descr_alternative_insert, + unwrap_spec = ['self', ObjSpace, int, W_Root ] ), + append = interp2app( Alternative.descr_alternative_append, + unwrap_spec = ['self', ObjSpace, W_Root ] ), + ) Sequence.descr_alternative_append = descr_alternative_append Sequence.descr_alternative_insert = descr_alternative_insert @@ -293,26 +293,26 @@ Sequence.typedef = TypeDef("Sequence", GrammarElement.typedef, - __getitem__ = interp2app( Sequence.descr_alternative___getitem__, - unwrap_spec=['self',ObjSpace,int]), - __setitem__ = interp2app( Sequence.descr_alternative___setitem__, - unwrap_spec=['self',ObjSpace,int,W_Root]), - __delitem__ = interp2app( Sequence.descr_alternative___delitem__, - unwrap_spec=['self',ObjSpace,int]), - insert = interp2app( Sequence.descr_alternative_insert, - unwrap_spec = ['self', ObjSpace, int, W_Root ] ), - append = interp2app( Sequence.descr_alternative_append, - unwrap_spec = ['self', ObjSpace, W_Root ] ), - ) + __getitem__ = interp2app( Sequence.descr_alternative___getitem__, + unwrap_spec=['self',ObjSpace,int]), + __setitem__ = interp2app( Sequence.descr_alternative___setitem__, + unwrap_spec=['self',ObjSpace,int,W_Root]), + __delitem__ = interp2app( Sequence.descr_alternative___delitem__, + unwrap_spec=['self',ObjSpace,int]), + insert = interp2app( Sequence.descr_alternative_insert, + unwrap_spec = ['self', ObjSpace, int, W_Root ] ), + append = interp2app( Sequence.descr_alternative_append, + unwrap_spec = ['self', ObjSpace, W_Root ] ), + ) def descr_kleenestar___getitem__(self, space, idx ): if idx!=0: - raise OperationError( space.w_ValueError, space.wrap("KleeneStar only support one child")) + raise OperationError( space.w_ValueError, space.wrap("KleeneStar only support one child")) return space.wrap(self.args[idx]) def descr_kleenestar___setitem__(self, space, idx, w_rule ): if idx!=0: - raise OperationError( space.w_ValueError, space.wrap("KleeneStar only support one child")) + raise OperationError( space.w_ValueError, space.wrap("KleeneStar only support one child")) rule = space.interp_w(GrammarElement, w_rule) self.args[idx] = rule @@ -320,10 +320,10 @@ KleeneStar.descr_kleenestar___setitem__ = descr_kleenestar___setitem__ KleeneStar.typedef = TypeDef("KleeneStar", GrammarElement.typedef, - __getitem__ = interp2app(KleeneStar.descr_kleenestar___getitem__, - unwrap_spec=[ 'self', ObjSpace, int]), - __setitem__ = interp2app(KleeneStar.descr_kleenestar___setitem__, - unwrap_spec=[ 'self', ObjSpace, int, W_Root ]), - ) + __getitem__ = interp2app(KleeneStar.descr_kleenestar___getitem__, + unwrap_spec=[ 'self', ObjSpace, int]), + __setitem__ = interp2app(KleeneStar.descr_kleenestar___setitem__, + unwrap_spec=[ 'self', ObjSpace, int, W_Root ]), + ) Token.typedef = TypeDef("Token", GrammarElement.typedef ) Modified: pypy/dist/pypy/module/recparser/test/test_compilehooks.py ============================================================================== --- pypy/dist/pypy/module/recparser/test/test_compilehooks.py (original) +++ pypy/dist/pypy/module/recparser/test/test_compilehooks.py Wed Aug 9 12:16:17 2006 @@ -49,7 +49,7 @@ def change_globals(ast, enc): class ChangeGlobalsVisitor: def visitConst(self, node): - pass + pass def defaultvisit(self, node): for child in node.getChildNodes(): @@ -66,7 +66,7 @@ # install the hook import parser parser.install_compiler_hook(change_globals) - # check that the visitor changed all globals - # in the code into Consts - # TODO - # simplest version of the test : dis(code) | grep -v LOAD_GLOBAL == dis(code) + # check that the visitor changed all globals + # in the code into Consts + # TODO + # simplest version of the test : dis(code) | grep -v LOAD_GLOBAL == dis(code) Modified: pypy/dist/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/objspace.py Wed Aug 9 12:16:17 2006 @@ -96,11 +96,11 @@ def interp_w(self, RequiredClass, w_obj, can_be_None=False): """ - Unwrap w_obj, checking that it is an instance of the required internal - interpreter class (a subclass of Wrappable). - """ - if can_be_None and self.is_w(w_obj, self.w_None): - return None + Unwrap w_obj, checking that it is an instance of the required internal + interpreter class (a subclass of Wrappable). + """ + if can_be_None and self.is_w(w_obj, self.w_None): + return None from pypy.objspace.cpy.typedef import cpython2rpython return cpython2rpython(self, RequiredClass, w_obj) interp_w._annspecialcase_ = 'specialize:arg(1)' Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Wed Aug 9 12:16:17 2006 @@ -526,7 +526,7 @@ return vptr def rtype_type(self, hop): - if hop.s_result.is_constant(): + if hop.s_result.is_constant(): return hop.inputconst(hop.r_result, hop.s_result.const) instance_repr = self.common_repr() vinst, = hop.inputargs(instance_repr) Modified: pypy/dist/pypy/rpython/numpy/aarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/aarray.py (original) +++ pypy/dist/pypy/rpython/numpy/aarray.py Wed Aug 9 12:16:17 2006 @@ -27,8 +27,8 @@ } def __init__(self, knowntype, typecode): self.knowntype = knowntype - self.typecode = typecode - self.rank = 1 + self.typecode = typecode + self.rank = 1 def can_be_none(self): return True @@ -83,7 +83,7 @@ _about_ = numpy.array def compute_result_annotation(self, arg_list, *args_s, **kwds_s): - if not isinstance(arg_list, SomeList): + if not isinstance(arg_list, SomeList): raise AnnotatorError("numpy.array expects SomeList") # First guess type from input list @@ -93,7 +93,7 @@ # now see if the dtype arg over-rides the typecode dtype = None - if len(args_s)>0: + if len(args_s)>0: dtype = args_s[0] if "dtype" in kwds_s: dtype = kwds_s["dtype"] Modified: pypy/dist/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rclass.py Wed Aug 9 12:16:17 2006 @@ -407,7 +407,7 @@ return ootype.oostring(instance, self.ll_const(-1)) def rtype_type(self, hop): - if hop.s_result.is_constant(): + if hop.s_result.is_constant(): return hop.inputconst(hop.r_result, hop.s_result.const) vinst, = hop.inputargs(self) if hop.args_s[0].can_be_none(): Modified: pypy/dist/pypy/tool/pytest/genreportdata.py ============================================================================== --- pypy/dist/pypy/tool/pytest/genreportdata.py (original) +++ pypy/dist/pypy/tool/pytest/genreportdata.py Wed Aug 9 12:16:17 2006 @@ -3,8 +3,8 @@ import py import sys if sys.version_info[:2] != (2,3): - raise RuntimeError("Genreportdata.py needs Python 2.3") - + raise RuntimeError("Genreportdata.py needs Python 2.3") + mydir = py.magic.autopath().dirpath().realpath() from pypy.tool.pytest import htmlreport from pypy.tool.pytest import confpath Modified: pypy/dist/pypy/translator/backendopt/merge_if_blocks.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/merge_if_blocks.py (original) +++ pypy/dist/pypy/translator/backendopt/merge_if_blocks.py Wed Aug 9 12:16:17 2006 @@ -35,9 +35,9 @@ default.prevblock = firstblock default.args = [get_new_arg(arg) for arg in default.args] for block, case in chain: - link = block.exits[1] + link = block.exits[1] links.append(link) - link.exitcase = case + link.exitcase = case link.llexitcase = case.value link.prevblock = firstblock link.args = [get_new_arg(arg) for arg in link.args] Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Wed Aug 9 12:16:17 2006 @@ -106,7 +106,7 @@ def OP_GC_CALL_RTTI_DESTRUCTOR(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] line = '%s(%s);' % (args[0], ', '.join(args[1:])) - return line + return line def OP_GC_FREE(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] Modified: pypy/dist/pypy/translator/cli/database.py ============================================================================== --- pypy/dist/pypy/translator/cli/database.py (original) +++ pypy/dist/pypy/translator/cli/database.py Wed Aug 9 12:16:17 2006 @@ -40,7 +40,7 @@ DEFINED_INT_SYMBOLICS = {'MALLOC_ZERO_FILLED':1} def isnan(v): - return v != v*1.0 or (v == 1.0 and v == 2.0) + return v != v*1.0 or (v == 1.0 and v == 2.0) def isinf(v): return v!=0 and (v == v*2) Modified: pypy/dist/pypy/translator/goal/gcbench.py ============================================================================== --- pypy/dist/pypy/translator/goal/gcbench.py (original) +++ pypy/dist/pypy/translator/goal/gcbench.py Wed Aug 9 12:16:17 2006 @@ -3,46 +3,46 @@ # of Post Communications. # It was modified by Hans Boehm of Silicon Graphics. # -# This is no substitute for real applications. No actual application -# is likely to behave in exactly this way. However, this benchmark was -# designed to be more representative of real applications than other -# Java GC benchmarks of which we are aware. -# It attempts to model those properties of allocation requests that -# are important to current GC techniques. -# It is designed to be used either to obtain a single overall performance -# number, or to give a more detailed estimate of how collector -# performance varies with object lifetimes. It prints the time -# required to allocate and collect balanced binary trees of various -# sizes. Smaller trees result in shorter object lifetimes. Each cycle -# allocates roughly the same amount of memory. -# Two data structures are kept around during the entire process, so -# that the measured performance is representative of applications -# that maintain some live in-memory data. One of these is a tree -# containing many pointers. The other is a large array containing -# double precision floating point numbers. Both should be of comparable -# size. +# This is no substitute for real applications. No actual application +# is likely to behave in exactly this way. However, this benchmark was +# designed to be more representative of real applications than other +# Java GC benchmarks of which we are aware. +# It attempts to model those properties of allocation requests that +# are important to current GC techniques. +# It is designed to be used either to obtain a single overall performance +# number, or to give a more detailed estimate of how collector +# performance varies with object lifetimes. It prints the time +# required to allocate and collect balanced binary trees of various +# sizes. Smaller trees result in shorter object lifetimes. Each cycle +# allocates roughly the same amount of memory. +# Two data structures are kept around during the entire process, so +# that the measured performance is representative of applications +# that maintain some live in-memory data. One of these is a tree +# containing many pointers. The other is a large array containing +# double precision floating point numbers. Both should be of comparable +# size. # -# The results are only really meaningful together with a specification -# of how much memory was used. It is possible to trade memory for -# better time performance. This benchmark should be run in a 32 MB -# heap, though we don't currently know how to enforce that uniformly. +# The results are only really meaningful together with a specification +# of how much memory was used. It is possible to trade memory for +# better time performance. This benchmark should be run in a 32 MB +# heap, though we don't currently know how to enforce that uniformly. # -# Unlike the original Ellis and Kovac benchmark, we do not attempt -# measure pause times. This facility should eventually be added back -# in. There are several reasons for omitting it for now. The original -# implementation depended on assumptions about the thread scheduler -# that don't hold uniformly. The results really measure both the -# scheduler and GC. Pause time measurements tend to not fit well with -# current benchmark suites. As far as we know, none of the current -# commercial Java implementations seriously attempt to minimize GC pause -# times. +# Unlike the original Ellis and Kovac benchmark, we do not attempt +# measure pause times. This facility should eventually be added back +# in. There are several reasons for omitting it for now. The original +# implementation depended on assumptions about the thread scheduler +# that don't hold uniformly. The results really measure both the +# scheduler and GC. Pause time measurements tend to not fit well with +# current benchmark suites. As far as we know, none of the current +# commercial Java implementations seriously attempt to minimize GC pause +# times. # -# Known deficiencies: -# - No way to check on memory use -# - No cyclic data structures -# - No attempt to measure variation with object size -# - Results are sensitive to locking cost, but we dont -# check for proper locking +# Known deficiencies: +# - No way to check on memory use +# - No cyclic data structures +# - No attempt to measure variation with object size +# - Results are sensitive to locking cost, but we dont +# check for proper locking import os, time def println(s): Modified: pypy/dist/pypy/translator/js/demo/jsdemo/servermessage.py ============================================================================== --- pypy/dist/pypy/translator/js/demo/jsdemo/servermessage.py (original) +++ pypy/dist/pypy/translator/js/demo/jsdemo/servermessage.py Wed Aug 9 12:16:17 2006 @@ -21,8 +21,8 @@ #proxy messages -#PMSG_PING = "ping" #server wants to hear from client -#PMSG_PONG = "pong" #server responds to client's ping +#PMSG_PING = "ping" #server wants to hear from client +#PMSG_PONG = "pong" #server responds to client's ping PMSG_DEF_PLAYFIELD = "def_playfield" PMSG_DEF_ICON = "def_icon" PMSG_PLAYER_ICON = "player_icon" Modified: pypy/dist/pypy/translator/microbench/pybench/pybench.py ============================================================================== --- pypy/dist/pypy/translator/microbench/pybench/pybench.py (original) +++ pypy/dist/pypy/translator/microbench/pybench/pybench.py Wed Aug 9 12:16:17 2006 @@ -409,7 +409,7 @@ self.head.append( html.link(rel="Stylesheet", type="text/css", href="MochiKit-1.1/examples/sortable_tables/sortable_tables.css")) self.head.append( - html.script(rel="JavaScript", type="text/javascript", src="MochiKit-1.1/lib/MochiKit/MochiKit.js")) + html.script(rel="JavaScript", type="text/javascript", src="MochiKit-1.1/lib/MochiKit/MochiKit.js")) self.head.append( html.script(rel="JavaScript", type="text/javascript", src="MochiKit-1.1/examples/sortable_tables/sortable_tables.js")) From arigo at codespeak.net Wed Aug 9 12:20:56 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 12:20:56 +0200 (CEST) Subject: [pypy-svn] r31191 - in pypy/dist/pypy: config config/test doc doc/js interpreter interpreter/astcompiler/test module/_md5 module/_random module/_ssl module/_stackless module/bz2 module/bz2/test module/fcntl module/fcntl/test module/mmap module/mmap/test module/rctime module/wraptest objspace/cclp objspace/std objspace/std/benchmark objspace/std/test rpython/numpy rpython/numpy/test tool/build tool/build/test translator/js/doc Message-ID: <20060809102056.3EB4010068@code0.codespeak.net> Author: arigo Date: Wed Aug 9 12:20:46 2006 New Revision: 31191 Modified: pypy/dist/pypy/config/pypyoption.py (props changed) pypy/dist/pypy/config/test/test_pypyoption.py (props changed) pypy/dist/pypy/doc/js/ (props changed) pypy/dist/pypy/doc/summer-of-pypy.txt (contents, props changed) pypy/dist/pypy/interpreter/astcompiler/test/ (props changed) pypy/dist/pypy/interpreter/opcodeorder.py (props changed) pypy/dist/pypy/module/_md5/ (props changed) pypy/dist/pypy/module/_md5/__init__.py (contents, props changed) pypy/dist/pypy/module/_md5/interp_md5.py (contents, props changed) pypy/dist/pypy/module/_random/ (props changed) pypy/dist/pypy/module/_random/__init__.py (contents, props changed) pypy/dist/pypy/module/_random/interp_random.py (contents, props changed) pypy/dist/pypy/module/_ssl/bio.py (props changed) pypy/dist/pypy/module/_ssl/ssl.py (props changed) pypy/dist/pypy/module/_stackless/clonable.py (props changed) pypy/dist/pypy/module/bz2/ (props changed) pypy/dist/pypy/module/bz2/__init__.py (props changed) pypy/dist/pypy/module/bz2/app_bz2.py (props changed) pypy/dist/pypy/module/bz2/bzlib.py (props changed) pypy/dist/pypy/module/bz2/fileobject.py (props changed) pypy/dist/pypy/module/bz2/interp_bz2.py (props changed) pypy/dist/pypy/module/bz2/test/ (props changed) pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py (props changed) pypy/dist/pypy/module/bz2/test/test_bz2_file.py (props changed) pypy/dist/pypy/module/fcntl/ (props changed) pypy/dist/pypy/module/fcntl/__init__.py (props changed) pypy/dist/pypy/module/fcntl/app_fcntl.py (props changed) pypy/dist/pypy/module/fcntl/interp_fcntl.py (props changed) pypy/dist/pypy/module/fcntl/test/ (props changed) pypy/dist/pypy/module/fcntl/test/test_fcntl.py (props changed) pypy/dist/pypy/module/mmap/ (props changed) pypy/dist/pypy/module/mmap/__init__.py (props changed) pypy/dist/pypy/module/mmap/app_mmap.py (props changed) pypy/dist/pypy/module/mmap/interp_mmap.py (props changed) pypy/dist/pypy/module/mmap/test/ (props changed) pypy/dist/pypy/module/mmap/test/test_mmap.py (props changed) pypy/dist/pypy/module/rctime/strptime.txt (props changed) pypy/dist/pypy/module/wraptest/ (props changed) pypy/dist/pypy/objspace/cclp/ (props changed) pypy/dist/pypy/objspace/cclp/__init__.py (props changed) pypy/dist/pypy/objspace/cclp/global_state.py (props changed) pypy/dist/pypy/objspace/cclp/misc.py (props changed) pypy/dist/pypy/objspace/cclp/plan.txt (props changed) pypy/dist/pypy/objspace/cclp/scheduler.py (props changed) pypy/dist/pypy/objspace/cclp/space.py (props changed) pypy/dist/pypy/objspace/cclp/thread.py (props changed) pypy/dist/pypy/objspace/cclp/thunk.py (props changed) pypy/dist/pypy/objspace/cclp/types.py (props changed) pypy/dist/pypy/objspace/cclp/variable.py (props changed) pypy/dist/pypy/objspace/std/benchmark/ (props changed) pypy/dist/pypy/objspace/std/dictstrobject.py (props changed) pypy/dist/pypy/objspace/std/strjoinobject.py (props changed) pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (props changed) pypy/dist/pypy/objspace/std/test/test_dictstrobject.py (props changed) pypy/dist/pypy/objspace/std/test/test_prebuiltint.py (props changed) pypy/dist/pypy/objspace/std/test/test_strjoinobject.py (props changed) pypy/dist/pypy/rpython/numpy/__init__.py (props changed) pypy/dist/pypy/rpython/numpy/aarray.py (props changed) pypy/dist/pypy/rpython/numpy/implementation.py (props changed) pypy/dist/pypy/rpython/numpy/rarray.py (props changed) pypy/dist/pypy/rpython/numpy/test/conftest.py (props changed) pypy/dist/pypy/rpython/numpy/test/test_array.py (props changed) pypy/dist/pypy/tool/build/README.txt (props changed) pypy/dist/pypy/tool/build/config.py (props changed) pypy/dist/pypy/tool/build/conftest.py (props changed) pypy/dist/pypy/tool/build/systemoption.py (props changed) pypy/dist/pypy/tool/build/test/fake.py (props changed) pypy/dist/pypy/tool/build/test/test_client.py (props changed) pypy/dist/pypy/tool/build/test/test_request_storage.py (props changed) pypy/dist/pypy/tool/build/test/test_server.py (props changed) pypy/dist/pypy/translator/js/doc/ (props changed) pypy/dist/pypy/translator/js/doc/domref.txt (props changed) Log: fixeol Modified: pypy/dist/pypy/doc/summer-of-pypy.txt ============================================================================== --- pypy/dist/pypy/doc/summer-of-pypy.txt (original) +++ pypy/dist/pypy/doc/summer-of-pypy.txt Wed Aug 9 12:20:46 2006 @@ -1,98 +1,98 @@ -Summer of PyPy -======================= - -Open Call for Proposals for PyPy! ------------------------------------ - -If you are a student and would like to work in an area related -to PyPy (there are tons of them :-)) you may consider applying -for reimbursement of your travel+expenses for attending PyPy sprints. -For this you need to come up with a brief synopsis/proposal that contains: - -- a clear description of intended work -- one or more deliverables with an estimated timeframe -- your background and why you'd like to tackle it - -To help you with preparing your proposal you can -chat to some developers first (on #pypy irc.freenode.net) or -post to ``pypy-dev at codespeak net`` and ask for feedback. - -You may want to check out `project ideas`_ to get some inspiration. - -The resulting synopsis should be sent to `pypy-tb at codespeak net` -from which you should get feedback after some days. There is no particular -deadline. - -In case your proposal is accepted you will get an assigned -mentor who will give concrete feedback on questions and code - -apart from the fact that you can always use the IRC channel or -pypy-dev for discussions/help. - -Accepted students can attend sprints and get reimbursements -for travel and accomodation costs. - -.. _`project ideas`: project-ideas.html - -Procedural details ------------------------ - -As an accepted student you should register early for PyPy sprints -(see the according announcements sent to pypy-dev) -which usually get announced on the developer mailing list. It's -possible that not all sprint participations can be funded but -currently there are not a priori limits. - -Note that - similar to Google's Summer of code - there are hardly -any restrictions for participating: you can make use of your -work in other contexts as well apart from the fact that -you need to license your works under the MIT license. - -If you have any questions regarding procedural details -(travels, receipts, filling out the template, etc.) -please send mail to ``pypy-sop at codespeak net``. - - -getting reimbursements -++++++++++++++++++++++ - -For getting reimbursements you must collect your travel receipts -(train/plane tickets etc.). For each sprint you need -to fill out a template that will be send to you after -your proposal got accepted. We aim at getting -reimbursements to your bank account after each two sprints. - -Currently the reimbursements are determined and -calculated like this: - - * full reimbursement for travel costs (flights, train etc.) - * per-diem of 80 Euro per Day for accomodation - * per-diem of 40 Euro per Day for food - -However, if you do not fulfil the tasks described in -your proposal the actual reimbursements may get reduced. - - -Accepted Proposals ---------------------------- - -- `Ajax in python based on PyPy's JavaScript backend`_ - by Maciej Fijalkowski, mentored by Eric van Riet Paap - -- `Write and port modules from CPython with ctypes`_ - by Lawrence Oluyede, mentored by Anders Chrigstroem - -- `Complete gencli, the PyPy CLI backend`_ - by Antonio Cuni, mentored by Armin Rigo - -Note that Maciej, Lawrence and Antonio also have their proposal -registered with the Google's Summer of Code under the PSF_. - - -.. _PSF: http://code.google.com/soc/psf/about.html - -.. _`Ajax in python based on PyPy's JavaScript backend`: http://code.google.com/soc/psf/appinfo.html?csaid=12134111E3C90670 - -.. _`Write and port modules from CPython with ctypes`: http://codespeak.net/svn/user/rhymes/proposal_abstract.txt - -.. _`Complete gencli, the PyPy CLI backend`: http://code.google.com/soc/psf/appinfo.html?csaid=CB4EC4BA30BDDBCD - +Summer of PyPy +======================= + +Open Call for Proposals for PyPy! +----------------------------------- + +If you are a student and would like to work in an area related +to PyPy (there are tons of them :-)) you may consider applying +for reimbursement of your travel+expenses for attending PyPy sprints. +For this you need to come up with a brief synopsis/proposal that contains: + +- a clear description of intended work +- one or more deliverables with an estimated timeframe +- your background and why you'd like to tackle it + +To help you with preparing your proposal you can +chat to some developers first (on #pypy irc.freenode.net) or +post to ``pypy-dev at codespeak net`` and ask for feedback. + +You may want to check out `project ideas`_ to get some inspiration. + +The resulting synopsis should be sent to `pypy-tb at codespeak net` +from which you should get feedback after some days. There is no particular +deadline. + +In case your proposal is accepted you will get an assigned +mentor who will give concrete feedback on questions and code - +apart from the fact that you can always use the IRC channel or +pypy-dev for discussions/help. + +Accepted students can attend sprints and get reimbursements +for travel and accomodation costs. + +.. _`project ideas`: project-ideas.html + +Procedural details +----------------------- + +As an accepted student you should register early for PyPy sprints +(see the according announcements sent to pypy-dev) +which usually get announced on the developer mailing list. It's +possible that not all sprint participations can be funded but +currently there are not a priori limits. + +Note that - similar to Google's Summer of code - there are hardly +any restrictions for participating: you can make use of your +work in other contexts as well apart from the fact that +you need to license your works under the MIT license. + +If you have any questions regarding procedural details +(travels, receipts, filling out the template, etc.) +please send mail to ``pypy-sop at codespeak net``. + + +getting reimbursements +++++++++++++++++++++++ + +For getting reimbursements you must collect your travel receipts +(train/plane tickets etc.). For each sprint you need +to fill out a template that will be send to you after +your proposal got accepted. We aim at getting +reimbursements to your bank account after each two sprints. + +Currently the reimbursements are determined and +calculated like this: + + * full reimbursement for travel costs (flights, train etc.) + * per-diem of 80 Euro per Day for accomodation + * per-diem of 40 Euro per Day for food + +However, if you do not fulfil the tasks described in +your proposal the actual reimbursements may get reduced. + + +Accepted Proposals +--------------------------- + +- `Ajax in python based on PyPy's JavaScript backend`_ + by Maciej Fijalkowski, mentored by Eric van Riet Paap + +- `Write and port modules from CPython with ctypes`_ + by Lawrence Oluyede, mentored by Anders Chrigstroem + +- `Complete gencli, the PyPy CLI backend`_ + by Antonio Cuni, mentored by Armin Rigo + +Note that Maciej, Lawrence and Antonio also have their proposal +registered with the Google's Summer of Code under the PSF_. + + +.. _PSF: http://code.google.com/soc/psf/about.html + +.. _`Ajax in python based on PyPy's JavaScript backend`: http://code.google.com/soc/psf/appinfo.html?csaid=12134111E3C90670 + +.. _`Write and port modules from CPython with ctypes`: http://codespeak.net/svn/user/rhymes/proposal_abstract.txt + +.. _`Complete gencli, the PyPy CLI backend`: http://code.google.com/soc/psf/appinfo.html?csaid=CB4EC4BA30BDDBCD + Modified: pypy/dist/pypy/module/_md5/__init__.py ============================================================================== --- pypy/dist/pypy/module/_md5/__init__.py (original) +++ pypy/dist/pypy/module/_md5/__init__.py Wed Aug 9 12:20:46 2006 @@ -1,10 +1,10 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - appleveldefs = {} - - interpleveldefs = { - # constants / module definitions - 'digest_size' : 'space.wrap(16)', - 'new' : 'interp_md5.new_md5', +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + appleveldefs = {} + + interpleveldefs = { + # constants / module definitions + 'digest_size' : 'space.wrap(16)', + 'new' : 'interp_md5.new_md5', 'md5' : 'interp_md5.new_md5'} \ No newline at end of file Modified: pypy/dist/pypy/module/_md5/interp_md5.py ============================================================================== --- pypy/dist/pypy/module/_md5/interp_md5.py (original) +++ pypy/dist/pypy/module/_md5/interp_md5.py Wed Aug 9 12:20:46 2006 @@ -1,50 +1,50 @@ -from pypy.interpreter.error import OperationError -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app -from pypy.interpreter.baseobjspace import Wrappable -from pypy.lib.md5 import MD5Type - -class W_MD5Type(Wrappable): - """A wrappable box around an interp level md5 object.""" - def __init__(self, md5o=None): - if md5o is None: - self.md5 = MD5Type() - else: - self.md5 = md5o - - def update(self, space, arg): - self.md5.update(arg) - update.unwrap_spec = ['self', ObjSpace, str] - - def digest(self, space): - d = self.md5.digest() - return space.wrap(d) - digest.unwrap_spec = ['self', ObjSpace] - - def hexdigest(self, space): - hd = self.md5.hexdigest() - return space.wrap(hd) - hexdigest.unwrap_spec = ['self', ObjSpace] - - def copy(self, space): - cmd5 = self.md5.copy() - return W_MD5Type(cmd5) - copy.unwrap_spec = ['self', ObjSpace] - -W_MD5Type.typedef = TypeDef("W_MD5Type", - update = interp2app(W_MD5Type.update, unwrap_spec=W_MD5Type.update.unwrap_spec), - digest = interp2app(W_MD5Type.digest, unwrap_spec=W_MD5Type.digest.unwrap_spec), - hexdigest = interp2app(W_MD5Type.hexdigest, unwrap_spec=W_MD5Type.hexdigest.unwrap_spec), - copy = interp2app(W_MD5Type.copy, unwrap_spec=W_MD5Type.copy.unwrap_spec), - ) -def new_md5(space, arg=''): - """ - Return a new md5 crypto object. - If arg is present, the method call update(arg) is made. - """ - - w_crypto = W_MD5Type() - if len(arg) != 0: - w_crypto.update(space, arg) - return w_crypto -new_md5.unwrap_spec = [ObjSpace, str] +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app +from pypy.interpreter.baseobjspace import Wrappable +from pypy.lib.md5 import MD5Type + +class W_MD5Type(Wrappable): + """A wrappable box around an interp level md5 object.""" + def __init__(self, md5o=None): + if md5o is None: + self.md5 = MD5Type() + else: + self.md5 = md5o + + def update(self, space, arg): + self.md5.update(arg) + update.unwrap_spec = ['self', ObjSpace, str] + + def digest(self, space): + d = self.md5.digest() + return space.wrap(d) + digest.unwrap_spec = ['self', ObjSpace] + + def hexdigest(self, space): + hd = self.md5.hexdigest() + return space.wrap(hd) + hexdigest.unwrap_spec = ['self', ObjSpace] + + def copy(self, space): + cmd5 = self.md5.copy() + return W_MD5Type(cmd5) + copy.unwrap_spec = ['self', ObjSpace] + +W_MD5Type.typedef = TypeDef("W_MD5Type", + update = interp2app(W_MD5Type.update, unwrap_spec=W_MD5Type.update.unwrap_spec), + digest = interp2app(W_MD5Type.digest, unwrap_spec=W_MD5Type.digest.unwrap_spec), + hexdigest = interp2app(W_MD5Type.hexdigest, unwrap_spec=W_MD5Type.hexdigest.unwrap_spec), + copy = interp2app(W_MD5Type.copy, unwrap_spec=W_MD5Type.copy.unwrap_spec), + ) +def new_md5(space, arg=''): + """ + Return a new md5 crypto object. + If arg is present, the method call update(arg) is made. + """ + + w_crypto = W_MD5Type() + if len(arg) != 0: + w_crypto.update(space, arg) + return w_crypto +new_md5.unwrap_spec = [ObjSpace, str] Modified: pypy/dist/pypy/module/_random/__init__.py ============================================================================== --- pypy/dist/pypy/module/_random/__init__.py (original) +++ pypy/dist/pypy/module/_random/__init__.py Wed Aug 9 12:20:46 2006 @@ -1,32 +1,32 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - appleveldefs = {} - - interpleveldefs = { - 'Random' : 'interp_random.W_Random', - 'seed' : 'interp_random.get_random_method(space, "seed")', - 'getstate' : 'interp_random.get_random_method(space, "getstate")', - 'setstate' : 'interp_random.get_random_method(space, "setstate")', - 'jumpahead' : 'interp_random.get_random_method(space, "jumpahead")', - 'randrange' : 'interp_random.get_random_method(space, "randrange")', - 'randint' : 'interp_random.get_random_method(space, "randint")', - 'choice' : 'interp_random.get_random_method(space, "choice")', - 'shuffle' : 'interp_random.get_random_method(space, "shuffle")', - 'sample' : 'interp_random.get_random_method(space, "sample")', - 'random' : 'interp_random.get_random_method(space, "random")', - 'uniform' : 'interp_random.get_random_method(space, "uniform")', - 'betavariate' : 'interp_random.get_random_method(space, "betavariate")', - 'expovariate' : 'interp_random.get_random_method(space, "expovariate")', - 'gammavariate' : 'interp_random.get_random_method(space, "gammavariate")', - 'jumpahead' : 'interp_random.get_random_method(space, "jumpahead")', - 'gauss' : 'interp_random.get_random_method(space, "gauss")', - 'lognormvariate' : 'interp_random.get_random_method(space, "lognormvariate")', - 'normalvariate' : 'interp_random.get_random_method(space, "normalvariate")', - 'vonmisesvariate' : 'interp_random.get_random_method(space, "vonmisesvariate")', - 'paretovariate' : 'interp_random.get_random_method(space, "paretovariate")', - 'cunifvariate' : 'interp_random.get_random_method(space, "cunifvariate")', - 'weibullvariate' : 'interp_random.get_random_method(space, "weibullvariate")', - 'whseed' : 'interp_random.get_random_method(space, "whseed")', # officially obsolete - } +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + appleveldefs = {} + + interpleveldefs = { + 'Random' : 'interp_random.W_Random', + 'seed' : 'interp_random.get_random_method(space, "seed")', + 'getstate' : 'interp_random.get_random_method(space, "getstate")', + 'setstate' : 'interp_random.get_random_method(space, "setstate")', + 'jumpahead' : 'interp_random.get_random_method(space, "jumpahead")', + 'randrange' : 'interp_random.get_random_method(space, "randrange")', + 'randint' : 'interp_random.get_random_method(space, "randint")', + 'choice' : 'interp_random.get_random_method(space, "choice")', + 'shuffle' : 'interp_random.get_random_method(space, "shuffle")', + 'sample' : 'interp_random.get_random_method(space, "sample")', + 'random' : 'interp_random.get_random_method(space, "random")', + 'uniform' : 'interp_random.get_random_method(space, "uniform")', + 'betavariate' : 'interp_random.get_random_method(space, "betavariate")', + 'expovariate' : 'interp_random.get_random_method(space, "expovariate")', + 'gammavariate' : 'interp_random.get_random_method(space, "gammavariate")', + 'jumpahead' : 'interp_random.get_random_method(space, "jumpahead")', + 'gauss' : 'interp_random.get_random_method(space, "gauss")', + 'lognormvariate' : 'interp_random.get_random_method(space, "lognormvariate")', + 'normalvariate' : 'interp_random.get_random_method(space, "normalvariate")', + 'vonmisesvariate' : 'interp_random.get_random_method(space, "vonmisesvariate")', + 'paretovariate' : 'interp_random.get_random_method(space, "paretovariate")', + 'cunifvariate' : 'interp_random.get_random_method(space, "cunifvariate")', + 'weibullvariate' : 'interp_random.get_random_method(space, "weibullvariate")', + 'whseed' : 'interp_random.get_random_method(space, "whseed")', # officially obsolete + } \ No newline at end of file Modified: pypy/dist/pypy/module/_random/interp_random.py ============================================================================== --- pypy/dist/pypy/module/_random/interp_random.py (original) +++ pypy/dist/pypy/module/_random/interp_random.py Wed Aug 9 12:20:46 2006 @@ -1,629 +1,629 @@ -from pypy.interpreter.error import OperationError -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app -from pypy.interpreter.baseobjspace import Wrappable -from pypy.rpython.rarithmetic import r_uint - -from math import log as _log, exp as _exp, pi as _pi, e as _e -from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin -from math import floor as _floor - -def _verify(w_name, w_computed, w_expected): - if abs(w_computed - w_expected) > 1e-7: - raise OperationError( - space.w_ValueError, - space.wrap( - "computed value for %s deviates too much " - "(computed %g, expected %g)" % (w_name, w_computed, w_expected))) - - -NV_MAGICCONST = 4 * _exp(-0.5)/_sqrt(2.0) -_verify('NV_MAGICCONST', NV_MAGICCONST, 1.71552776992141) - -TWOPI = 2.0*_pi -_verify('TWOPI', TWOPI, 6.28318530718) - -LOG4 = _log(4.0) -_verify('LOG4', LOG4, 1.38629436111989) - -SG_MAGICCONST = 1.0 + _log(4.5) -_verify('SG_MAGICCONST', SG_MAGICCONST, 2.50407739677627) - -#del _verify - -def descr_new__(space, w_subtype, w_anything=NoneNotWrapped): - x = space.allocate_instance(W_Random, w_subtype) - W_Random.__init__(x, space, w_anything) - return space.wrap(x) - -class W_Random(Wrappable): - """A wrappable box around an interp level md5 object.""" - VERSION = 1 # used by getstate/setstate - - def __init__(self, space, anything=NoneNotWrapped): - """Initialize an instance. - - Optional argument x controls seeding, as for Random.seed(). - """ - self.seed(space, anything) - - - def seed(self, space, w_a=NoneNotWrapped): - """Initialize internal state from hashable object. - - None or no argument seeds from current time. - - If a is not None or an int or long, hash(a) is used instead. - - If a is an int or long, a is used directly. Distinct values between - 0 and 27814431486575L inclusive are guaranteed to yield distinct - internal states (this guarantee is specific to the default - Wichmann-Hill generator). - """ - if w_a is None: - # Initialize from current time - import time - a = int(time.time() * 256) - else: - a = space.int_w(space.hash(w_a)) - - a, x = divmod(a, 30268) - a, y = divmod(a, 30306) - a, z = divmod(a, 30322) - self._seed = int(x)+1, int(y)+1, int(z)+1 - - self.gauss_next = None - seed.unwrap_spec = ['self', ObjSpace, W_Root] - - - def random(self, space): - """Get the next random number in the range [0.0, 1.0).""" - - # Wichman-Hill random number generator. - # - # Wichmann, B. A. & Hill, I. D. (1982) - # Algorithm AS 183: - # An efficient and portable pseudo-random number generator - # Applied Statistics 31 (1982) 188-190 - # - # see also: - # Correction to Algorithm AS 183 - # Applied Statistics 33 (1984) 123 - # - # McLeod, A. I. (1985) - # A remark on Algorithm AS 183 - # Applied Statistics 34 (1985),198-200 - - # This part is thread-unsafe: - # BEGIN CRITICAL SECTION - x, y, z = self._seed - x = (171 * x) % 30269 - y = (172 * y) % 30307 - z = (170 * z) % 30323 - self._seed = x, y, z - # END CRITICAL SECTION - - # Note: on a platform using IEEE-754 double arithmetic, this can - # never return 0.0 (asserted by Tim; proof too long for a comment). - randf = (x/30269.0 + y/30307.0 + z/30323.0) % 1.0 - return space.wrap(randf) - random.unwrap_spec = ['self', ObjSpace] - - def getstate(self, space): - """Return internal state; can be passed to setstate() later.""" - st = (self.VERSION, self._seed, self.gauss_next) - return space.wrap(st) - getstate.unwrap_spec = ['self', ObjSpace] - - def setstate(self, space, w_state): - """Restore internal state from object returned by getstate().""" - u_state = space.unwrap(w_state) - print u_state - version = u_state[0] - if version == 1: - self._seed = u_state[1] - self.gauss_next = u_state[2] - else: - raise OperationError(space.w_ValueError, - space.wrap("state with version %s passed to " - "Random.setstate() of version %s" % - (version, self.VERSION))) - setstate.unwrap_spec = ['self', ObjSpace, W_Root] - - def jumpahead(self, space, w_n): - """Act as if n calls to random() were made, but quickly. - - n is an int, greater than or equal to 0. - - Example use: If you have 2 threads and know that each will - consume no more than a million random numbers, create two Random - objects r1 and r2, then do - r2.setstate(r1.getstate()) - r2.jumpahead(1000000) - Then r1 and r2 will use guaranteed-disjoint segments of the full - period. - """ - - if not space.is_true(space.ge(w_n, space.wrap(0))): - raise OperationError(space.w_ValueError, - space.wrap("n must be >= 0")) - x, y, z = self._seed - x = (x * space.int_w(space.pow(space.wrap(171), w_n, space.wrap(30269)))) % 30269 - y = (y * space.int_w(space.pow(space.wrap(172), w_n, space.wrap(30307)))) % 30307 - z = (z * space.int_w(space.pow(space.wrap(170), w_n, space.wrap(30323)))) % 30323 - self._seed = x, y, z - jumpahead.unwrap_spec = ['self', ObjSpace, W_Root] - - - def _whseed(self, space, x=0, y=0, z=0): - """Set the Wichmann-Hill seed from (x, y, z). - - These must be integers in the range [0, 256). - """ - if not (0 <= x < 256 and 0 <= y < 256 and 0 <= z < 256): - raise OperationError(space.w_ValueError, - space.wrap('seeds must be in range(0, 256)')) - if 0 == x == y == z: - # Initialize from current time - import time - t = (int(time.time()) &0x7fffff) * 256 - t = (t&0xffffff) ^ (t>>24) - t, x = divmod(t, 256) - t, y = divmod(t, 256) - t, z = divmod(t, 256) - # Zero is a poor seed, so substitute 1 - self._seed = (x or 1, y or 1, z or 1) - - self.gauss_next = None - _whseed.unwrap_spec = ['self', ObjSpace, int, int, int] - - def whseed(self, space, w_a=NoneNotWrapped): - """Seed from hashable object's hash code. - - None or no argument seeds from current time. It is not guaranteed - that objects with distinct hash codes lead to distinct internal - states. - - This is obsolete, provided for compatibility with the seed routine - used prior to Python 2.1. Use the .seed() method instead. - """ - - if w_a is None: - self._whseed(ObjSpace) - return - else: - a = space.int_w(space.hash(w_a)) - a, x = divmod(a, 256) - a, y = divmod(a, 256) - a, z = divmod(a, 256) - x = (x + a) % 256 or 1 - y = (y + a) % 256 or 1 - z = (z + a) % 256 or 1 - self._whseed(ObjSpace, x, y, z) - whseed.unwrap_spec = ['self', ObjSpace, W_Root] - -## -------------------- pickle support ------------------- - - def __getstate__(self, space): # for pickle - return self.getstate() - - def __setstate__(self, space, state): # for pickle - self.setstate(state) - - def randrange(self, space, start, w_stop=NoneNotWrapped, step=1): - """Choose a random item from range(start, stop[, step]). - - This fixes the problem with randint() which includes the - endpoint; in Python this is usually not what you want. - Do not supply the 'int' and 'default' arguments. - """ - # This code is a bit messy to make it fast for the - # common case while still doing adequate error checking. - if w_stop is None: - if start > 0: - return space.wrap(int(self.random() * start)) - raise OperationError(space.w_ValueError, - space.wrap("empty range for randrange()")) - - # stop argument supplied. - istop = space.int_w(w_stop) - if step == 1 and start < istop: - fl = _floor(space.unwrap(self.random(space))*(istop - start)) - return space.wrap(int(start + fl)) - - if step == 1: - raise OperationError(space.w_ValueError, - space.wrap("empty range for randrange()")) - - # Non-unit step argument supplied. - if step > 0: - n = ((istop - start) + step - 1) / step - elif step < 0: - n = ((istop - start) + step + 1) / step - else: - raise OperationError(space.w_ValueError, - space.wrap("zero step for randrange()")) - - if n <= 0: - raise OperationError(space.w_ValueError, - space.wrap("empty range for randrange()")) - - res = start + step*int(space.unwrap(self.random(space)) * n) - return space.wrap(int(res)) - randrange.unwrap_spec = ['self', ObjSpace, int, W_Root, int] - - def randint(self, space, a, b): - """Return random integer in range [a, b], including both end points. - """ - return self.randrange(space, a, space.wrap(b+1)) - randint.unwrap_spec = ['self', ObjSpace, int, int] - - def choice(self, space, w_seq): - """Choose a random element from a non-empty sequence.""" - length = space.int_w(space.len(w_seq)) - ind = int(space.unwrap(self.random(space)) * length) - return space.getitem(w_seq, space.wrap(ind)) - choice.unwrap_spec = ['self', ObjSpace, W_Root] - - def shuffle(self, space, w_x, w_random=NoneNotWrapped): - """x, random=random.random -> shuffle list x in place; return None. - - Optional arg random is a 0-argument function returning a random - float in [0.0, 1.0); by default, the standard random.random. - - Note that for even rather small len(x), the total number of - permutations of x is larger than the period of most random number - generators; this implies that "most" permutations of a long - sequence can never be generated. - """ - - if w_random is None: - w_random = space.getattr(space.wrap(self), space.wrap('random')) - length = space.unwrap(space.len(w_x)) - - for i in xrange(length-1, 0, -1): - # pick an element in x[:i+1] with which to exchange x[i] - j = int(space.float_w(space.call_function(w_random)) * (i+1)) - w_i = space.wrap(i) - w_j = space.wrap(j) - w_xi = space.getitem(w_x, w_i) - w_xj = space.getitem(w_x, w_j) - space.setitem(w_x, w_i, w_xj) - space.setitem(w_x, w_j, w_xi) - shuffle.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] - - def uniform(self, space, a, b): - """Get a random number in the range [a, b).""" - return space.wrap(a + (b-a) * space.unwrap(self.random(space))) - uniform.unwrap_spec = ['self', ObjSpace, int, int] - - def normalvariate(self, space, mu, sigma): - """Normal distribution. - - mu is the mean, and sigma is the standard deviation. - - """ - # mu = mean, sigma = standard deviation - - # Uses Kinderman and Monahan method. Reference: Kinderman, - # A.J. and Monahan, J.F., "Computer generation of random - # variables using the ratio of uniform deviates", ACM Trans - # Math Software, 3, (1977), pp257-260. - while 1: - u1 = space.unwrap(self.random(space)) - u2 = 1.0 - space.unwrap(self.random(space)) - z = NV_MAGICCONST*(u1-0.5)/u2 - zz = z*z/4.0 - if zz <= -_log(u2): - break - return space.wrap(mu + z*sigma) - normalvariate.unwrap_spec = ['self', ObjSpace, float, float] - - - def lognormvariate(self, space, mu, sigma): - """Log normal distribution. - - If you take the natural logarithm of this distribution, you'll get a - normal distribution with mean mu and standard deviation sigma. - mu can have any value, and sigma must be greater than zero. - - """ - return space.wrap(_exp(space.unwrap(self.normalvariate(space, mu, sigma)))) - lognormvariate.unwrap_spec = ['self', ObjSpace, float, float] - - def cunifvariate(self, space, mean, arc): - """Circular uniform distribution. - - mean is the mean angle, and arc is the range of the distribution, - centered around the mean angle. Both values must be expressed in - radians. Returned values range between mean - arc/2 and - mean + arc/2 and are normalized to between 0 and pi. - - Deprecated in version 2.3. Use: - (mean + arc * (Random.random() - 0.5)) % Math.pi - - """ - # mean: mean angle (in radians between 0 and pi) - # arc: range of distribution (in radians between 0 and pi) - - return space.wrap((mean + arc * (space.unwrap(self.random(space)) - 0.5)) % _pi) - cunifvariate.unwrap_spec = ['self', ObjSpace, float, float] - - def expovariate(self, space, lambd): - """Exponential distribution. - - lambd is 1.0 divided by the desired mean. (The parameter would be - called "lambda", but that is a reserved word in Python.) Returned - values range from 0 to positive infinity. - - """ - # lambd: rate lambd = 1/mean - # ('lambda' is a Python reserved word) - - random = self.random - u = space.unwrap(random(space)) - while u <= 1e-7: - u = space.unwrap(random(space)) - return space.wrap(-_log(u)/lambd) - expovariate.unwrap_spec = ['self', ObjSpace, float] - - def vonmisesvariate(self, space, mu, kappa): - """Circular data distribution. - - mu is the mean angle, expressed in radians between 0 and 2*pi, and - kappa is the concentration parameter, which must be greater than or - equal to zero. If kappa is equal to zero, this distribution reduces - to a uniform random angle over the range 0 to 2*pi. - - """ - # mu: mean angle (in radians between 0 and 2*pi) - # kappa: concentration parameter kappa (>= 0) - # if kappa = 0 generate uniform random angle - - # Based upon an algorithm published in: Fisher, N.I., - # "Statistical Analysis of Circular Data", Cambridge - # University Press, 1993. - - # Thanks to Magnus Kessler for a correction to the - # implementation of step 4. - - random = self.random - if kappa <= 1e-6: - return TWOPI * space.unwrap(random(space)) - - a = 1.0 + _sqrt(1.0 + 4.0 * kappa * kappa) - b = (a - _sqrt(2.0 * a))/(2.0 * kappa) - r = (1.0 + b * b)/(2.0 * b) - - while 1: - u1 = space.unwrap(random(space)) - - z = _cos(_pi * u1) - f = (1.0 + r * z)/(r + z) - c = kappa * (r - f) - - u2 = space.unwrap(random(space)) - - if not (u2 >= c * (2.0 - c) and u2 > c * _exp(1.0 - c)): - break - - u3 = space.unwrap(random(space)) - if u3 > 0.5: - theta = (mu % TWOPI) + _acos(f) - else: - theta = (mu % TWOPI) - _acos(f) - - return space.wrap(theta) - vonmisesvariate.unwrap_spec = ['self', ObjSpace, float, float] - - def gammavariate(self, space, alpha, beta): - """Gamma distribution. Not the gamma function! - - Conditions on the parameters are alpha > 0 and beta > 0. - - """ - - # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 - - # Warning: a few older sources define the gamma distribution in terms - # of alpha > -1.0 - if alpha <= 0.0 or beta <= 0.0: - raise OperationError(space.w_ValueError, - space.wrap('gammavariate: alpha and beta must be > 0.0')) - - random = self.random - if alpha > 1.0: - - # Uses R.C.H. Cheng, "The generation of Gamma - # variables with non-integral shape parameters", - # Applied Statistics, (1977), 26, No. 1, p71-74 - - ainv = _sqrt(2.0 * alpha - 1.0) - bbb = alpha - LOG4 - ccc = alpha + ainv - - while 1: - u1 = space.unwrap(random(space)) - if not 1e-7 < u1 < .9999999: - continue - u2 = 1.0 - space.unwrap(random(space)) - v = _log(u1/(1.0-u1))/ainv - x = alpha*_exp(v) - z = u1*u1*u2 - r = bbb+ccc*v-x - if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= _log(z): - return space.wrap(x * beta) - - elif alpha == 1.0: - # expovariate(1) - u = space.unwrap(random(space)) - while u <= 1e-7: - u = space.unwrap(random(space)) - return space.wrap(-_log(u) * beta) - - else: # alpha is between 0 and 1 (exclusive) - - # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle - - while 1: - u = space.unwrap(random(space)) - b = (_e + alpha)/_e - p = b*u - if p <= 1.0: - x = pow(p, 1.0/alpha) - else: - # p > 1 - x = -_log((b-p)/alpha) - u1 = space.unwrap(random(space)) - if not (((p <= 1.0) and (u1 > _exp(-x))) or - ((p > 1) and (u1 > pow(x, alpha - 1.0)))): - break - return space.wrap(x * beta) - gammavariate.unwrap_spec = ['self', ObjSpace, float, float] - - def stdgamma(self, space, alpha, ainv, bbb, ccc): - # This method was (and shall remain) undocumented. - # This method is deprecated - # for the following reasons: - # 1. Returns same as .gammavariate(alpha, 1.0) - # 2. Requires caller to provide 3 extra arguments - # that are functions of alpha anyway - # 3. Can't be used for alpha < 0.5 - - # ainv = sqrt(2 * alpha - 1) - # bbb = alpha - log(4) - # ccc = alpha + ainv - - # XXX there is no warning support in pypy ! - print "The stdgamma function is deprecated; use gammavariate() instead" - - return self.gammavariate(space, alpha, 1.0) - stdgamma.unwrap_spec = ['self', ObjSpace, float, float, float, float] - - def gauss(self, space, mu, sigma): - """Gaussian distribution. - - mu is the mean, and sigma is the standard deviation. This is - slightly faster than the normalvariate() function. - - Not thread-safe without a lock around calls. - - """ - - # When x and y are two variables from [0, 1), uniformly - # distributed, then - # - # cos(2*pi*x)*sqrt(-2*log(1-y)) - # sin(2*pi*x)*sqrt(-2*log(1-y)) - # - # are two *independent* variables with normal distribution - # (mu = 0, sigma = 1). - # (Lambert Meertens) - # (corrected version; bug discovered by Mike Miller, fixed by LM) - - # Multithreading note: When two threads call this function - # simultaneously, it is possible that they will receive the - # same return value. The window is very small though. To - # avoid this, you have to use a lock around all calls. (I - # didn't want to slow this down in the serial case by using a - # lock here.) - - random = self.random - z = self.gauss_next - self.gauss_next = None - if z is None: - x2pi = space.unwrap(random(space)) * TWOPI - g2rad = _sqrt(-2.0 * _log(1.0 - space.unwrap(random(space)))) - z = _cos(x2pi) * g2rad - self.gauss_next = _sin(x2pi) * g2rad - - return space.wrap(mu + z*sigma) - gauss.unwrap_spec = ['self', ObjSpace, float, float] - -## -------------------- beta -------------------- -## See -## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470 -## for Ivan Frohne's insightful analysis of why the original implementation: -## -## def betavariate(self, alpha, beta): -## # Discrete Event Simulation in C, pp 87-88. -## -## y = self.expovariate(alpha) -## z = self.expovariate(1.0/beta) -## return z/(y+z) -## -## was dead wrong, and how it probably got that way. - - def betavariate(self, space, alpha, beta): - """Beta distribution. - - Conditions on the parameters are alpha > -1 and beta} > -1. - Returned values range between 0 and 1. - - """ - - # This version due to Janne Sinkkonen, and matches all the std - # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). - y = space.unwrap(self.gammavariate(space, alpha, 1.)) - if y == 0: - return space.wrap(0.0) - else: - return space.wrap(y / (y + space.unwrap(self.gammavariate(space, beta, 1.)))) - betavariate.unwrap_spec = ['self', ObjSpace, float, float] - - def paretovariate(self, space, alpha): - """Pareto distribution. alpha is the shape parameter.""" - # Jain, pg. 495 - - u = 1.0 - space.unwrap(self.random(space)) - return space.wrap(1.0 / pow(u, 1.0/alpha)) - paretovariate.unwrap_spec = ['self', ObjSpace, float] - - def weibullvariate(self, space, alpha, beta): - """Weibull distribution. - - alpha is the scale parameter and beta is the shape parameter. - - """ - # Jain, pg. 499; bug fix courtesy Bill Arms - - u = 1.0 - space.unwrap(self.random(space)) - return space.wrap(alpha * pow(-_log(u), 1.0/beta)) - weibullvariate.unwrap_spec = ['self', ObjSpace, float, float] - - -W_Random.typedef = TypeDef("W_Random", - __new__ = interp2app(descr_new__), - seed = interp2app(W_Random.seed), - random = interp2app(W_Random.random), - getstate = interp2app(W_Random.getstate), - setstate = interp2app(W_Random.setstate), - jumpahead = interp2app(W_Random.jumpahead), - _whseed = interp2app(W_Random._whseed), - whseed = interp2app(W_Random.whseed), - randrange = interp2app(W_Random.randrange), - randint = interp2app(W_Random.randint), - choice = interp2app(W_Random.choice), - shuffle = interp2app(W_Random.shuffle), - uniform = interp2app(W_Random.uniform), - normalvariate = interp2app(W_Random.normalvariate), - lognormvariate = interp2app(W_Random.lognormvariate), - cunifvariate = interp2app(W_Random.cunifvariate), - expovariate = interp2app(W_Random.expovariate), - vonmisesvariate = interp2app(W_Random.vonmisesvariate), - gammavariate = interp2app(W_Random.gammavariate), - gauss = interp2app(W_Random.gauss), - betavariate = interp2app(W_Random.betavariate), - paretovariate = interp2app(W_Random.paretovariate), - weibullvariate = interp2app(W_Random.weibullvariate), - stdgamma = interp2app(W_Random.stdgamma), - ) -_inst_map = {} - -def get_random_method(space, attrname): - try: - w_self = _inst_map[space] - except KeyError: - _inst_map[space] = w_self = W_Random(space, None) - w_method = space.getattr(w_self,space.wrap(attrname)) - return w_method +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app +from pypy.interpreter.baseobjspace import Wrappable +from pypy.rpython.rarithmetic import r_uint + +from math import log as _log, exp as _exp, pi as _pi, e as _e +from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin +from math import floor as _floor + +def _verify(w_name, w_computed, w_expected): + if abs(w_computed - w_expected) > 1e-7: + raise OperationError( + space.w_ValueError, + space.wrap( + "computed value for %s deviates too much " + "(computed %g, expected %g)" % (w_name, w_computed, w_expected))) + + +NV_MAGICCONST = 4 * _exp(-0.5)/_sqrt(2.0) +_verify('NV_MAGICCONST', NV_MAGICCONST, 1.71552776992141) + +TWOPI = 2.0*_pi +_verify('TWOPI', TWOPI, 6.28318530718) + +LOG4 = _log(4.0) +_verify('LOG4', LOG4, 1.38629436111989) + +SG_MAGICCONST = 1.0 + _log(4.5) +_verify('SG_MAGICCONST', SG_MAGICCONST, 2.50407739677627) + +#del _verify + +def descr_new__(space, w_subtype, w_anything=NoneNotWrapped): + x = space.allocate_instance(W_Random, w_subtype) + W_Random.__init__(x, space, w_anything) + return space.wrap(x) + +class W_Random(Wrappable): + """A wrappable box around an interp level md5 object.""" + VERSION = 1 # used by getstate/setstate + + def __init__(self, space, anything=NoneNotWrapped): + """Initialize an instance. + + Optional argument x controls seeding, as for Random.seed(). + """ + self.seed(space, anything) + + + def seed(self, space, w_a=NoneNotWrapped): + """Initialize internal state from hashable object. + + None or no argument seeds from current time. + + If a is not None or an int or long, hash(a) is used instead. + + If a is an int or long, a is used directly. Distinct values between + 0 and 27814431486575L inclusive are guaranteed to yield distinct + internal states (this guarantee is specific to the default + Wichmann-Hill generator). + """ + if w_a is None: + # Initialize from current time + import time + a = int(time.time() * 256) + else: + a = space.int_w(space.hash(w_a)) + + a, x = divmod(a, 30268) + a, y = divmod(a, 30306) + a, z = divmod(a, 30322) + self._seed = int(x)+1, int(y)+1, int(z)+1 + + self.gauss_next = None + seed.unwrap_spec = ['self', ObjSpace, W_Root] + + + def random(self, space): + """Get the next random number in the range [0.0, 1.0).""" + + # Wichman-Hill random number generator. + # + # Wichmann, B. A. & Hill, I. D. (1982) + # Algorithm AS 183: + # An efficient and portable pseudo-random number generator + # Applied Statistics 31 (1982) 188-190 + # + # see also: + # Correction to Algorithm AS 183 + # Applied Statistics 33 (1984) 123 + # + # McLeod, A. I. (1985) + # A remark on Algorithm AS 183 + # Applied Statistics 34 (1985),198-200 + + # This part is thread-unsafe: + # BEGIN CRITICAL SECTION + x, y, z = self._seed + x = (171 * x) % 30269 + y = (172 * y) % 30307 + z = (170 * z) % 30323 + self._seed = x, y, z + # END CRITICAL SECTION + + # Note: on a platform using IEEE-754 double arithmetic, this can + # never return 0.0 (asserted by Tim; proof too long for a comment). + randf = (x/30269.0 + y/30307.0 + z/30323.0) % 1.0 + return space.wrap(randf) + random.unwrap_spec = ['self', ObjSpace] + + def getstate(self, space): + """Return internal state; can be passed to setstate() later.""" + st = (self.VERSION, self._seed, self.gauss_next) + return space.wrap(st) + getstate.unwrap_spec = ['self', ObjSpace] + + def setstate(self, space, w_state): + """Restore internal state from object returned by getstate().""" + u_state = space.unwrap(w_state) + print u_state + version = u_state[0] + if version == 1: + self._seed = u_state[1] + self.gauss_next = u_state[2] + else: + raise OperationError(space.w_ValueError, + space.wrap("state with version %s passed to " + "Random.setstate() of version %s" % + (version, self.VERSION))) + setstate.unwrap_spec = ['self', ObjSpace, W_Root] + + def jumpahead(self, space, w_n): + """Act as if n calls to random() were made, but quickly. + + n is an int, greater than or equal to 0. + + Example use: If you have 2 threads and know that each will + consume no more than a million random numbers, create two Random + objects r1 and r2, then do + r2.setstate(r1.getstate()) + r2.jumpahead(1000000) + Then r1 and r2 will use guaranteed-disjoint segments of the full + period. + """ + + if not space.is_true(space.ge(w_n, space.wrap(0))): + raise OperationError(space.w_ValueError, + space.wrap("n must be >= 0")) + x, y, z = self._seed + x = (x * space.int_w(space.pow(space.wrap(171), w_n, space.wrap(30269)))) % 30269 + y = (y * space.int_w(space.pow(space.wrap(172), w_n, space.wrap(30307)))) % 30307 + z = (z * space.int_w(space.pow(space.wrap(170), w_n, space.wrap(30323)))) % 30323 + self._seed = x, y, z + jumpahead.unwrap_spec = ['self', ObjSpace, W_Root] + + + def _whseed(self, space, x=0, y=0, z=0): + """Set the Wichmann-Hill seed from (x, y, z). + + These must be integers in the range [0, 256). + """ + if not (0 <= x < 256 and 0 <= y < 256 and 0 <= z < 256): + raise OperationError(space.w_ValueError, + space.wrap('seeds must be in range(0, 256)')) + if 0 == x == y == z: + # Initialize from current time + import time + t = (int(time.time()) &0x7fffff) * 256 + t = (t&0xffffff) ^ (t>>24) + t, x = divmod(t, 256) + t, y = divmod(t, 256) + t, z = divmod(t, 256) + # Zero is a poor seed, so substitute 1 + self._seed = (x or 1, y or 1, z or 1) + + self.gauss_next = None + _whseed.unwrap_spec = ['self', ObjSpace, int, int, int] + + def whseed(self, space, w_a=NoneNotWrapped): + """Seed from hashable object's hash code. + + None or no argument seeds from current time. It is not guaranteed + that objects with distinct hash codes lead to distinct internal + states. + + This is obsolete, provided for compatibility with the seed routine + used prior to Python 2.1. Use the .seed() method instead. + """ + + if w_a is None: + self._whseed(ObjSpace) + return + else: + a = space.int_w(space.hash(w_a)) + a, x = divmod(a, 256) + a, y = divmod(a, 256) + a, z = divmod(a, 256) + x = (x + a) % 256 or 1 + y = (y + a) % 256 or 1 + z = (z + a) % 256 or 1 + self._whseed(ObjSpace, x, y, z) + whseed.unwrap_spec = ['self', ObjSpace, W_Root] + +## -------------------- pickle support ------------------- + + def __getstate__(self, space): # for pickle + return self.getstate() + + def __setstate__(self, space, state): # for pickle + self.setstate(state) + + def randrange(self, space, start, w_stop=NoneNotWrapped, step=1): + """Choose a random item from range(start, stop[, step]). + + This fixes the problem with randint() which includes the + endpoint; in Python this is usually not what you want. + Do not supply the 'int' and 'default' arguments. + """ + # This code is a bit messy to make it fast for the + # common case while still doing adequate error checking. + if w_stop is None: + if start > 0: + return space.wrap(int(self.random() * start)) + raise OperationError(space.w_ValueError, + space.wrap("empty range for randrange()")) + + # stop argument supplied. + istop = space.int_w(w_stop) + if step == 1 and start < istop: + fl = _floor(space.unwrap(self.random(space))*(istop - start)) + return space.wrap(int(start + fl)) + + if step == 1: + raise OperationError(space.w_ValueError, + space.wrap("empty range for randrange()")) + + # Non-unit step argument supplied. + if step > 0: + n = ((istop - start) + step - 1) / step + elif step < 0: + n = ((istop - start) + step + 1) / step + else: + raise OperationError(space.w_ValueError, + space.wrap("zero step for randrange()")) + + if n <= 0: + raise OperationError(space.w_ValueError, + space.wrap("empty range for randrange()")) + + res = start + step*int(space.unwrap(self.random(space)) * n) + return space.wrap(int(res)) + randrange.unwrap_spec = ['self', ObjSpace, int, W_Root, int] + + def randint(self, space, a, b): + """Return random integer in range [a, b], including both end points. + """ + return self.randrange(space, a, space.wrap(b+1)) + randint.unwrap_spec = ['self', ObjSpace, int, int] + + def choice(self, space, w_seq): + """Choose a random element from a non-empty sequence.""" + length = space.int_w(space.len(w_seq)) + ind = int(space.unwrap(self.random(space)) * length) + return space.getitem(w_seq, space.wrap(ind)) + choice.unwrap_spec = ['self', ObjSpace, W_Root] + + def shuffle(self, space, w_x, w_random=NoneNotWrapped): + """x, random=random.random -> shuffle list x in place; return None. + + Optional arg random is a 0-argument function returning a random + float in [0.0, 1.0); by default, the standard random.random. + + Note that for even rather small len(x), the total number of + permutations of x is larger than the period of most random number + generators; this implies that "most" permutations of a long + sequence can never be generated. + """ + + if w_random is None: + w_random = space.getattr(space.wrap(self), space.wrap('random')) + length = space.unwrap(space.len(w_x)) + + for i in xrange(length-1, 0, -1): + # pick an element in x[:i+1] with which to exchange x[i] + j = int(space.float_w(space.call_function(w_random)) * (i+1)) + w_i = space.wrap(i) + w_j = space.wrap(j) + w_xi = space.getitem(w_x, w_i) + w_xj = space.getitem(w_x, w_j) + space.setitem(w_x, w_i, w_xj) + space.setitem(w_x, w_j, w_xi) + shuffle.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def uniform(self, space, a, b): + """Get a random number in the range [a, b).""" + return space.wrap(a + (b-a) * space.unwrap(self.random(space))) + uniform.unwrap_spec = ['self', ObjSpace, int, int] + + def normalvariate(self, space, mu, sigma): + """Normal distribution. + + mu is the mean, and sigma is the standard deviation. + + """ + # mu = mean, sigma = standard deviation + + # Uses Kinderman and Monahan method. Reference: Kinderman, + # A.J. and Monahan, J.F., "Computer generation of random + # variables using the ratio of uniform deviates", ACM Trans + # Math Software, 3, (1977), pp257-260. + while 1: + u1 = space.unwrap(self.random(space)) + u2 = 1.0 - space.unwrap(self.random(space)) + z = NV_MAGICCONST*(u1-0.5)/u2 + zz = z*z/4.0 + if zz <= -_log(u2): + break + return space.wrap(mu + z*sigma) + normalvariate.unwrap_spec = ['self', ObjSpace, float, float] + + + def lognormvariate(self, space, mu, sigma): + """Log normal distribution. + + If you take the natural logarithm of this distribution, you'll get a + normal distribution with mean mu and standard deviation sigma. + mu can have any value, and sigma must be greater than zero. + + """ + return space.wrap(_exp(space.unwrap(self.normalvariate(space, mu, sigma)))) + lognormvariate.unwrap_spec = ['self', ObjSpace, float, float] + + def cunifvariate(self, space, mean, arc): + """Circular uniform distribution. + + mean is the mean angle, and arc is the range of the distribution, + centered around the mean angle. Both values must be expressed in + radians. Returned values range between mean - arc/2 and + mean + arc/2 and are normalized to between 0 and pi. + + Deprecated in version 2.3. Use: + (mean + arc * (Random.random() - 0.5)) % Math.pi + + """ + # mean: mean angle (in radians between 0 and pi) + # arc: range of distribution (in radians between 0 and pi) + + return space.wrap((mean + arc * (space.unwrap(self.random(space)) - 0.5)) % _pi) + cunifvariate.unwrap_spec = ['self', ObjSpace, float, float] + + def expovariate(self, space, lambd): + """Exponential distribution. + + lambd is 1.0 divided by the desired mean. (The parameter would be + called "lambda", but that is a reserved word in Python.) Returned + values range from 0 to positive infinity. + + """ + # lambd: rate lambd = 1/mean + # ('lambda' is a Python reserved word) + + random = self.random + u = space.unwrap(random(space)) + while u <= 1e-7: + u = space.unwrap(random(space)) + return space.wrap(-_log(u)/lambd) + expovariate.unwrap_spec = ['self', ObjSpace, float] + + def vonmisesvariate(self, space, mu, kappa): + """Circular data distribution. + + mu is the mean angle, expressed in radians between 0 and 2*pi, and + kappa is the concentration parameter, which must be greater than or + equal to zero. If kappa is equal to zero, this distribution reduces + to a uniform random angle over the range 0 to 2*pi. + + """ + # mu: mean angle (in radians between 0 and 2*pi) + # kappa: concentration parameter kappa (>= 0) + # if kappa = 0 generate uniform random angle + + # Based upon an algorithm published in: Fisher, N.I., + # "Statistical Analysis of Circular Data", Cambridge + # University Press, 1993. + + # Thanks to Magnus Kessler for a correction to the + # implementation of step 4. + + random = self.random + if kappa <= 1e-6: + return TWOPI * space.unwrap(random(space)) + + a = 1.0 + _sqrt(1.0 + 4.0 * kappa * kappa) + b = (a - _sqrt(2.0 * a))/(2.0 * kappa) + r = (1.0 + b * b)/(2.0 * b) + + while 1: + u1 = space.unwrap(random(space)) + + z = _cos(_pi * u1) + f = (1.0 + r * z)/(r + z) + c = kappa * (r - f) + + u2 = space.unwrap(random(space)) + + if not (u2 >= c * (2.0 - c) and u2 > c * _exp(1.0 - c)): + break + + u3 = space.unwrap(random(space)) + if u3 > 0.5: + theta = (mu % TWOPI) + _acos(f) + else: + theta = (mu % TWOPI) - _acos(f) + + return space.wrap(theta) + vonmisesvariate.unwrap_spec = ['self', ObjSpace, float, float] + + def gammavariate(self, space, alpha, beta): + """Gamma distribution. Not the gamma function! + + Conditions on the parameters are alpha > 0 and beta > 0. + + """ + + # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 + + # Warning: a few older sources define the gamma distribution in terms + # of alpha > -1.0 + if alpha <= 0.0 or beta <= 0.0: + raise OperationError(space.w_ValueError, + space.wrap('gammavariate: alpha and beta must be > 0.0')) + + random = self.random + if alpha > 1.0: + + # Uses R.C.H. Cheng, "The generation of Gamma + # variables with non-integral shape parameters", + # Applied Statistics, (1977), 26, No. 1, p71-74 + + ainv = _sqrt(2.0 * alpha - 1.0) + bbb = alpha - LOG4 + ccc = alpha + ainv + + while 1: + u1 = space.unwrap(random(space)) + if not 1e-7 < u1 < .9999999: + continue + u2 = 1.0 - space.unwrap(random(space)) + v = _log(u1/(1.0-u1))/ainv + x = alpha*_exp(v) + z = u1*u1*u2 + r = bbb+ccc*v-x + if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= _log(z): + return space.wrap(x * beta) + + elif alpha == 1.0: + # expovariate(1) + u = space.unwrap(random(space)) + while u <= 1e-7: + u = space.unwrap(random(space)) + return space.wrap(-_log(u) * beta) + + else: # alpha is between 0 and 1 (exclusive) + + # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle + + while 1: + u = space.unwrap(random(space)) + b = (_e + alpha)/_e + p = b*u + if p <= 1.0: + x = pow(p, 1.0/alpha) + else: + # p > 1 + x = -_log((b-p)/alpha) + u1 = space.unwrap(random(space)) + if not (((p <= 1.0) and (u1 > _exp(-x))) or + ((p > 1) and (u1 > pow(x, alpha - 1.0)))): + break + return space.wrap(x * beta) + gammavariate.unwrap_spec = ['self', ObjSpace, float, float] + + def stdgamma(self, space, alpha, ainv, bbb, ccc): + # This method was (and shall remain) undocumented. + # This method is deprecated + # for the following reasons: + # 1. Returns same as .gammavariate(alpha, 1.0) + # 2. Requires caller to provide 3 extra arguments + # that are functions of alpha anyway + # 3. Can't be used for alpha < 0.5 + + # ainv = sqrt(2 * alpha - 1) + # bbb = alpha - log(4) + # ccc = alpha + ainv + + # XXX there is no warning support in pypy ! + print "The stdgamma function is deprecated; use gammavariate() instead" + + return self.gammavariate(space, alpha, 1.0) + stdgamma.unwrap_spec = ['self', ObjSpace, float, float, float, float] + + def gauss(self, space, mu, sigma): + """Gaussian distribution. + + mu is the mean, and sigma is the standard deviation. This is + slightly faster than the normalvariate() function. + + Not thread-safe without a lock around calls. + + """ + + # When x and y are two variables from [0, 1), uniformly + # distributed, then + # + # cos(2*pi*x)*sqrt(-2*log(1-y)) + # sin(2*pi*x)*sqrt(-2*log(1-y)) + # + # are two *independent* variables with normal distribution + # (mu = 0, sigma = 1). + # (Lambert Meertens) + # (corrected version; bug discovered by Mike Miller, fixed by LM) + + # Multithreading note: When two threads call this function + # simultaneously, it is possible that they will receive the + # same return value. The window is very small though. To + # avoid this, you have to use a lock around all calls. (I + # didn't want to slow this down in the serial case by using a + # lock here.) + + random = self.random + z = self.gauss_next + self.gauss_next = None + if z is None: + x2pi = space.unwrap(random(space)) * TWOPI + g2rad = _sqrt(-2.0 * _log(1.0 - space.unwrap(random(space)))) + z = _cos(x2pi) * g2rad + self.gauss_next = _sin(x2pi) * g2rad + + return space.wrap(mu + z*sigma) + gauss.unwrap_spec = ['self', ObjSpace, float, float] + +## -------------------- beta -------------------- +## See +## http://sourceforge.net/bugs/?func=detailbug&bug_id=130030&group_id=5470 +## for Ivan Frohne's insightful analysis of why the original implementation: +## +## def betavariate(self, alpha, beta): +## # Discrete Event Simulation in C, pp 87-88. +## +## y = self.expovariate(alpha) +## z = self.expovariate(1.0/beta) +## return z/(y+z) +## +## was dead wrong, and how it probably got that way. + + def betavariate(self, space, alpha, beta): + """Beta distribution. + + Conditions on the parameters are alpha > -1 and beta} > -1. + Returned values range between 0 and 1. + + """ + + # This version due to Janne Sinkkonen, and matches all the std + # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). + y = space.unwrap(self.gammavariate(space, alpha, 1.)) + if y == 0: + return space.wrap(0.0) + else: + return space.wrap(y / (y + space.unwrap(self.gammavariate(space, beta, 1.)))) + betavariate.unwrap_spec = ['self', ObjSpace, float, float] + + def paretovariate(self, space, alpha): + """Pareto distribution. alpha is the shape parameter.""" + # Jain, pg. 495 + + u = 1.0 - space.unwrap(self.random(space)) + return space.wrap(1.0 / pow(u, 1.0/alpha)) + paretovariate.unwrap_spec = ['self', ObjSpace, float] + + def weibullvariate(self, space, alpha, beta): + """Weibull distribution. + + alpha is the scale parameter and beta is the shape parameter. + + """ + # Jain, pg. 499; bug fix courtesy Bill Arms + + u = 1.0 - space.unwrap(self.random(space)) + return space.wrap(alpha * pow(-_log(u), 1.0/beta)) + weibullvariate.unwrap_spec = ['self', ObjSpace, float, float] + + +W_Random.typedef = TypeDef("W_Random", + __new__ = interp2app(descr_new__), + seed = interp2app(W_Random.seed), + random = interp2app(W_Random.random), + getstate = interp2app(W_Random.getstate), + setstate = interp2app(W_Random.setstate), + jumpahead = interp2app(W_Random.jumpahead), + _whseed = interp2app(W_Random._whseed), + whseed = interp2app(W_Random.whseed), + randrange = interp2app(W_Random.randrange), + randint = interp2app(W_Random.randint), + choice = interp2app(W_Random.choice), + shuffle = interp2app(W_Random.shuffle), + uniform = interp2app(W_Random.uniform), + normalvariate = interp2app(W_Random.normalvariate), + lognormvariate = interp2app(W_Random.lognormvariate), + cunifvariate = interp2app(W_Random.cunifvariate), + expovariate = interp2app(W_Random.expovariate), + vonmisesvariate = interp2app(W_Random.vonmisesvariate), + gammavariate = interp2app(W_Random.gammavariate), + gauss = interp2app(W_Random.gauss), + betavariate = interp2app(W_Random.betavariate), + paretovariate = interp2app(W_Random.paretovariate), + weibullvariate = interp2app(W_Random.weibullvariate), + stdgamma = interp2app(W_Random.stdgamma), + ) +_inst_map = {} + +def get_random_method(space, attrname): + try: + w_self = _inst_map[space] + except KeyError: + _inst_map[space] = w_self = W_Random(space, None) + w_method = space.getattr(w_self,space.wrap(attrname)) + return w_method From mwh at codespeak.net Wed Aug 9 12:34:23 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 9 Aug 2006 12:34:23 +0200 (CEST) Subject: [pypy-svn] r31192 - in pypy/dist/pypy: interpreter objspace/std tool Message-ID: <20060809103423.569DD10050@code0.codespeak.net> Author: mwh Date: Wed Aug 9 12:34:22 2006 New Revision: 31192 Added: pypy/dist/pypy/tool/readdictinfo.py Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/objspace/std/dictmultiobject.py Log: make the MeasuringDictImplemetation work in a translated pypy-c. dumps information to a file called 'dictinfo.txt' in the local directory, you can use the new pypy/tool/readdictinfo.py to make sense of this. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Wed Aug 9 12:34:22 2006 @@ -163,6 +163,10 @@ w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) + # XXX ugh: + from pypy.objspace.std.dictmultiobject import MEASURE_DICT, report + if MEASURE_DICT: + report() def __repr__(self): try: Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 9 12:34:22 2006 @@ -1,7 +1,7 @@ from pypy.objspace.std.objspace import * from pypy.interpreter import gateway -from pypy.rpython.objectmodel import r_dict +from pypy.rpython.objectmodel import r_dict, we_are_translated class DictImplementation(object): @@ -116,10 +116,10 @@ MEASURE_DICT = False if MEASURE_DICT: - import time + import time, py _dict_infos = [] - class DictInfo: + class DictInfo(object): def __init__(self): self.getitems = 0; self.setitems = 0; self.delitems = 0 self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 @@ -140,14 +140,19 @@ self.createtime = time.time() self.lifetime = -1.0 - # very probable stack from here: - # 0 - us - # 1 - MeasuringDictImplementation.__init__ - # 2 - W_DictMultiObject.__init__ - # 3 - space.newdict - # 4 - newdict's caller. let's look at that - frame = sys._getframe(4) - self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) + if not we_are_translated(): + # very probable stack from here: + # 0 - us + # 1 - MeasuringDictImplementation.__init__ + # 2 - W_DictMultiObject.__init__ + # 3 - space.newdict + # 4 - newdict's caller. let's look at that + try: + frame = sys._getframe(4) + except ValueError: + pass # might be at import time + else: + self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) _dict_infos.append(self) def __repr__(self): @@ -252,6 +257,31 @@ self.info.listings += 1 return self.content.items() + _example = DictInfo() + del _dict_infos[-1] + tmpl = ''' + os.write(fd, "%(attr)s") + os.write(fd, ": ") + os.write(fd, str(info.%(attr)s)) + os.write(fd, "\\n") + ''' + bodySrc = [] + for attr in _example.__dict__: + if attr == 'sig': + continue + bodySrc.append(tmpl%locals()) + exec py.code.Source(''' + def _report_one(fd, info): + %s + '''%''.join(bodySrc)).compile() + + def report(): + fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY, 0644) + for info in _dict_infos: + os.write(fd, '------------------\n') + _report_one(fd, info) + os.close(fd) + def reportDictInfo(): d = {} if not _dict_infos: @@ -278,8 +308,8 @@ print '('+str(stillAlive), 'still alive)' print d - import atexit - atexit.register(reportDictInfo) + #import atexit + #atexit.register(reportDictInfo) class W_DictMultiObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef Added: pypy/dist/pypy/tool/readdictinfo.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/readdictinfo.py Wed Aug 9 12:34:22 2006 @@ -0,0 +1,49 @@ +import autopath + +# this is for use with a pypy-c build with multidicts and using the +# MeasuringDictImplementation -- it will create a file called +# 'dictinfo.txt' in the local directory and this file will turn the +# contents back into DictInfo objects. + +# run with python -i ! + +from pypy.objspace.std.dictmultiobject import DictInfo + +import sys + +infile = open(sys.argv[1]) + +infos = [] + +for line in infile: + if line == '------------------\n': + curr = object.__new__(DictInfo) + infos.append(curr) + else: + attr, val = [s.strip() for s in line.split(':')] + if '.' in val: + val = float(val) + else: + val = int(val) + setattr(curr, attr, val) + +def histogram(infos, keyattr, *attrs): + r = {} + for info in infos: + v = getattr(info, keyattr) + l = r.setdefault(v, [0, {}]) + l[0] += 1 + for a in attrs: + d2 = l[1].setdefault(a, {}) + v2 = getattr(info, a) + d2[v2] = d2.get(v2, 0) + 1 + return sorted(r.items()) + +import pprint +try: + import readline +except ImportError: + pass +else: + import rlcompleter + readline.parse_and_bind('tab: complete') From auc at codespeak.net Wed Aug 9 12:35:47 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 9 Aug 2006 12:35:47 +0200 (CEST) Subject: [pypy-svn] r31193 - in pypy/dist/pypy/objspace: . cclp test Message-ID: <20060809103547.3F01210050@code0.codespeak.net> Author: auc Date: Wed Aug 9 12:35:42 2006 New Revision: 31193 Modified: pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: entailment constraint on vars. Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Wed Aug 9 12:35:42 2006 @@ -1,7 +1,7 @@ from pypy.interpreter import baseobjspace, gateway, typedef from pypy.objspace.cclp.misc import w, ClonableCoroutine -from pypy.objspace.constraint.domain import W_FiniteDomain +#from pypy.objspace.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root @@ -11,6 +11,7 @@ def __init__(w_self, space): # ring of aliases or bound value w_self.w_bound_to = w_self + w_self.entails = {} # byneed flag w_self.needed = False @@ -31,10 +32,14 @@ class W_CVar(W_Var): - def __init__(w_self, space, w_dom): + def __init__(w_self, space, w_dom): #, w_name): assert isinstance(w_dom, W_FiniteDomain) W_Var.__init__(w_self, space) w_self.w_dom = w_dom + #w_self.name = space.str_w(w_name) + + def name_w(w_self): + return w_self.name def domain_of(space, w_v): assert isinstance(w_v, W_CVar) @@ -50,6 +55,16 @@ def __init__(w_self, exc): w_self.exc = exc +#-- Constraint --------------------------------------------- + +## class W_Constraint(baseobjspace.Wrappable): +## def __init__(self, object_space): +## self._space = object_space + +## W_Constraint.typedef = typedef.TypeDef( +## "W_Constraint") + + #-- Misc --------------------------------------------------- def deref(space, w_var): Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Wed Aug 9 12:35:42 2006 @@ -3,6 +3,7 @@ from pypy.objspace.std.model import StdObjSpaceMultiMethod from pypy.objspace.std.listobject import W_ListObject, W_TupleObject from pypy.objspace.std.dictobject import W_DictObject +from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.cclp.misc import w, v, ClonableCoroutine from pypy.objspace.cclp.global_state import scheduler @@ -19,10 +20,11 @@ return w_v app_newvar = gateway.interp2app(newvar) -def domain(space, w_values): +def domain(space, w_values):#, w_name): assert isinstance(w_values, W_ListObject) + #assert isinstance(w_name, W_StringObject) w_dom = W_FiniteDomain(space, w_values) - w_var = W_CVar(space, w_dom) + w_var = W_CVar(space, w_dom)#, w_name) w("CVAR", str(w_var)) return w_var app_domain = gateway.interp2app(domain) @@ -161,8 +163,7 @@ space.wrap("This future is read-only for you, pal")) -#-- BIND ----------------------------- - +#-- BIND, ENTAIL---------------------------- def bind(space, w_var, w_obj): """1. aliasing of unbound variables @@ -175,6 +176,13 @@ space.bind(w_var, w_obj) app_bind = gateway.interp2app(bind) +def entail(space, w_v1, w_v2): + "X -> Y" + assert isinstance(w_v1, W_Var) + assert isinstance(w_v2, W_Var) + space.entail(w_v1, w_v2) +app_entail = gateway.interp2app(entail) + def bind__Var_Root(space, w_var, w_obj): #w("var val", str(id(w_var))) # 3. var and value @@ -245,7 +253,8 @@ if w_fut._client == ClonableCoroutine.w_getcurrent(space): raise_future_binding(space) return bind__Var_Var(space, w_var, w_fut) #and for me ... - + + bind_mm = StdObjSpaceMultiMethod('bind', 2) bind_mm.register(bind__Var_Root, W_Var, W_Root) bind_mm.register(bind__Var_Var, W_Var, W_Var) @@ -257,6 +266,28 @@ bind_mm.register(bind__CVar_Var, W_CVar, W_Var) all_mms['bind'] = bind_mm + +def entail__Var_Var(space, w_v1, w_v2): + w(" :entail Var Var") + if space.is_true(space.is_bound(w_v1)): + if space.is_true(space.is_bound(w_v2)): + return unify(space, + deref(space, w_v1), + deref(space, w_v2)) + return _assign_aliases(space, w_v2, deref(space, w_v1)) + else: + return _entail(space, w_v1, w_v2) + +entail_mm = StdObjSpaceMultiMethod('entail', 2) +entail_mm.register(entail__Var_Var, W_Var, W_Var) +all_mms['entail'] = entail_mm + +def _entail(space, w_v1, w_v2): + assert isinstance(w_v1, W_Var) + assert isinstance(w_v2, W_Var) + w_v1.entails[w_v2] = True + return space.w_None + def _assign_aliases(space, w_var, w_val): w(" :assign") assert isinstance(w_var, W_Var) @@ -271,9 +302,18 @@ break # switch to next w_curr = w_next + _assign_entailed(space, w_var, w_val) w(" :assigned") return space.w_None +def _assign_entailed(space, w_var, w_val): + w(" :assign entailed") + for var in w_var.entails: + if space.is_true(space.is_free(var)): + _assign_aliases(space, var, w_val) + else: + unify(space, deref(space, var), w_val) + def _assign(space, w_var, w_val): assert isinstance(w_var, W_Var) if isinstance(w_var, W_CVar): Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Wed Aug 9 12:35:42 2006 @@ -21,13 +21,15 @@ from pypy.objspace.cclp.global_state import scheduler -from pypy.objspace.cclp.space import app_newspace, app_choose, W_CSpace +#-- COMP. SPACE -------------------------------------------- + +from pypy.objspace.cclp.space import app_newspace, app_choose, W_CSpace #app_tell #-- VARIABLE ------------------------------------------------ from pypy.objspace.cclp.variable import app_newvar, wait, app_wait, app_wait_needed, \ app_is_aliased, app_is_free, app_is_bound, app_alias_of, alias_of, app_bind, \ - app_unify, W_Var, W_CVar, W_Future, app_domain, all_mms as variable_mms + app_unify, W_Var, W_CVar, W_Future, app_domain, all_mms as variable_mms, app_entail from pypy.objspace.cclp.types import app_domain_of @@ -214,6 +216,7 @@ setattr(space, name, boundmethod) # store into 'space' instance # /multimethods hack + #-- variable ------- space.setitem(space.builtin.w_dict, space.wrap('newvar'), space.wrap(app_newvar)) space.setitem(space.builtin.w_dict, space.wrap('domain'), @@ -230,6 +233,8 @@ space.wrap(app_is_aliased)) space.setitem(space.builtin.w_dict, space.wrap('bind'), space.wrap(app_bind)) + space.setitem(space.builtin.w_dict, space.wrap('entail'), + space.wrap(app_entail)) space.setitem(space.builtin.w_dict, space.wrap('unify'), space.wrap(app_unify)) ## #-- comp space --- @@ -240,7 +245,7 @@ space.wrap(domain.app_make_fd)) space.setitem(space.builtin.w_dict, space.wrap('intersection'), space.wrap(domain.app_intersection)) -## #-- constraint ---- +## #-- constraints ---- ## space.setitem(space.builtin.w_dict, space.wrap('make_expression'), ## space.wrap(constraint.app_make_expression)) ## space.setitem(space.builtin.w_dict, space.wrap('AllDistinct'), @@ -276,6 +281,8 @@ space.wrap(app_newspace)) space.setitem(space.builtin.w_dict, space.wrap('choose'), space.wrap(app_choose)) +## space.setitem(space.builtin.w_dict, space.wrap('tell'), +## space.wrap(app_tell)) #-- misc ----- space.setitem(space.builtin.w_dict, space.wrap('interp_id'), Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 9 12:35:42 2006 @@ -193,6 +193,43 @@ raises(UnificationError, unify, f1.b, 24) + def test_entail(self): + X, Y = newvar(), newvar() + entail(X, Y) + unify(42, X) + assert X == Y == 42 + + X, Y = newvar(), newvar() + entail(X, Y) + unify(42, Y) + assert is_free(X) + assert Y == 42 + unify(X, 42) + assert X == Y == 42 + + X, Y = newvar(), newvar() + entail(X, Y) + unify(42, Y) + assert is_free(X) + assert Y == 42 + raises(UnificationError, unify, X, True) + + X, Y = newvar(), newvar() + entail(X, Y) + unify(X, Y) + assert is_free(X) + assert is_free(Y) + unify(Y, 42) + assert X == Y == 42 + + X, Y, O = newvar(), newvar(), newvar() + entail(X, O) + entail(Y, O) + unify(Y, 42) + assert is_free(X) + assert Y == O == 42 + + class AppTest_LogicFutures(object): def setup_class(cls): @@ -739,3 +776,28 @@ schedule() assert len(sched_all()['threads']) == 1 + def test_tell(self): + + skip("not finished ...") + + def problem(): + X, Y = domain([1, 2], 'X'), domain([1, 2, 3], 'Y') + tell(make_expression([X, Y], 'X + Y > 4')) + + + def solve(spc, X): + while 1: + status = spc.ask() + if status == 1: + unify(X, status) + break + + switch_debug_info() + s = newspace(problem) + Finished = newvar() + stacklet(solve, s, Finished) + + wait(Finished) + + assert domain_of(X) == FiniteDomain([2]) + assert domain_of(Y) == FiniteDomain([3]) From arigo at codespeak.net Wed Aug 9 12:44:15 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 12:44:15 +0200 (CEST) Subject: [pypy-svn] r31194 - pypy/dist/pypy/translator/llvm Message-ID: <20060809104415.AFDD610050@code0.codespeak.net> Author: arigo Date: Wed Aug 9 12:44:14 2006 New Revision: 31194 Modified: pypy/dist/pypy/translator/llvm/database.py Log: Fix LLVM tests. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Aug 9 12:44:14 2006 @@ -15,6 +15,7 @@ from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.memory.lladdress import NULL from pypy.rpython.objectmodel import Symbolic, ComputedIntSymbolic +from pypy.rpython.objectmodel import CDefinedIntSymbolic log = log.database @@ -496,8 +497,10 @@ r(from_), indices_as_str) elif isinstance(value, ComputedIntSymbolic): - # XXX what does this do? Is this safe? + # force the ComputedIntSymbolic to become a real integer value now repr = '%d' % value.compute_fn() + elif isinstance(value, CDefinedIntSymbolic): + repr = CDEFINED_VALUE[value.expr] else: raise NotImplementedError("symbolic: %r" % (value,)) @@ -544,3 +547,8 @@ raise Exception("unsupported offset") return from_, indices, to + +# reprs for specific CDefinedIntSymbolic constants +CDEFINED_VALUE = { + 'MALLOC_ZERO_FILLED': '1', + } From arigo at codespeak.net Wed Aug 9 12:48:37 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 12:48:37 +0200 (CEST) Subject: [pypy-svn] r31195 - pypy/dist/pypy/interpreter Message-ID: <20060809104837.78ADE10050@code0.codespeak.net> Author: arigo Date: Wed Aug 9 12:48:35 2006 New Revision: 31195 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: Fix typo, restore annotatability. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Wed Aug 9 12:48:35 2006 @@ -456,7 +456,7 @@ w_obj.getclass(self).getname(self, '?')) raise OperationError(self.w_TypeError, self.wrap(msg)) return obj - interp_w._annspecialcase_ = 'specialize:arg(1)' + interp_w._annspecialcase_ = 'specialize:arg(1)' def unpackiterable(self, w_iterable, expected_length=-1): """Unpack an iterable object into a real (interpreter-level) list. From mwh at codespeak.net Wed Aug 9 12:50:50 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 9 Aug 2006 12:50:50 +0200 (CEST) Subject: [pypy-svn] r31196 - pypy/dist/pypy/interpreter Message-ID: <20060809105050.72F5810050@code0.codespeak.net> Author: mwh Date: Wed Aug 9 12:50:49 2006 New Revision: 31196 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: fix import stupidity Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Wed Aug 9 12:50:49 2006 @@ -164,8 +164,9 @@ if w_exitfunc is not None: self.call_function(w_exitfunc) # XXX ugh: - from pypy.objspace.std.dictmultiobject import MEASURE_DICT, report + from pypy.objspace.std.dictmultiobject import MEASURE_DICT if MEASURE_DICT: + from pypy.objspace.std.dictmultiobject import report report() def __repr__(self): From arigo at codespeak.net Wed Aug 9 13:08:37 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 13:08:37 +0200 (CEST) Subject: [pypy-svn] r31198 - pypy/dist/pypy/tool/test Message-ID: <20060809110837.E122F10053@code0.codespeak.net> Author: arigo Date: Wed Aug 9 13:08:36 2006 New Revision: 31198 Added: pypy/dist/pypy/tool/test/test_tab.py (contents, props changed) Log: A test for tabs in the source files. Added: pypy/dist/pypy/tool/test/test_tab.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/test/test_tab.py Wed Aug 9 13:08:36 2006 @@ -0,0 +1,30 @@ +""" +Verify that the PyPy source files have no tabs. +""" + +import autopath +import os + +ROOT = autopath.pypydir +EXCLUDE = {'/translator/pyrex/Pyrex': True} + + +def test_no_tabs(): + def walk(reldir): + if reldir in EXCLUDE: + return + if reldir: + path = os.path.join(ROOT, *reldir.split('/')) + else: + path = ROOT + if os.path.isfile(path): + if path.lower().endswith('.py'): + f = open(path, 'r') + data = f.read() + f.close() + assert '\t' not in data, "%r contains tabs!" % (reldir,) + elif os.path.isdir(path): + for entry in os.listdir(path): + if not entry.startswith('.'): + walk('%s/%s' % (reldir, entry)) + walk('') From arigo at codespeak.net Wed Aug 9 13:47:07 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 13:47:07 +0200 (CEST) Subject: [pypy-svn] r31199 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20060809114707.C690B10053@code0.codespeak.net> Author: arigo Date: Wed Aug 9 13:47:06 2006 New Revision: 31199 Modified: pypy/dist/pypy/module/__builtin__/app_inspect.py pypy/dist/pypy/module/__builtin__/test/test_builtin.py Log: Eat almost all exceptions in hasattr(), not just AttributeError. This is a debatable feature of CPython, but our own translation toolsuite relies on it occasionally, e.g. by doing hasattr(random_object, '__name__') which sometimes gives a TypeError. Modified: pypy/dist/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/dist/pypy/module/__builtin__/app_inspect.py Wed Aug 9 13:47:06 2006 @@ -85,7 +85,9 @@ try: getattr(obj, attr) return True - except AttributeError: + except (KeyboardInterrupt, SystemExit): + raise + except: return False # Replaced by the interp-level helper space.callable(): Modified: pypy/dist/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_builtin.py Wed Aug 9 13:47:06 2006 @@ -417,6 +417,16 @@ assert Y().f() == ((Y,), {}) assert Y().f(42, x=43) == ((Y, 42), {'x': 43}) + def test_hasattr(self): + class X(object): + def broken(): pass + abc = property(broken) + x = X() + x.foo = 42 + assert hasattr(x, '__class__') + assert hasattr(x, 'foo') + assert not hasattr(x, 'bar') + assert not hasattr(x, 'abc') # CPython compliance class TestInternal: From arigo at codespeak.net Wed Aug 9 13:47:27 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 13:47:27 +0200 (CEST) Subject: [pypy-svn] r31200 - pypy/dist/pypy/annotation/test Message-ID: <20060809114727.5CB0110063@code0.codespeak.net> Author: arigo Date: Wed Aug 9 13:47:25 2006 New Revision: 31200 Modified: pypy/dist/pypy/annotation/test/test_model.py Log: pypy-c compliance. Modified: pypy/dist/pypy/annotation/test/test_model.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_model.py (original) +++ pypy/dist/pypy/annotation/test/test_model.py Wed Aug 9 13:47:25 2006 @@ -76,8 +76,12 @@ pass class B2(object): pass - class B3(object, A0): - pass + try: + class B3(object, A0): + pass + except TypeError: # if A0 is also a new-style class, e.g. in PyPy + class B3(A0, object): + pass assert commonbase(A1,A2) is A0 assert commonbase(A1,A0) is A0 assert commonbase(A1,A1) is A1 From arigo at codespeak.net Wed Aug 9 15:17:08 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 9 Aug 2006 15:17:08 +0200 (CEST) Subject: [pypy-svn] r31202 - in pypy/dist/pypy: module/posix module/posix/test rpython rpython/lltypesystem/module rpython/module rpython/ootypesystem/module translator/c translator/c/src translator/c/test Message-ID: <20060809131708.6709710069@code0.codespeak.net> Author: arigo Date: Wed Aug 9 15:17:05 2006 New Revision: 31202 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py pypy/dist/pypy/rpython/extfunctable.py pypy/dist/pypy/rpython/lltypesystem/module/ll_os.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/ootypesystem/module/ll_os.py pypy/dist/pypy/translator/c/extfunc.py pypy/dist/pypy/translator/c/src/ll_os.h pypy/dist/pypy/translator/c/test/test_extfunc.py Log: os.fork(), os.waitpid(), os._exit(). Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Wed Aug 9 15:17:05 2006 @@ -43,6 +43,7 @@ 'pipe' : 'interp_posix.pipe', 'chmod' : 'interp_posix.chmod', 'rename' : 'interp_posix.rename', + '_exit' : 'interp_posix._exit', } if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' @@ -58,6 +59,10 @@ interpleveldefs['symlink'] = 'interp_posix.symlink' if hasattr(os, 'readlink'): interpleveldefs['readlink'] = 'interp_posix.readlink' + if hasattr(os, 'fork'): + interpleveldefs['fork'] = 'interp_posix.fork' + if hasattr(os, 'waitpid'): + interpleveldefs['waitpid'] = 'interp_posix.waitpid' for constant in dir(os): Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Wed Aug 9 15:17:05 2006 @@ -367,3 +367,16 @@ raise wrap_oserror(space, e) return space.wrap(result) readlink.unwrap_spec = [ObjSpace, str] + +def fork(space): + pid = os.fork() + return space.wrap(pid) + +def waitpid(space, pid, options): + pid, status = os.waitpid(pid, options) + return space.newtuple([space.wrap(pid), space.wrap(status)]) +waitpid.unwrap_spec = [ObjSpace, int, int] + +def _exit(space, status): + os._exit(status) +_exit.unwrap_spec = [ObjSpace, int] Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Wed Aug 9 15:17:05 2006 @@ -95,6 +95,16 @@ assert isinstance(self.posix.strerror(0), str) assert isinstance(self.posix.strerror(1), str) + if hasattr(__import__(os.name), "fork"): + def test_fork(self): + os = self.posix + pid = os.fork() + if pid == 0: # child + os._exit(4) + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + # XXX check status1 + class AppTestEnvironment(object): def setup_class(cls): cls.space = space Modified: pypy/dist/pypy/rpython/extfunctable.py ============================================================================== --- pypy/dist/pypy/rpython/extfunctable.py (original) +++ pypy/dist/pypy/rpython/extfunctable.py Wed Aug 9 15:17:05 2006 @@ -166,6 +166,13 @@ record_call(Implementation.ll_pipe_result, [SomeInteger()]*2, 'OS_PIPE') return SomeTuple((SomeInteger(),)*2) +def waitpidannotation(*args): + from pypy.rpython.lltypesystem.module.ll_os import Implementation + from pypy.annotation.model import SomeInteger, SomeTuple + record_call(Implementation.ll_waitpid_result, [SomeInteger()]*2, + 'OS_WAITPID') + return SomeTuple((SomeInteger(),)*2) + def frexpannotation(*args): from pypy.annotation.model import SomeInteger, SomeTuple, SomeFloat from pypy.rpython.lltypesystem.module.ll_math import ll_frexp_result @@ -209,6 +216,7 @@ declare(os.pipe , pipeannotation, 'll_os/pipe') declare(os.chmod , noneannotation, 'll_os/chmod') declare(os.rename , noneannotation, 'll_os/rename') +declare(os._exit , noneannotation, 'll_os/_exit') if hasattr(os, 'getpid'): declare(os.getpid , int, 'll_os/getpid') if hasattr(os, 'link'): @@ -217,6 +225,10 @@ declare(os.symlink , noneannotation, 'll_os/symlink') if hasattr(os, 'readlink'): declare(os.readlink , str, 'll_os/readlink') +if hasattr(os, 'fork'): + declare(os.fork , int, 'll_os/fork') +if hasattr(os, 'waitpid'): + declare(os.waitpid , waitpidannotation, 'll_os/waitpid') declare(os.path.exists, bool , 'll_os_path/exists') declare(os.path.isdir, bool , 'll_os_path/isdir') Modified: pypy/dist/pypy/rpython/lltypesystem/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/module/ll_os.py Wed Aug 9 15:17:05 2006 @@ -7,6 +7,7 @@ STAT_RESULT = rtupletype.TUPLE_TYPE([lltype.Signed]*10).TO PIPE_RESULT = rtupletype.TUPLE_TYPE([lltype.Signed]*2).TO +WAITPID_RESULT = rtupletype.TUPLE_TYPE([lltype.Signed]*2).TO class Implementation(BaseOS, LLSupport): @@ -57,3 +58,10 @@ s = lltype.malloc(STR, n) ll_strcpy(s, buffer, n) return s + + def ll_waitpid_result(fd1, fd2): + tup = lltype.malloc(WAITPID_RESULT) + tup.item0 = fd1 + tup.item1 = fd2 + return tup + ll_waitpid_result = staticmethod(ll_waitpid_result) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Aug 9 15:17:05 2006 @@ -159,6 +159,19 @@ ll_readlink_into.suggested_primitive = True ll_readlink_into = staticmethod(ll_readlink_into) + def ll_os_fork(cls): + return os.fork() + ll_os_fork.suggested_primitive = True + + def ll_os_waitpid(cls, pid, options): + pid, status = os.waitpid(pid, options) + return cls.ll_waitpid_result(pid, status) + ll_os_waitpid.suggested_primitive = True + + def ll_os__exit(cls, status): + os._exit(status) + ll_os__exit.suggested_primitive = True + # ____________________________________________________________ # opendir/readdir Modified: pypy/dist/pypy/rpython/ootypesystem/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/module/ll_os.py Wed Aug 9 15:17:05 2006 @@ -12,6 +12,7 @@ STAT_RESULT = _make_tuple([ootype.Signed]*10) PIPE_RESULT = _make_tuple([ootype.Signed]*2) +WAITPID_RESULT = _make_tuple([ootype.Signed]*2) class Implementation(BaseOS, OOSupport): @@ -45,3 +46,10 @@ def ll_os_readlink(cls, path): return cls.to_rstr(os.readlink(path)) ll_os_readlink.suggested_primitive = True + + def ll_waitpid_result(fd1, fd2): + tup = ootype.new(WAITPID_RESULT) + tup.item0 = fd1 + tup.item1 = fd2 + return tup + ll_waitpid_result = staticmethod(ll_waitpid_result) Modified: pypy/dist/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/dist/pypy/translator/c/extfunc.py Wed Aug 9 15:17:05 2006 @@ -8,6 +8,7 @@ from pypy.rpython.module import ll_time, ll_math, ll_strtod from pypy.rpython.module import ll_stackless, ll_stack from pypy.rpython.lltypesystem.module.ll_os import STAT_RESULT, PIPE_RESULT +from pypy.rpython.lltypesystem.module.ll_os import WAITPID_RESULT from pypy.rpython.lltypesystem.module.ll_os import Implementation as impl from pypy.rpython.lltypesystem.module import ll_math as ll_math2 from pypy.rpython.lltypesystem.module import ll_strtod as ll_strtod2 @@ -54,6 +55,9 @@ impl.ll_os_link.im_func: 'LL_os_link', impl.ll_os_symlink.im_func: 'LL_os_symlink', impl.ll_readlink_into: 'LL_readlink_into', + impl.ll_os_fork.im_func: 'LL_os_fork', + impl.ll_os_waitpid.im_func: 'LL_os_waitpid', + impl.ll_os__exit.im_func: 'LL_os__exit', ll_time.ll_time_clock: 'LL_time_clock', ll_time.ll_time_sleep: 'LL_time_sleep', ll_time.ll_time_time: 'LL_time_time', @@ -112,6 +116,7 @@ yield ('RPyMODF_RESULT', ll_math2.MODF_RESULT) yield ('RPySTAT_RESULT', STAT_RESULT) yield ('RPyPIPE_RESULT', PIPE_RESULT) + yield ('RPyWAITPID_RESULT', WAITPID_RESULT) def predeclare_utility_functions(db, rtyper): # Common utility functions Modified: pypy/dist/pypy/translator/c/src/ll_os.h ============================================================================== --- pypy/dist/pypy/translator/c/src/ll_os.h (original) +++ pypy/dist/pypy/translator/c/src/ll_os.h Wed Aug 9 15:17:05 2006 @@ -76,6 +76,9 @@ void LL_os_link(RPyString * path1, RPyString * path2); void LL_os_symlink(RPyString * path1, RPyString * path2); long LL_readlink_into(RPyString *path, RPyString *buffer); +long LL_os_fork(void); +RPyWAITPID_RESULT* LL_os_waitpid(long pid, long options); +void LL_os__exit(long status); void LL_os_putenv(RPyString * name_eq_value); void LL_os_unsetenv(RPyString * name); RPyString* LL_os_environ(int idx); @@ -363,6 +366,31 @@ #endif +#ifdef HAVE_FORK +long LL_os_fork(void) { + int pid = fork(); + if (pid == -1) + RPYTHON_RAISE_OSERROR(errno); + return pid; +} +#endif + +#ifdef LL_NEED_OS_WAITPID +RPyWAITPID_RESULT* LL_os_waitpid(long pid, long options) { + int status; + pid = waitpid(pid, &status, options); + if (pid == -1) { + RPYTHON_RAISE_OSERROR(errno); + return NULL; + } + return ll_waitpid_result(pid, status); +} +#endif + +void LL_os__exit(long status) { + _exit((int)status); +} + #ifdef HAVE_PUTENV /* Note that this doesn't map to os.putenv, it is the name=value * version of C. See ros.py for the fake implementation. Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/c/test/test_extfunc.py Wed Aug 9 15:17:05 2006 @@ -553,6 +553,20 @@ assert os.path.islink(tmpfile2) assert not os.path.islink(tmpfile3) +if hasattr(os, 'fork'): + def test_fork(): + def does_stuff(): + pid = os.fork() + if pid == 0: # child + os._exit(4) + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + return status1 + f1 = compile(does_stuff, []) + status1 = f1() + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == 4 + # ____________________________________________________________ def _real_getenv(var): From mwh at codespeak.net Wed Aug 9 16:17:10 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 9 Aug 2006 16:17:10 +0200 (CEST) Subject: [pypy-svn] r31203 - pypy/dist/pypy/objspace/std Message-ID: <20060809141710.6247210069@code0.codespeak.net> Author: mwh Date: Wed Aug 9 16:17:08 2006 New Revision: 31203 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: record even more info: the size of the dict on seeing a non-string. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 9 16:17:08 2006 @@ -134,8 +134,10 @@ self.iterations = 0 self.listings = 0 - self.seen_non_string = 0 + self.seen_non_string_in_write = 0 self.seen_non_string_in_read_first = 0 + self.size_on_non_string_seen_in_read = -1 + self.size_on_non_string_seen_in_write = -1 self.createtime = time.time() self.lifetime = -1.0 @@ -184,9 +186,11 @@ return space.is_true(space.isinstance(w_key, space.w_str)) def _read(self, w_key): self.info.reads += 1 - if not self.info.seen_non_string and not self._is_str(w_key): - self.info.seen_non_string = True + if not self.info.seen_non_string_in_write \ + and not self.info.seen_non_string_in_read_first \ + and not self._is_str(w_key): self.info.seen_non_string_in_read_first = True + self.info.size_on_non_string_seen_in_read = len(self.content) hit = w_key in self.content if hit: self.info.hits += 1 @@ -198,17 +202,20 @@ self._read(w_key) return self.content[w_key] def setitem(self, w_key, w_value): - if not self.info.seen_non_string and not self._is_str(w_key): - self.info.seen_non_string = True + if not self.info.seen_non_string_in_write and not self._is_str(w_key): + self.info.seen_non_string_in_write = True + self.info.size_on_non_string_seen_in_write = len(self.content) self.info.setitems += 1 self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) return self def delitem(self, w_key): - if not self.info.seen_non_string and not self._is_str(w_key): + if not self.info.seen_non_string_in_write \ + and not self.info.seen_non_string_in_read_first \ + and not self._is_str(w_key): self.info.seen_non_string_in_read_first = True - self.info.seen_non_string = True + self.info.size_on_non_string_seen_in_read = len(self.content) self.info.delitems += 1 self.info.writes += 1 del self.content[w_key] @@ -266,7 +273,7 @@ os.write(fd, "\\n") ''' bodySrc = [] - for attr in _example.__dict__: + for attr in sorted(_example.__dict__): if attr == 'sig': continue bodySrc.append(tmpl%locals()) From auc at codespeak.net Wed Aug 9 17:38:20 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 9 Aug 2006 17:38:20 +0200 (CEST) Subject: [pypy-svn] r31207 - in pypy/dist/pypy/objspace: . cclp cclp/constraint test Message-ID: <20060809153820.F170E10072@code0.codespeak.net> Author: auc Date: Wed Aug 9 17:38:15 2006 New Revision: 31207 Added: pypy/dist/pypy/objspace/cclp/constraint/ pypy/dist/pypy/objspace/cclp/constraint/__init__.py pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/domain.py Modified: pypy/dist/pypy/objspace/cclp/misc.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: tell (to declare cosntraints) Added: pypy/dist/pypy/objspace/cclp/constraint/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/__init__.py Wed Aug 9 17:38:15 2006 @@ -0,0 +1 @@ +# pass Added: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Wed Aug 9 17:38:15 2006 @@ -0,0 +1,214 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter import baseobjspace, typedef, gateway +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.function import Function + +from pypy.objspace.std.listobject import W_ListObject +from pypy.objspace.std.stringobject import W_StringObject + +from pypy.objspace.constraint.computationspace import W_ComputationSpace + +from pypy.objspace.cclp.types import W_Constraint, W_CVar as W_Variable + +from pypy.objspace.std.model import StdObjSpaceMultiMethod + +from pypy.objspace.constraint.btree import BTree +from pypy.objspace.constraint.util import sort, reverse + +import operator + +all_mms = {} + + +#-- Exceptions --------------------------------------- + +class ConsistencyFailure(Exception): + """The repository is not in a consistent state""" + pass + +class DomainlessVariables(Exception): + """A constraint can't be defined on variables + without a domain""" + pass + +#-- Constraints ------------------------------------------ + +class W_AbstractConstraint(W_Constraint): + + def __init__(self, object_space, w_variables): + """variables is a list of variables which appear in the formula""" + W_Constraint.__init__(self, object_space) + assert isinstance(w_variables, W_ListObject) + assert self._space.is_true(self._space.ge(self._space.len(w_variables), + self._space.newint(1))) + self._names_to_vars = {} + for var in w_variables.wrappeditems: + self._names_to_vars[var.name_w()] = var + self._variables = w_variables.wrappeditems #unwrap once ... + + def w_affected_variables(self): + """ Return a list of all variables affected by this constraint """ + return self._space.newlist(self._variables) + + def affected_variables(self): + return self._variables + + def w_knows_var(self, w_variable): + assert isinstance(w_variable, W_Variable) + return self._space.newbool(w_variable in self._variables) + + def w_revise(self): + return self._space.newbool(self.space.revise()) + + +## def __eq__(self, other): #FIXME and parent +## if not isinstance(other, self.__class__): return False +## return self._variables == other._variables + +W_AbstractConstraint.typedef = typedef.TypeDef( + "W_AbstractConstraint", + W_Constraint.typedef, + affected_variables = interp2app(W_AbstractConstraint.w_affected_variables), + knows_var = interp2app(W_AbstractConstraint.w_knows_var), + revise = interp2app(W_AbstractConstraint.w_revise)) + + + +from pypy.module.__builtin__.compiling import eval as ev +def make_filter__List_String(object_space, w_variables, w_formula): + """NOT RPYTHON""" + assert isinstance(w_variables, W_ListObject) + assert isinstance(w_formula, W_StringObject) + items = object_space.unpackiterable(w_variables) + for it in items: + assert isinstance(it, W_Variable) + var_ids = ','.join([var.name_w() + for var in items]) + func_head = 'lambda ' + var_ids + ':' + expr = func_head + object_space.str_w(w_formula) + func_obj = ev(object_space, object_space.wrap(expr), object_space.newdict([]), + object_space.newdict([])) + assert isinstance(func_obj, Function) + return func_obj + +make_filter_mm = StdObjSpaceMultiMethod('make_filter', 2) +make_filter_mm.register(make_filter__List_String, W_ListObject, W_StringObject) +all_mms['make_filter'] = make_filter_mm + +class W_Expression(W_AbstractConstraint): + """A constraint represented as a python expression.""" + + def __init__(self, object_space, w_variables, w_formula): + """variables is a list of variables which appear in the formula + formula is a python expression that will be evaluated as a boolean""" + W_AbstractConstraint.__init__(self, object_space, w_variables) + self.formula = self._space.str_w(w_formula) + # self.filter_func is a function taking keyword arguments and returning a boolean + self.filter_func = self._space.make_filter(w_variables, w_formula) + + def test_solution(self, sol_dict): + """test a solution against this constraint + accept a mapping of variable names to value""" + args = [] + for var in self._variables: + assert isinstance(var, W_Variable) + args.append(sol_dict[var.w_name()]) + return self.filter_func(*args) + + def _init_result_cache(self): + """key = (variable,value), value = [has_success,has_failure]""" + result_cache = self._space.newdict([]) + for var in self._variables: + assert isinstance(var, W_Variable) + result_cache.content[var.w_name()] = self._space.newdict([]) + return result_cache + + def _assign_values(self): + variables = [] + kwargs = self._space.newdict([]) + for variable in self._variables: + assert isinstance(variable, W_Variable) + domain = variable.w_dom + values = domain.w_get_values() + variables.append((domain.size(), + [variable, values, self._space.newint(0), + self._space.len(values)])) + kwargs.content[variable.w_name()] = values.wrappeditems[0] + # sort variables to instanciate those with fewer possible values first + sort(variables) + res_kwargs = [] + go_on = 1 + while go_on: +# res_kwargs.append( kwargs) + yield kwargs + # try to instanciate the next variable + + for size, curr in variables: + assert isinstance(curr[0], W_Variable) + w_name = curr[0].w_name() + assert isinstance(w_name, W_StringObject) + if self._space.int_w(curr[2]) + 1 < self._space.int_w(curr[-1]): + curr[2] = self._space.add(curr[2], self._space.newint(1)) + kwargs.content[w_name] = curr[1].wrappeditems[self._space.int_w(curr[2])] + break + else: + curr[2] = self._space.newint(0) + kwargs.content[w_name] = curr[1].wrappeditems[0] + else: + # it's over + go_on = 0 +# return res_kwargs + + def revise(self): + """generic propagation algorithm for n-ary expressions""" + maybe_entailed = True + ffunc = self.filter_func + result_cache = self._init_result_cache() + for kwargs in self._assign_values(): + if maybe_entailed: + for varname, val in kwargs.content.iteritems(): + if val not in result_cache.content[varname].content: + break + else: + continue + if self._space.is_true(self._space.call(self._space.wrap(ffunc), + self._space.newlist([]), kwargs)): + for var, val in kwargs.content.items(): + result_cache.content[var].content[val] = self._space.w_True + else: + maybe_entailed = False + + try: + for varname, keep in result_cache.content.items(): + domain = self._names_to_vars[self._space.str_w(varname)].w_dom + domain.w_remove_values(self._space.newlist([val + for val in domain._values.content.keys() + if val not in keep.content])) + + except ConsistencyFailure: + raise ConsistencyFailure('Inconsistency while applying %s' % \ + repr(self)) + except KeyError: + # There are no more value in result_cache + pass + + return maybe_entailed + + + def __repr__(self): + return '<%s>' % self.formula + +W_Expression.typedef = typedef.TypeDef("W_Expression", + W_AbstractConstraint.typedef) +# revise = interp2app(W_Expression.w_revise)) + + + +def make_expression(o_space, w_variables, w_formula): + """create a new constraint of type Expression or BinaryExpression + The chosen class depends on the number of variables in the constraint""" + assert len(w_variables.wrappeditems) > 0 + return W_Expression(o_space, w_variables, w_formula) +app_make_expression = gateway.interp2app(make_expression) + Added: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Wed Aug 9 17:38:15 2006 @@ -0,0 +1,124 @@ +from pypy.interpreter.error import OperationError + +from pypy.interpreter import typedef, gateway +from pypy.interpreter.gateway import interp2app + +from pypy.objspace.std.listobject import W_ListObject, W_TupleObject + +from pypy.objspace.std.model import StdObjSpaceMultiMethod + +from pypy.objspace.cclp.types import W_AbstractDomain + +all_mms = {} + +class ConsistencyFailure(Exception): + """The repository is not in a consistent state""" + pass + + +class W_FiniteDomain(W_AbstractDomain): + """ + Variable Domain with a finite set of possible values + """ + + def __init__(self, space, w_values): + """values is a list of values in the domain + This class uses a dictionnary to make sure that there are + no duplicate values""" + W_AbstractDomain.__init__(self, space) + #XXX a pure dict used to work there (esp. in revise) + self._values = space.newdict([]) + self.set_values(w_values) + + def set_values(self, w_values): + """Objects in the value set can't be unwrapped unless we + specialize on specific types - this might need speccialization + of revise & friends + """ + for w_v in w_values.wrappeditems: + self._space.setitem(self._values, w_v, self._space.w_True) + + def w_remove_value(self, w_value): + """Remove value of domain and check for consistency""" + self._space.delitem(self._values, w_value) + self._value_removed() + + def w_remove_values(self, w_values): + """Remove values of domain and check for consistency""" + assert isinstance(w_values, W_ListObject) + self.remove_values(w_values.wrappeditems) + + def remove_values(self, values): + assert isinstance(values, list) + try: + if len(values) > 0: + for w_val in values: + del self._values.content[w_val] + self._value_removed() + except KeyError: + raise OperationError(self._space.w_RuntimeError, + self._space.wrap("attempt to remove unkown value from domain")) + + def w_size(self): + return self._space.newint(self.size()) + + def size(self): + """computes the size of a finite domain""" + return self._space.len(self._values).intval + __len__ = size + + def w_get_values(self): + """return all the values in the domain + in an indexable sequence""" + return self._space.newlist(self.get_values()) + + def get_values(self): + return [x for x in self._values.content.keys()] + + def __repr__(self): + return '' % str(self.w_get_values()) + + def __eq__(self, w_other): + if not isinstance(w_other, W_FiniteDomain): + return self._space.newbool(False) + return self._space.newbool(self._space.eq_w(self._values, w_other._values)) + + def __ne__(self, w_other): + return not self == w_other + + +# function bolted into the space to serve as constructor +def make_fd(space, w_values): + assert isinstance(w_values, W_ListObject) + return space.wrap(W_FiniteDomain(space, w_values)) +app_make_fd = gateway.interp2app(make_fd) + + +def intersection(space, w_fd1, w_fd2): + assert isinstance(w_fd1, W_FiniteDomain) + assert isinstance(w_fd2, W_FiniteDomain) + return space.intersection(w_fd1, w_fd2) +app_intersection = gateway.interp2app(intersection) + + +def intersection__FiniteDomain_FiniteDomain(space, w_fd1, w_fd2): + w_v1 = w_fd1._values.content + res = [w_v for w_v in w_fd2._values.content + if w_v in w_v1] + return make_fd(space, space.newlist(res)) + +intersection_mm = StdObjSpaceMultiMethod('intersection', 2) +intersection_mm.register(intersection__FiniteDomain_FiniteDomain, + W_FiniteDomain, W_FiniteDomain) +all_mms['intersection'] = intersection_mm + +W_FiniteDomain.typedef = typedef.TypeDef( + "W_FiniteDomain", + W_AbstractDomain.typedef, + remove_value = interp2app(W_FiniteDomain.w_remove_value), + remove_values = interp2app(W_FiniteDomain.w_remove_values), + get_values = interp2app(W_FiniteDomain.w_get_values), + __eq__ = interp2app(W_FiniteDomain.__eq__), + __ne__ = interp2app(W_FiniteDomain.__ne__), + size = interp2app(W_FiniteDomain.w_size)) + Modified: pypy/dist/pypy/objspace/cclp/misc.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/misc.py (original) +++ pypy/dist/pypy/objspace/cclp/misc.py Wed Aug 9 17:38:15 2006 @@ -29,4 +29,3 @@ NO_DEBUG_INFO[0] = not NO_DEBUG_INFO[0] app_switch_debug_info = gateway.interp2app(switch_debug_info) - Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Wed Aug 9 17:38:15 2006 @@ -76,7 +76,10 @@ assert isinstance(thread, ClonableCoroutine) w(".. REMOVING", str(id(thread))) assert thread not in self._blocked - del self._traced[thread] + try: + del self._traced[thread] + except KeyError: + w(".. removing non-traced thread") l = thread._prev r = thread._next l._next = r Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Wed Aug 9 17:38:15 2006 @@ -5,7 +5,7 @@ from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.cclp.misc import ClonableCoroutine, w -from pypy.objspace.cclp.thunk import CSpaceThunk +from pypy.objspace.cclp.thunk import CSpaceThunk, PropagatorThunk from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.variable import newvar @@ -42,6 +42,13 @@ app_choose = gateway.interp2app(choose) +from pypy.objspace.cclp.constraint import constraint + +def tell(space, w_constraint): + assert isinstance(w_constraint, constraint.W_AbstractConstraint) + ClonableCoroutine.w_getcurrent(space)._cspace.tell(w_constraint) +app_tell = gateway.interp2app(tell) + class W_CSpace(baseobjspace.Wrappable): @@ -54,8 +61,8 @@ # choice mgmt self._choice = newvar(space) self._committed = newvar(space) - # constraint store ... - + # merging + self._merged = newvar(space) def w_ask(self): scheduler[0].wait_stable(self) @@ -82,24 +89,23 @@ def tell(self, w_constraint): - pass - -W_CSpace.typedef = typedef.TypeDef("W_CSpace", - ask = gateway.interp2app(W_CSpace.w_ask), - commit = gateway.interp2app(W_CSpace.w_commit)) - + space = self.space + w_coro = ClonableCoroutine(space) + thunk = PropagatorThunk(space, w_constraint, w_coro, self._merged) + w_coro.bind(thunk) + if not we_are_translated(): + w("PROPAGATOR, thread", str(id(w_coro))) + w_coro._cspace = self + scheduler[0].add_new_thread(w_coro) + scheduler[0].schedule() + def w_merge(self): + self.space.bind(self._merged, self.space.w_True) -## def clone(self): -## if self.is_top_level(): -## raise OperationError(self.space.w_RuntimeError, -## self.space.wrap("Clone"+forbidden_boilerplate)) -## new = CSpace(self.distributor.clone(), parent=self) -## new.distributor.cspace = new -## for thread in self.threads: -## tclone = thread.clone() -## tclone.cspace = new -## new.threads[tclone] = True +W_CSpace.typedef = typedef.TypeDef("W_CSpace", + ask = gateway.interp2app(W_CSpace.w_ask), + commit = gateway.interp2app(W_CSpace.w_commit), + merge = gateway.interp2app(W_CSpace.w_merge)) Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 9 17:38:15 2006 @@ -1,10 +1,23 @@ from pypy.module._stackless.coroutine import _AppThunk -from pypy.objspace.cclp.misc import w +from pypy.objspace.cclp.misc import w from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue + +def logic_args(args): + "returns logic vars found in unpacked normalized args" + assert isinstance(args, tuple) + pos = args[0] + kwa = args[1] + pos_l = [arg for arg in pos + if isinstance(arg, W_Var)] + kwa_l = [arg for arg in kwa.keys() + if isinstance(arg, W_Var)] + return pos_l + kwa_l + #-- Thunk ----------------------------------------- + class ProcedureThunk(_AppThunk): def __init__(self, space, w_callable, args, coro): _AppThunk.__init__(self, space, coro.costate, w_callable, args) @@ -82,16 +95,30 @@ scheduler[0].schedule() +from pypy.interpreter.argument import Arguments +from pypy.module._stackless.interp_coroutine import AbstractThunk +class PropagatorThunk(AbstractThunk): + def __init__(self, space, w_constraint, coro, Merged): + self.space = space + self.coro = coro + self.const = w_constraint + self.Merged = Merged -def logic_args(args): - "returns logic vars found in unpacked normalized args" - assert isinstance(args, tuple) - pos = args[0] - kwa = args[1] - pos_l = [arg for arg in pos - if isinstance(arg, W_Var)] - kwa_l = [arg for arg in kwa.keys() - if isinstance(arg, W_Var)] - return pos_l + kwa_l + def call(self): + try: + while 1: + entailed = self.const.revise() + if entailed: + break + Obs = W_Var(self.space) + self.space.entail(self.Merged, Obs) + for Sync in [var.w_dom.give_synchronizer() + for var in self.const._variables]: + self.space.entail(Sync, Obs) + self.space.wait(Obs) + finally: + self.coro._dead = True + scheduler[0].remove_thread(self.coro) + scheduler[0].schedule() Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Wed Aug 9 17:38:15 2006 @@ -1,7 +1,6 @@ from pypy.interpreter import baseobjspace, gateway, typedef from pypy.objspace.cclp.misc import w, ClonableCoroutine -#from pypy.objspace.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root @@ -32,15 +31,19 @@ class W_CVar(W_Var): - def __init__(w_self, space, w_dom): #, w_name): - assert isinstance(w_dom, W_FiniteDomain) + def __init__(w_self, space, w_dom, w_name): + assert isinstance(w_dom, W_AbstractDomain) W_Var.__init__(w_self, space) w_self.w_dom = w_dom - #w_self.name = space.str_w(w_name) + w_self.name = space.str_w(w_name) + w_self.w_nam = w_name def name_w(w_self): return w_self.name + def w_name(w_self): + return w_self.w_nam + def domain_of(space, w_v): assert isinstance(w_v, W_CVar) return w_v.w_dom @@ -57,12 +60,44 @@ #-- Constraint --------------------------------------------- -## class W_Constraint(baseobjspace.Wrappable): -## def __init__(self, object_space): -## self._space = object_space +class W_Constraint(baseobjspace.Wrappable): + def __init__(self, object_space): + self._space = object_space + +W_Constraint.typedef = typedef.TypeDef( + "W_Constraint") + +class W_AbstractDomain(baseobjspace.Wrappable): + """Implements the functionnality related to the changed flag. + Can be used as a starting point for concrete domains""" + + def __init__(self, space): + self._space = space + self.__changed = W_Var(self._space) + + def clear_change(self): + #XXX called after revise ? + assert self._space.is_true(self._space.is_bound(self.__changed)) + self.__changed = W_Var(self._space) + + def give_synchronizer(self): + return self.__changed + + def _value_removed(self): + """The implementation of remove_value should call this method""" + self._space.bind(self.__changed, self._space.newbool(True)) + self.clear_change() + + if self.size() == 0: # self._space.eq_w(self.w_size(), self._space.newint(0)): + raise OperationError(self._space.w_RuntimeError, + self._space.wrap('ConsistencyFailure')) + + def w__del__(self): + self._space.bind(self.__changed, self._space.newbool(False)) + +W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain") +## has_changed = interp2app(W_AbstractDomain.w_has_changed)) -## W_Constraint.typedef = typedef.TypeDef( -## "W_Constraint") #-- Misc --------------------------------------------------- Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Wed Aug 9 17:38:15 2006 @@ -9,7 +9,7 @@ from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import deref, W_Var, W_CVar, W_Future, W_FailedValue -from pypy.objspace.constraint.domain import W_FiniteDomain +from pypy.objspace.cclp.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root all_mms = {} @@ -20,11 +20,11 @@ return w_v app_newvar = gateway.interp2app(newvar) -def domain(space, w_values):#, w_name): +def domain(space, w_values, w_name): assert isinstance(w_values, W_ListObject) - #assert isinstance(w_name, W_StringObject) + assert isinstance(w_name, W_StringObject) w_dom = W_FiniteDomain(space, w_values) - w_var = W_CVar(space, w_dom)#, w_name) + w_var = W_CVar(space, w_dom, w_name) w("CVAR", str(w_var)) return w_var app_domain = gateway.interp2app(domain) @@ -202,7 +202,7 @@ def bind__CVar_Root(space, w_cvar, w_obj): #XXX we should (want to) be able to test membership # in a wrapped against wrappeds into a non-wrapped dict - if [True for elt in w_cvar.w_dom._values + if [True for elt in w_cvar.w_dom._values.content if space.is_true(space.eq(w_obj, elt))]: return bind__Var_Root(space, w_cvar, w_obj) raise_unification_failure(space, "value not in variable domain") @@ -317,7 +317,7 @@ def _assign(space, w_var, w_val): assert isinstance(w_var, W_Var) if isinstance(w_var, W_CVar): - if not w_val in w_var.w_dom._values: + if not w_val in w_var.w_dom._values.content: raise_unification_failure(space, "assignment out of domain") w_var.w_bound_to = w_val Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Wed Aug 9 17:38:15 2006 @@ -23,7 +23,7 @@ #-- COMP. SPACE -------------------------------------------- -from pypy.objspace.cclp.space import app_newspace, app_choose, W_CSpace #app_tell +from pypy.objspace.cclp.space import app_newspace, app_choose, W_CSpace, app_tell #-- VARIABLE ------------------------------------------------ @@ -37,8 +37,8 @@ #-- CONSTRAINTS ---------------------------------------------- -## #------ domains ------------------ -from pypy.objspace.constraint import domain +## #------ domains ------------------ +from pypy.objspace.cclp.constraint import domain all_mms.update(domain.all_mms) W_FiniteDomain = domain.W_FiniteDomain @@ -50,8 +50,8 @@ ## W_ComputationSpace = computationspace.W_ComputationSpace ## # ---- constraints ---------------- -## from pypy.objspace.constraint import constraint -## all_mms.update(constraint.all_mms) +from pypy.objspace.cclp.constraint import constraint +all_mms.update(constraint.all_mms) ## #----- distributors --------------- ## from pypy.objspace.constraint import distributor @@ -246,8 +246,8 @@ space.setitem(space.builtin.w_dict, space.wrap('intersection'), space.wrap(domain.app_intersection)) ## #-- constraints ---- -## space.setitem(space.builtin.w_dict, space.wrap('make_expression'), -## space.wrap(constraint.app_make_expression)) + space.setitem(space.builtin.w_dict, space.wrap('make_expression'), + space.wrap(constraint.app_make_expression)) ## space.setitem(space.builtin.w_dict, space.wrap('AllDistinct'), ## space.wrap(constraint.app_make_alldistinct)) ## #-- distributor -- @@ -281,8 +281,8 @@ space.wrap(app_newspace)) space.setitem(space.builtin.w_dict, space.wrap('choose'), space.wrap(app_choose)) -## space.setitem(space.builtin.w_dict, space.wrap('tell'), -## space.wrap(app_tell)) + space.setitem(space.builtin.w_dict, space.wrap('tell'), + space.wrap(app_tell)) #-- misc ----- space.setitem(space.builtin.w_dict, space.wrap('interp_id'), Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 9 17:38:15 2006 @@ -665,7 +665,7 @@ def test_cvar(self): - d = domain([1, 2, 4]) + d = domain([1, 2, 4], '') raises(UnificationError, bind, d, 42) bind(d, 2) @@ -675,31 +675,31 @@ pass f = Foo() - d = domain([Foo(), f, Foo()]) + d = domain([Foo(), f, Foo()], '') raises(UnificationError, bind, d, Foo()) bind(d, f) assert d == f - d1 = domain([1, 2, 3]) - d2 = domain([2, 3, 4]) - d3 = domain([5, 6]) + d1 = domain([1, 2, 3], '') + d2 = domain([2, 3, 4], '') + d3 = domain([5, 6], '') raises(UnificationError, unify, d1, d3) unify(d1, d2) assert alias_of(d1, d2) assert domain_of(d1) == domain_of(d2) == FiniteDomain([2, 3]) - d1 = domain([1, 2, 3]) - d4 = domain([3, 4]) + d1 = domain([1, 2, 3], '') + d4 = domain([3, 4], '') unify(d1, d4) assert d1 == d4 == 3 - d1 = domain([1, 2]) + d1 = domain([1, 2], '') x = newvar() unify(d1, x) assert alias_of(x, d1) raises(UnificationError, unify, x, 42) - d1 = domain([1, 2]) + d1 = domain([1, 2], '') x = newvar() unify(d1, x) assert alias_of(x, d1) @@ -778,26 +778,22 @@ def test_tell(self): - skip("not finished ...") - def problem(): X, Y = domain([1, 2], 'X'), domain([1, 2, 3], 'Y') tell(make_expression([X, Y], 'X + Y > 4')) - def solve(spc, X): while 1: status = spc.ask() if status == 1: unify(X, status) break + spc.merge() - switch_debug_info() s = newspace(problem) Finished = newvar() stacklet(solve, s, Finished) - wait(Finished) - assert domain_of(X) == FiniteDomain([2]) - assert domain_of(Y) == FiniteDomain([3]) + schedule() + assert len(sched_all()['threads']) == 1 From auc at codespeak.net Thu Aug 10 10:20:05 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 10 Aug 2006 10:20:05 +0200 (CEST) Subject: [pypy-svn] r31221 - in pypy/dist/pypy/objspace: . cclp cclp/constraint cclp/constraint/test constraint/test Message-ID: <20060810082005.1AB951005A@code0.codespeak.net> Author: auc Date: Thu Aug 10 10:20:01 2006 New Revision: 31221 Added: pypy/dist/pypy/objspace/cclp/constraint/test/ pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/constraint/test/test_fd.py pypy/dist/pypy/objspace/logic.py Log: updated tests for domains & constraints Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Thu Aug 10 10:20:01 2006 @@ -59,7 +59,8 @@ return self._space.newbool(w_variable in self._variables) def w_revise(self): - return self._space.newbool(self.space.revise()) + return self._space.newbool(self.revise()) + ## def __eq__(self, other): #FIXME and parent @@ -182,9 +183,9 @@ try: for varname, keep in result_cache.content.items(): domain = self._names_to_vars[self._space.str_w(varname)].w_dom - domain.w_remove_values(self._space.newlist([val - for val in domain._values.content.keys() - if val not in keep.content])) + domain.remove_values([val + for val in domain._values.content.keys() + if val not in keep.content]) except ConsistencyFailure: raise ConsistencyFailure('Inconsistency while applying %s' % \ @@ -200,8 +201,8 @@ return '<%s>' % self.formula W_Expression.typedef = typedef.TypeDef("W_Expression", - W_AbstractConstraint.typedef) -# revise = interp2app(W_Expression.w_revise)) + W_AbstractConstraint.typedef, + revise = interp2app(W_Expression.w_revise)) @@ -212,3 +213,71 @@ return W_Expression(o_space, w_variables, w_formula) app_make_expression = gateway.interp2app(make_expression) + +class W_AllDistinct(W_AbstractConstraint): + """Contraint: all values must be distinct""" + + def __init__(self, object_space, w_variables): + W_AbstractConstraint.__init__(self, object_space, w_variables) + # worst case complexity + #self.__cost = len(w_variables.wrappeditems) * (len(w_variables.wrappeditems) - 1) / 2 + + def revise(self): + _spc = self._space + + ord_vars = BTree() + for variable in self._variables: + ord_vars.add((variable.w_dom).size(), + (variable, variable.w_dom)) + + variables = ord_vars.values() + +## variables = [(_spc.int_w(w_cs.w_dom(variable).w_size()), +## variable, w_cs.w_dom(variable)) +## for variable in self._variables] +## variables.sort() + + # if a domain has a size of 1, + # then the value must be removed from the other domains + for var, dom in variables: + if dom.size() == 1: + #print "AllDistinct removes values" + for _var, _dom in variables: + if not _var._same_as(var): + try: + _dom.remove_value(dom.get_values()[0]) + except KeyError, e: + # we ignore errors caused by the removal of + # non existing values + pass + + # if there are less values than variables, the constraint fails + values = {} + for var, dom in variables: + for val in dom.w_get_values().wrappeditems: + values[val] = 0 + + if len(values) < len(variables): + #print "AllDistinct failed" + raise OperationError(_spc.w_RuntimeError, + _spc.wrap("ConsistencyFailure")) + + # the constraint is entailed if all domains have a size of 1 + for _var, dom in variables: + if not dom.size() == 1: + return False + + # Question : did we *really* completely check + # our own alldistinctness predicate ? + #print "All distinct entailed" + return True + +W_AllDistinct.typedef = typedef.TypeDef( + "W_AllDistinct", W_AbstractConstraint.typedef, + revise = interp2app(W_AllDistinct.w_revise)) + +# function bolted into the space to serve as constructor +def make_alldistinct(object_space, w_variables): + assert len(w_variables.wrappeditems) > 0 + return object_space.wrap(W_AllDistinct(object_space, w_variables)) +app_make_alldistinct = gateway.interp2app(make_alldistinct) Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Thu Aug 10 10:20:01 2006 @@ -38,27 +38,27 @@ for w_v in w_values.wrappeditems: self._space.setitem(self._values, w_v, self._space.w_True) - def w_remove_value(self, w_value): + def remove_value(self, w_value): """Remove value of domain and check for consistency""" - self._space.delitem(self._values, w_value) + del self._values.content[w_value] self._value_removed() def w_remove_values(self, w_values): """Remove values of domain and check for consistency""" assert isinstance(w_values, W_ListObject) - self.remove_values(w_values.wrappeditems) - - def remove_values(self, values): - assert isinstance(values, list) try: - if len(values) > 0: - for w_val in values: - del self._values.content[w_val] - self._value_removed() + self.remove_values(w_values.wrappeditems) except KeyError: raise OperationError(self._space.w_RuntimeError, self._space.wrap("attempt to remove unkown value from domain")) + def remove_values(self, values): + assert isinstance(values, list) + if len(values) > 0: + for w_val in values: + del self._values.content[w_val] + self._value_removed() + def w_size(self): return self._space.newint(self.size()) @@ -115,7 +115,7 @@ W_FiniteDomain.typedef = typedef.TypeDef( "W_FiniteDomain", W_AbstractDomain.typedef, - remove_value = interp2app(W_FiniteDomain.w_remove_value), + remove_value = interp2app(W_FiniteDomain.remove_value), remove_values = interp2app(W_FiniteDomain.w_remove_values), get_values = interp2app(W_FiniteDomain.w_get_values), __eq__ = interp2app(W_FiniteDomain.__eq__), Added: pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py Thu Aug 10 10:20:01 2006 @@ -0,0 +1,62 @@ +from pypy.conftest import gettestobjspace +from py.test import skip + +class AppTest_AllDistinct(object): + + def setup_class(cls): + cls.space = gettestobjspace('logic', usemodules=('_stackless', )) + + + def test_instantiate(self): + v1 = domain([1, 2], 'v1') + v2 = domain([1, 2], 'v2') + cstr = all_diff([v1, v2]) + variables = cstr.affected_variables() + assert variables is not None + assert len(variables) == 2 + assert cstr.knows_var(v1) + assert cstr.knows_var(v2) + + def test_revise(self): + v1 = domain([1, 2], 'v1') + v2 = domain([1, 2], 'v2') + cstr = all_diff([v1, v2]) + assert cstr.revise() == False # not entailed + + v3 = domain([1], 'v3') + v4 = domain([2], 'v4') + cstr = all_diff([v3, v4]) + assert cstr.revise() == True # entailed + + v5 = domain([1], 'v5') + v6 = domain([1], 'v6') + cstr = all_diff([v5, v6]) + raises(Exception, cstr.revise) + + v7 = domain([1, 2], 'v7') + v8 = domain([1, 2], 'v8') + cstr = all_diff([v2, v7, v8]) + raises(Exception, cstr.revise) + + v9 = domain([1], 'v9') + v10= domain([1, 2], 'v10') + cstr = all_diff([v9, v10]) + assert cstr.revise() == True + assert domain_of(v10).get_values() == [2] + +class AppTest_Expression(object): + + def setup_class(cls): + cls.space = gettestobjspace('logic', usemodules=('_stackless', )) + + def test_instantiate(self): + v1 = domain([1, 2], 'v1') + cstr = make_expression([v1], '2*v1==2') + assert str(cstr).startswith('' % prettyfy_id(id(w_self)) return '<%s@%s>' % (w_self.w_bound_to, prettyfy_id(id(w_self))) + + def _same_as(w_self, w_var): + assert isinstance(w_var, W_Var) + return w_self is w_var __str__ = __repr__ @@ -89,14 +94,14 @@ self.clear_change() if self.size() == 0: # self._space.eq_w(self.w_size(), self._space.newint(0)): - raise OperationError(self._space.w_RuntimeError, - self._space.wrap('ConsistencyFailure')) + raise OperationError(self._space.w_RuntimeError, + self._space.wrap('ConsistencyFailure')) def w__del__(self): self._space.bind(self.__changed, self._space.newbool(False)) -W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain") -## has_changed = interp2app(W_AbstractDomain.w_has_changed)) +W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain", + give_synchronizer = gateway.interp2app(W_AbstractDomain.give_synchronizer)) Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Thu Aug 10 10:20:01 2006 @@ -137,7 +137,6 @@ return space.newbool(False) app_alias_of = gateway.interp2app(alias_of) - #-- HELPERS ---------------------- def get_ring_tail(space, w_start): Modified: pypy/dist/pypy/objspace/constraint/test/test_fd.py ============================================================================== --- pypy/dist/pypy/objspace/constraint/test/test_fd.py (original) +++ pypy/dist/pypy/objspace/constraint/test/test_fd.py Thu Aug 10 10:20:01 2006 @@ -2,6 +2,7 @@ from py.test import skip class AppTest_FiniteDomain(object): + skip("outdated/obsolete") def setup_class(cls): cls.space = gettestobjspace('logic', usemodules=('_stackless', )) Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Thu Aug 10 10:20:01 2006 @@ -43,12 +43,6 @@ W_FiniteDomain = domain.W_FiniteDomain -## #-------- computationspace -------- -## from pypy.objspace.constraint import computationspace -## all_mms.update(computationspace.all_mms) - -## W_ComputationSpace = computationspace.W_ComputationSpace - ## # ---- constraints ---------------- from pypy.objspace.cclp.constraint import constraint all_mms.update(constraint.all_mms) @@ -237,19 +231,16 @@ space.wrap(app_entail)) space.setitem(space.builtin.w_dict, space.wrap('unify'), space.wrap(app_unify)) -## #-- comp space --- -## space.setitem(space.builtin.w_dict, space.wrap('newspace'), -## space.wrap(computationspace.app_newspace)) -## #-- domain ------- + #-- domain ------- space.setitem(space.builtin.w_dict, space.wrap('FiniteDomain'), space.wrap(domain.app_make_fd)) space.setitem(space.builtin.w_dict, space.wrap('intersection'), space.wrap(domain.app_intersection)) -## #-- constraints ---- + #-- constraints ---- space.setitem(space.builtin.w_dict, space.wrap('make_expression'), space.wrap(constraint.app_make_expression)) -## space.setitem(space.builtin.w_dict, space.wrap('AllDistinct'), -## space.wrap(constraint.app_make_alldistinct)) + space.setitem(space.builtin.w_dict, space.wrap('all_diff'), + space.wrap(constraint.app_make_alldistinct)) ## #-- distributor -- ## space.setitem(space.builtin.w_dict, space.wrap('NaiveDistributor'), ## space.wrap(distributor.app_make_naive_distributor)) From arigo at codespeak.net Thu Aug 10 12:30:12 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Aug 2006 12:30:12 +0200 (CEST) Subject: [pypy-svn] r31222 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060810103012.9970E10034@code0.codespeak.net> Author: arigo Date: Thu Aug 10 12:30:11 2006 New Revision: 31222 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: My dates. Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Thu Aug 10 12:30:11 2006 @@ -11,7 +11,7 @@ Anders Chrigstroem 20-28/8 on-campus Samuele Pedroni 20-28/8 on-campus Michael Hudson 20-27 Brookfield Hall, Castletroy -Armin Rigo ? ? +Armin Rigo 21-28 ? Holger Krekel 18-28 v/dublin on-campus (faxed form) Maciej Fijalkowski 21-28?/8 Brookfield Hall, Castletroy Bea During 20-25/8 ? From mwh at codespeak.net Thu Aug 10 13:54:29 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 10 Aug 2006 13:54:29 +0200 (CEST) Subject: [pypy-svn] r31227 - pypy/dist/pypy/objspace/std Message-ID: <20060810115429.33D2F1005A@code0.codespeak.net> Author: mwh Date: Thu Aug 10 13:54:28 2006 New Revision: 31227 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: record even more information, tweaks. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Thu Aug 10 13:54:28 2006 @@ -121,6 +121,8 @@ class DictInfo(object): def __init__(self): + self.id = len(_dict_infos) + self.getitems = 0; self.setitems = 0; self.delitems = 0 self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 @@ -266,12 +268,7 @@ _example = DictInfo() del _dict_infos[-1] - tmpl = ''' - os.write(fd, "%(attr)s") - os.write(fd, ": ") - os.write(fd, str(info.%(attr)s)) - os.write(fd, "\\n") - ''' + tmpl = 'os.write(fd, "%(attr)s" + ": " + str(info.%(attr)s) + "\\n")' bodySrc = [] for attr in sorted(_example.__dict__): if attr == 'sig': @@ -279,15 +276,18 @@ bodySrc.append(tmpl%locals()) exec py.code.Source(''' def _report_one(fd, info): + os.write(fd, "_address" + ": " + str(id(info)) + "\\n") %s - '''%''.join(bodySrc)).compile() + '''%'\n '.join(bodySrc)).compile() def report(): + os.write(2, "starting to report!\n") fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY, 0644) for info in _dict_infos: os.write(fd, '------------------\n') _report_one(fd, info) os.close(fd) + os.write(2, "reporting done!\n") def reportDictInfo(): d = {} From arigo at codespeak.net Thu Aug 10 14:37:48 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Aug 2006 14:37:48 +0200 (CEST) Subject: [pypy-svn] r31229 - pypy/dist/pypy/tool Message-ID: <20060810123748.7CEA410068@code0.codespeak.net> Author: arigo Date: Thu Aug 10 14:37:47 2006 New Revision: 31229 Modified: pypy/dist/pypy/tool/tls.py Log: test Modified: pypy/dist/pypy/tool/tls.py ============================================================================== --- pypy/dist/pypy/tool/tls.py (original) +++ pypy/dist/pypy/tool/tls.py Thu Aug 10 14:37:47 2006 @@ -1,3 +1,4 @@ + ##"""Thread-local storage.""" ## ##try: From auc at codespeak.net Thu Aug 10 15:00:47 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 10 Aug 2006 15:00:47 +0200 (CEST) Subject: [pypy-svn] r31230 - in pypy/dist/pypy/objspace/cclp/constraint: . test Message-ID: <20060810130047.12A2510071@code0.codespeak.net> Author: auc Date: Thu Aug 10 15:00:43 2006 New Revision: 31230 Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py Log: removed one yield Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Thu Aug 10 15:00:43 2006 @@ -16,7 +16,6 @@ from pypy.objspace.constraint.btree import BTree from pypy.objspace.constraint.util import sort, reverse -import operator all_mms = {} @@ -62,11 +61,6 @@ return self._space.newbool(self.revise()) - -## def __eq__(self, other): #FIXME and parent -## if not isinstance(other, self.__class__): return False -## return self._variables == other._variables - W_AbstractConstraint.typedef = typedef.TypeDef( "W_AbstractConstraint", W_Constraint.typedef, @@ -131,42 +125,48 @@ for variable in self._variables: assert isinstance(variable, W_Variable) domain = variable.w_dom - values = domain.w_get_values() - variables.append((domain.size(), - [variable, values, self._space.newint(0), - self._space.len(values)])) - kwargs.content[variable.w_name()] = values.wrappeditems[0] + values = domain.get_values() + variables.append((domain.size(), [variable.w_name(), values, 0, len(values)])) + #kwargs.content[variable.w_name()] = values[0] # sort variables to instanciate those with fewer possible values first sort(variables) - res_kwargs = [] - go_on = 1 - while go_on: -# res_kwargs.append( kwargs) - yield kwargs - # try to instanciate the next variable - - for size, curr in variables: - assert isinstance(curr[0], W_Variable) - w_name = curr[0].w_name() - assert isinstance(w_name, W_StringObject) - if self._space.int_w(curr[2]) + 1 < self._space.int_w(curr[-1]): - curr[2] = self._space.add(curr[2], self._space.newint(1)) - kwargs.content[w_name] = curr[1].wrappeditems[self._space.int_w(curr[2])] - break - else: - curr[2] = self._space.newint(0) - kwargs.content[w_name] = curr[1].wrappeditems[0] + self._assign_values_state = variables + return kwargs + + def _next_value(self, kwargs): + + # try to instanciate the next variable + variables = self._assign_values_state + + for _, curr in variables: + w_name = curr[0] + dom_values = curr[1] + dom_index = curr[2] + dom_len = curr[3] + if dom_index < dom_len: + kwargs.content[w_name] = dom_values[curr[2]] + curr[2] = dom_index + 1 + break else: - # it's over - go_on = 0 -# return res_kwargs + curr[2] = 0 + kwargs.content[w_name] = dom_values[0] + else: + # it's over + raise StopIteration + return kwargs def revise(self): """generic propagation algorithm for n-ary expressions""" maybe_entailed = True ffunc = self.filter_func result_cache = self._init_result_cache() - for kwargs in self._assign_values(): + + kwargs = self._assign_values() + while 1: + try: + kwargs = self._next_value(kwargs) + except StopIteration: + break if maybe_entailed: for varname, val in kwargs.content.iteritems(): if val not in result_cache.content[varname].content: @@ -186,7 +186,6 @@ domain.remove_values([val for val in domain._values.content.keys() if val not in keep.content]) - except ConsistencyFailure: raise ConsistencyFailure('Inconsistency while applying %s' % \ repr(self)) Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Thu Aug 10 15:00:43 2006 @@ -1,6 +1,6 @@ from pypy.interpreter.error import OperationError -from pypy.interpreter import typedef, gateway +from pypy.interpreter import typedef, gateway, baseobjspace from pypy.interpreter.gateway import interp2app from pypy.objspace.std.listobject import W_ListObject, W_TupleObject @@ -27,6 +27,7 @@ no duplicate values""" W_AbstractDomain.__init__(self, space) #XXX a pure dict used to work there (esp. in revise) + assert isinstance(w_values, W_ListObject) self._values = space.newdict([]) self.set_values(w_values) @@ -40,6 +41,7 @@ def remove_value(self, w_value): """Remove value of domain and check for consistency""" + assert isinstance(w_value, baseobjspace.W_Root) del self._values.content[w_value] self._value_removed() Modified: pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py Thu Aug 10 15:00:43 2006 @@ -54,7 +54,7 @@ cstr = make_expression([v1], '2*v1==2') assert str(cstr).startswith(' Author: auc Date: Thu Aug 10 16:08:15 2006 New Revision: 31231 Added: pypy/dist/pypy/objspace/cclp/constraint/variable.py pypy/dist/pypy/objspace/cclp/interp_var.py Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: split files, cleanup Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Thu Aug 10 16:08:15 2006 @@ -7,7 +7,8 @@ from pypy.objspace.std.model import StdObjSpaceMultiMethod -from pypy.objspace.cclp.types import W_AbstractDomain +from pypy.objspace.cclp.types import W_AbstractDomain, W_Var +from pypy.objspace.cclp.interp_var import interp_bind all_mms = {} @@ -31,6 +32,27 @@ self._values = space.newdict([]) self.set_values(w_values) + + def clear_change(self): + assert not isinstance(self._changed.w_bound_to, W_Var) + self._changed = W_Var(self._space) + + def give_synchronizer(self): + return self._changed + + def _value_removed(self): + """The implementation of remove_value should call this method""" + interp_bind(self._space, self._changed, True) + self.clear_change() + + if self.size() == 0: + raise OperationError(self._space.w_RuntimeError, + self._space.wrap('ConsistencyFailure')) + +## def w__del__(self): +## self._space.bind(self._changed, self._space.newbool(False)) + + def set_values(self, w_values): """Objects in the value set can't be unwrapped unless we specialize on specific types - this might need speccialization @@ -89,6 +111,8 @@ return not self == w_other + + # function bolted into the space to serve as constructor def make_fd(space, w_values): assert isinstance(w_values, W_ListObject) Modified: pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py Thu Aug 10 16:08:15 2006 @@ -13,13 +13,9 @@ def test_remove_value(self): fd = FiniteDomain([1, 2, 3]) - Sync = fd.give_synchronizer() - def foo(fd): - fd.remove_value(2) - assert fd.size() == 2 - assert set(fd.get_values()) == set([1, 3]) - stacklet(foo, fd) - assert Sync == True + fd.remove_value(2) + assert fd.size() == 2 + assert set(fd.get_values()) == set([1, 3]) def test_remove_all_values(self): fd = FiniteDomain([3]) @@ -28,21 +24,14 @@ def test_remove_values(self): fd = FiniteDomain([1, 2, 3]) - Sync = fd.give_synchronizer() - def foo(fd): - fd.remove_values([1, 2]) - assert fd.size() == 1 - assert set(fd.get_values()) == set([3,]) - stacklet(foo, fd) - assert Sync == True + fd.remove_values([1, 2]) + assert fd.size() == 1 + assert set(fd.get_values()) == set([3,]) def test_remove_values_empty_list(self): fd = FiniteDomain([1, 2, 3]) - Sync = fd.give_synchronizer() - assert is_free(Sync) fd.remove_values([]) assert fd.size() == 3 - assert is_free(Sync) def test_intersection(self): """not used for now""" Added: pypy/dist/pypy/objspace/cclp/constraint/variable.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/variable.py Thu Aug 10 16:08:15 2006 @@ -0,0 +1,53 @@ +from pypy.interpreter import gateway, baseobjspace +from pypy.objspace.std.listobject import W_ListObject +from pypy.objspace.std.stringobject import W_StringObject + +from pypy.objspace.cclp.constraint.domain import W_FiniteDomain + +from pypy.objspace.cclp.types import deref, W_Var, W_CVar +from pypy.objspace.cclp.variable import bind_mm, raise_unification_failure, _alias, \ + _assign_aliases, bind__Var_Root, bind__Var_Var +from pypy.objspace.cclp.misc import w + +W_Root = baseobjspace.W_Root + +def domain(space, w_values, w_name): + assert isinstance(w_values, W_ListObject) + assert isinstance(w_name, W_StringObject) + w_dom = W_FiniteDomain(space, w_values) + w_var = W_CVar(space, w_dom, w_name) + w("CVAR", str(w_var)) + return w_var +app_domain = gateway.interp2app(domain) + + +def bind__CVar_Root(space, w_cvar, w_obj): + #XXX we should (want to) be able to test membership + # in a wrapped against wrappeds into a non-wrapped dict + if [True for elt in w_cvar.w_dom._values.content + if space.is_true(space.eq(w_obj, elt))]: + return bind__Var_Root(space, w_cvar, w_obj) + raise_unification_failure(space, "value not in variable domain") + +def bind__CVar_CVar(space, w_cvar1, w_cvar2): + w_inter_dom = space.intersection(w_cvar1.w_dom, w_cvar2.w_dom) + if w_inter_dom.__len__() > 0: + if w_inter_dom.__len__() == 1: + w_value = w_inter_dom.get_values()[0] + _assign_aliases(space, w_cvar1, w_value) + _assign_aliases(space, w_cvar2, w_value) + else: + w_cvar1.w_dom = w_cvar2.w_dom = w_inter_dom + _alias(space, w_cvar1, w_cvar2) + else: + raise_unification_failure(space, "incompatible domains") + +def bind__CVar_Var(space, w_cvar, w_var): + if space.is_true(space.is_bound(w_var)): + return bind__CVar_Root(space, w_cvar, w_var) + return bind__Var_Var(space, w_cvar, w_var) + + +bind_mm.register(bind__CVar_CVar, W_CVar, W_CVar) +bind_mm.register(bind__CVar_Root, W_CVar, W_Root) +bind_mm.register(bind__CVar_Var, W_CVar, W_Var) Added: pypy/dist/pypy/objspace/cclp/interp_var.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/interp_var.py Thu Aug 10 16:08:15 2006 @@ -0,0 +1,32 @@ +from pypy.objspace.cclp.variable import wait__Var, _assign_aliases, _entail +from pypy.objspace.cclp.types import W_Var + + + +def interp_wait(space, obj): + return wait__Var(space, obj) + + +class RebindingError(Exception): pass + +def interp_bind(space, w_var, obj): + if isinstance(w_var.w_bound_to, W_Var): + return _assign_aliases(space, w_var, obj) + if w_var.w_bound_to == obj: + return + raise RebindingError + +class EntailmentFailure(Exception): pass + +def interp_entail(space, w_v1, w_v2): + w_v1val = w_v1.w_bound_to + w_v2val = w_v2.w_bound_to + if not isinstance(w_v1val, W_Var): + if not isinstance(w_v2val, W_Var): + # let's be simpler than unify + if w_v1val != w_v2val: + raise EntailmentFailure + return _assign_aliases(space, w_v2, w_v1val) + else: + return _entail(space, w_v1, w_v2) + Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Thu Aug 10 16:08:15 2006 @@ -53,3 +53,5 @@ def this_thread(space): return ClonableCoroutine.w_getcurrent(space) app_this_thread = gateway.interp2app(this_thread) + + Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Thu Aug 10 16:08:15 2006 @@ -1,7 +1,10 @@ from pypy.module._stackless.coroutine import _AppThunk +from pypy.module._stackless.interp_coroutine import AbstractThunk + from pypy.objspace.cclp.misc import w from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue +from pypy.objspace.cclp.interp_var import interp_wait, interp_entail def logic_args(args): @@ -95,9 +98,6 @@ scheduler[0].schedule() -from pypy.interpreter.argument import Arguments -from pypy.module._stackless.interp_coroutine import AbstractThunk - class PropagatorThunk(AbstractThunk): def __init__(self, space, w_constraint, coro, Merged): self.space = space @@ -112,13 +112,14 @@ if entailed: break Obs = W_Var(self.space) - self.space.entail(self.Merged, Obs) + interp_entail(self.space, self.Merged, Obs) for Sync in [var.w_dom.give_synchronizer() for var in self.const._variables]: - self.space.entail(Sync, Obs) - self.space.wait(Obs) + interp_entail(self.space, Sync, Obs) + interp_wait(self.space, Obs) finally: self.coro._dead = True scheduler[0].remove_thread(self.coro) scheduler[0].schedule() + Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Thu Aug 10 16:08:15 2006 @@ -78,30 +78,10 @@ def __init__(self, space): self._space = space - self.__changed = W_Var(self._space) + self._changed = W_Var(self._space) - def clear_change(self): - #XXX called after revise ? - assert self._space.is_true(self._space.is_bound(self.__changed)) - self.__changed = W_Var(self._space) - - def give_synchronizer(self): - return self.__changed - - def _value_removed(self): - """The implementation of remove_value should call this method""" - self._space.bind(self.__changed, self._space.newbool(True)) - self.clear_change() - - if self.size() == 0: # self._space.eq_w(self.w_size(), self._space.newint(0)): - raise OperationError(self._space.w_RuntimeError, - self._space.wrap('ConsistencyFailure')) - def w__del__(self): - self._space.bind(self.__changed, self._space.newbool(False)) - -W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain", - give_synchronizer = gateway.interp2app(W_AbstractDomain.give_synchronizer)) +W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain") Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Thu Aug 10 16:08:15 2006 @@ -9,7 +9,6 @@ from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import deref, W_Var, W_CVar, W_Future, W_FailedValue -from pypy.objspace.cclp.constraint.domain import W_FiniteDomain W_Root = baseobjspace.W_Root all_mms = {} @@ -20,14 +19,6 @@ return w_v app_newvar = gateway.interp2app(newvar) -def domain(space, w_values, w_name): - assert isinstance(w_values, W_ListObject) - assert isinstance(w_name, W_StringObject) - w_dom = W_FiniteDomain(space, w_values) - w_var = W_CVar(space, w_dom, w_name) - w("CVAR", str(w_var)) - return w_var -app_domain = gateway.interp2app(domain) #-- Wait ------------------------------------------------- @@ -60,6 +51,7 @@ #-- Wait_needed -------------------------------------------- def wait_needed__Var(space, w_var): + w(":wait_needed", str(id(ClonableCoroutine.w_getcurrent(space)))) if space.is_true(space.is_free(w_var)): if w_var.needed: return @@ -74,6 +66,7 @@ return space.wait_needed(w_var) app_wait_needed = gateway.interp2app(wait_needed) + wait_needed_mm = StdObjSpaceMultiMethod('wait_needed', 1) wait_needed_mm.register(wait_needed__Var, W_Var) all_mms['wait_needed'] = wait_needed_mm @@ -181,6 +174,7 @@ assert isinstance(w_v2, W_Var) space.entail(w_v1, w_v2) app_entail = gateway.interp2app(entail) + def bind__Var_Root(space, w_var, w_obj): #w("var val", str(id(w_var))) @@ -198,14 +192,6 @@ raise_future_binding(space) return bind__Var_Root(space, w_fut, w_obj) # call-next-method ? -def bind__CVar_Root(space, w_cvar, w_obj): - #XXX we should (want to) be able to test membership - # in a wrapped against wrappeds into a non-wrapped dict - if [True for elt in w_cvar.w_dom._values.content - if space.is_true(space.eq(w_obj, elt))]: - return bind__Var_Root(space, w_cvar, w_obj) - raise_unification_failure(space, "value not in variable domain") - def bind__Var_Var(space, w_v1, w_v2): #w("var var") if space.is_true(space.is_bound(w_v1)): @@ -228,23 +214,6 @@ raise_future_binding(space) return bind__Var_Var(space, w_fut, w_var) -def bind__CVar_CVar(space, w_cvar1, w_cvar2): - w_inter_dom = space.intersection(w_cvar1.w_dom, w_cvar2.w_dom) - if w_inter_dom.__len__() > 0: - if w_inter_dom.__len__() == 1: - w_value = w_inter_dom.get_values()[0] - _assign_aliases(space, w_cvar1, w_value) - _assign_aliases(space, w_cvar2, w_value) - else: - w_cvar1.w_dom = w_cvar2.w_dom = w_inter_dom - _alias(space, w_cvar1, w_cvar2) - else: - raise_unification_failure(space, "incompatible domains") - -def bind__CVar_Var(space, w_cvar, w_var): - if space.is_true(space.is_bound(w_var)): - return bind__CVar_Root(space, w_cvar, w_var) - return bind__Var_Var(space, w_cvar, w_var) def bind__Var_Future(space, w_var, w_fut): if space.is_true(space.is_bound(w_fut)): #XXX write a test for me ! @@ -260,9 +229,6 @@ bind_mm.register(bind__Future_Root, W_Future, W_Root) bind_mm.register(bind__Future_Var, W_Future, W_Var) bind_mm.register(bind__Var_Future, W_Var, W_Future) -bind_mm.register(bind__CVar_CVar, W_CVar, W_CVar) -bind_mm.register(bind__CVar_Root, W_CVar, W_Root) -bind_mm.register(bind__CVar_Var, W_CVar, W_Var) all_mms['bind'] = bind_mm @@ -290,7 +256,7 @@ def _assign_aliases(space, w_var, w_val): w(" :assign") assert isinstance(w_var, W_Var) - assert isinstance(w_val, W_Root) + #assert isinstance(w_val, W_Root) w_curr = w_var while 1: w_next = w_curr.w_bound_to Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Thu Aug 10 16:08:15 2006 @@ -29,7 +29,9 @@ from pypy.objspace.cclp.variable import app_newvar, wait, app_wait, app_wait_needed, \ app_is_aliased, app_is_free, app_is_bound, app_alias_of, alias_of, app_bind, \ - app_unify, W_Var, W_CVar, W_Future, app_domain, all_mms as variable_mms, app_entail + app_unify, W_Var, W_CVar, W_Future, all_mms as variable_mms, app_entail + +from pypy.objspace.cclp.constraint.variable import app_domain from pypy.objspace.cclp.types import app_domain_of Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Thu Aug 10 16:08:15 2006 @@ -789,7 +789,7 @@ unify(X, status) break spc.merge() - + s = newspace(problem) Finished = newvar() stacklet(solve, s, Finished) From mwh at codespeak.net Thu Aug 10 16:11:40 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 10 Aug 2006 16:11:40 +0200 (CEST) Subject: [pypy-svn] r31232 - pypy/dist/pypy/objspace/std Message-ID: <20060810141140.B57EB10063@code0.codespeak.net> Author: mwh Date: Thu Aug 10 16:11:40 2006 New Revision: 31232 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: use os.O_TRUNC (oops) Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Thu Aug 10 16:11:40 2006 @@ -282,7 +282,7 @@ def report(): os.write(2, "starting to report!\n") - fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY, 0644) + fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) for info in _dict_infos: os.write(fd, '------------------\n') _report_one(fd, info) From auc at codespeak.net Thu Aug 10 17:41:01 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 10 Aug 2006 17:41:01 +0200 (CEST) Subject: [pypy-svn] r31234 - in pypy/dist/pypy/objspace: cclp cclp/constraint test Message-ID: <20060810154101.ACF4210063@code0.codespeak.net> Author: auc Date: Thu Aug 10 17:40:58 2006 New Revision: 31234 Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/cclp/variable.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: merge -- we can solve what does not need cloning Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Thu Aug 10 17:40:58 2006 @@ -16,7 +16,6 @@ from pypy.objspace.constraint.btree import BTree from pypy.objspace.constraint.util import sort, reverse - all_mms = {} @@ -127,7 +126,7 @@ domain = variable.w_dom values = domain.get_values() variables.append((domain.size(), [variable.w_name(), values, 0, len(values)])) - #kwargs.content[variable.w_name()] = values[0] + kwargs.content[variable.w_name()] = values[0] # sort variables to instanciate those with fewer possible values first sort(variables) self._assign_values_state = variables Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Thu Aug 10 17:40:58 2006 @@ -62,7 +62,9 @@ self._choice = newvar(space) self._committed = newvar(space) # merging + self._solution = newvar(space) self._merged = newvar(space) + def w_ask(self): scheduler[0].wait_stable(self) @@ -101,7 +103,7 @@ def w_merge(self): self.space.bind(self._merged, self.space.w_True) - + return self._solution Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Thu Aug 10 17:40:58 2006 @@ -4,7 +4,7 @@ from pypy.objspace.cclp.misc import w from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue -from pypy.objspace.cclp.interp_var import interp_wait, interp_entail +from pypy.objspace.cclp.interp_var import interp_wait, interp_entail, interp_bind def logic_args(args): @@ -92,6 +92,7 @@ self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) else: w(".% clean (valueless) EXIT of", str(id(self._coro))) + self.space.bind(cspace._solution, self.costate.w_tempval) self.space.bind(cspace._choice, self.space.wrap(SPACE_SOLUTION)) finally: scheduler[0].remove_thread(self._coro) @@ -107,17 +108,25 @@ def call(self): try: - while 1: - entailed = self.const.revise() - if entailed: - break - Obs = W_Var(self.space) - interp_entail(self.space, self.Merged, Obs) - for Sync in [var.w_dom.give_synchronizer() - for var in self.const._variables]: - interp_entail(self.space, Sync, Obs) - interp_wait(self.space, Obs) + try: + while 1: + entailed = self.const.revise() + if entailed: + break + Obs = W_Var(self.space) + interp_entail(self.space, self.Merged, Obs) + for Sync in [var.w_dom.give_synchronizer() + for var in self.const._variables]: + interp_entail(self.space, Sync, Obs) + interp_wait(self.space, Obs) + except: + import traceback + traceback.print_exc() finally: + # all values of dom size 1 are bound + for var in self.const._variables: + if var.w_dom.size() == 1: + interp_bind(self.space, var, var.w_dom.get_values()[0]) self.coro._dead = True scheduler[0].remove_thread(self.coro) scheduler[0].schedule() Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Thu Aug 10 17:40:58 2006 @@ -69,8 +69,7 @@ def __init__(self, object_space): self._space = object_space -W_Constraint.typedef = typedef.TypeDef( - "W_Constraint") +W_Constraint.typedef = typedef.TypeDef("W_Constraint") class W_AbstractDomain(baseobjspace.Wrappable): """Implements the functionnality related to the changed flag. Modified: pypy/dist/pypy/objspace/cclp/variable.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/variable.py (original) +++ pypy/dist/pypy/objspace/cclp/variable.py Thu Aug 10 17:40:58 2006 @@ -26,7 +26,7 @@ return w_obj def wait__Var(space, w_var): - w(":wait", str(id(ClonableCoroutine.w_getcurrent(space)))) + #w(":wait", str(id(ClonableCoroutine.w_getcurrent(space)))) if space.is_true(space.is_free(w_var)): scheduler[0].unblock_byneed_on(w_var) scheduler[0].add_to_blocked_on(w_var, ClonableCoroutine.w_getcurrent(space)) @@ -51,7 +51,7 @@ #-- Wait_needed -------------------------------------------- def wait_needed__Var(space, w_var): - w(":wait_needed", str(id(ClonableCoroutine.w_getcurrent(space)))) + #w(":wait_needed", str(id(ClonableCoroutine.w_getcurrent(space)))) if space.is_true(space.is_free(w_var)): if w_var.needed: return Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Thu Aug 10 17:40:58 2006 @@ -781,19 +781,21 @@ def problem(): X, Y = domain([1, 2], 'X'), domain([1, 2, 3], 'Y') tell(make_expression([X, Y], 'X + Y > 4')) + return (X, Y) def solve(spc, X): while 1: status = spc.ask() if status == 1: - unify(X, status) break - spc.merge() + unify(spc.merge(), X) s = newspace(problem) - Finished = newvar() - stacklet(solve, s, Finished) - wait(Finished) + Solution = newvar() + stacklet(solve, s, Solution) schedule() + + assert Solution == (2, 3) + assert len(sched_all()['threads']) == 1 From hpk at codespeak.net Thu Aug 10 19:25:29 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 10 Aug 2006 19:25:29 +0200 (CEST) Subject: [pypy-svn] r31240 - pypy/extradoc/eu-report Message-ID: <20060810172529.BE40510071@code0.codespeak.net> Author: hpk Date: Thu Aug 10 19:25:25 2006 New Revision: 31240 Modified: pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-interim-2006-07-25.pdf (props changed) pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf (props changed) Log: fix mime type to be application/PDF From rhymes at codespeak.net Fri Aug 11 10:56:19 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 11 Aug 2006 10:56:19 +0200 (CEST) Subject: [pypy-svn] r31249 - in pypy/dist/pypy/module: _ssl/test bz2/test fcntl/test mmap/test rctime/test Message-ID: <20060811085619.D00671005A@code0.codespeak.net> Author: rhymes Date: Fri Aug 11 10:56:15 2006 New Revision: 31249 Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/dist/pypy/module/bz2/test/test_bz2_file.py pypy/dist/pypy/module/fcntl/test/test_fcntl.py pypy/dist/pypy/module/mmap/test/test_mmap.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: explicit skip Modified: pypy/dist/pypy/module/_ssl/test/test_ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/test/test_ssl.py (original) +++ pypy/dist/pypy/module/_ssl/test/test_ssl.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("Windows is not supported") class AppTestSSL: Modified: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("bz2 module is not available on Windows") def setup_module(mod): Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("bz2 module is not available on Windows") def setup_module(mod): Modified: pypy/dist/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/test/test_fcntl.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("fcntl module is not available on Windows") def teardown_module(mod): Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("Windows is not supported") def teardown_module(mod): Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Fri Aug 11 10:56:15 2006 @@ -2,6 +2,7 @@ import os if os.name == "nt": + from py.test import skip skip("Windows is not supported") class AppTestRCTime: From mwh at codespeak.net Fri Aug 11 11:06:15 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 11 Aug 2006 11:06:15 +0200 (CEST) Subject: [pypy-svn] r31250 - pypy/dist/pypy/tool Message-ID: <20060811090615.1D2C010060@code0.codespeak.net> Author: mwh Date: Fri Aug 11 11:06:14 2006 New Revision: 31250 Modified: pypy/dist/pypy/tool/readdictinfo.py Log: memory reducing hacks, and some code to put data in a form that R can deal with. Modified: pypy/dist/pypy/tool/readdictinfo.py ============================================================================== --- pypy/dist/pypy/tool/readdictinfo.py (original) +++ pypy/dist/pypy/tool/readdictinfo.py Fri Aug 11 11:06:14 2006 @@ -1,5 +1,3 @@ -import autopath - # this is for use with a pypy-c build with multidicts and using the # MeasuringDictImplementation -- it will create a file called # 'dictinfo.txt' in the local directory and this file will turn the @@ -7,12 +5,26 @@ # run with python -i ! -from pypy.objspace.std.dictmultiobject import DictInfo - import sys infile = open(sys.argv[1]) +curr = None +slots = [] +for line in infile: + if line == '------------------\n': + if curr: + break + curr = 1 + else: + attr, val = [s.strip() for s in line.split(':')] + slots.append(attr) + +class DictInfo(object): + __slots__ = slots + +infile = open(sys.argv[1]) + infos = [] for line in infile: @@ -39,11 +51,57 @@ d2[v2] = d2.get(v2, 0) + 1 return sorted(r.items()) -import pprint -try: - import readline -except ImportError: - pass -else: - import rlcompleter - readline.parse_and_bind('tab: complete') +def reportDictInfos(): + d = {} + stillAlive = 0 + totLifetime = 0.0 + for info in infos: + for attr in slots: + if attr == 'maxcontents': + continue + v = getattr(info, attr) + if not isinstance(v, int): + continue + d[attr] = d.get(attr, 0) + v + if info.lifetime != -1.0: + totLifetime += info.lifetime + else: + stillAlive += 1 + print 'read info on', len(infos), 'dictionaries' + if stillAlive != len(infos): + print 'average lifetime', totLifetime/(len(infos) - stillAlive), + print '('+str(stillAlive), 'still alive at exit)' + print d + +def Rify(fname, *attributes): + output = open(fname, 'w') + for attr in attributes: + print >>output, attr, + print >>output + for info in infos: + for attr in attributes: + print >>output, getattr(info, attr), + print >>output + +if __name__ == '__main__': + reportDictInfos() + + # interactive stuff: + + import __builtin__ + + def displayhook(v): + if v is not None: + __builtin__._ = v + pprint.pprint(v) + sys.displayhook = displayhook + + import pprint + try: + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind('tab: complete') + From mwh at codespeak.net Fri Aug 11 11:23:54 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 11 Aug 2006 11:23:54 +0200 (CEST) Subject: [pypy-svn] r31251 - pypy/dist/pypy/objspace/std Message-ID: <20060811092354.1DB2F1006F@code0.codespeak.net> Author: mwh Date: Fri Aug 11 11:23:53 2006 New Revision: 31251 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: make the _dict_info list not global Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Fri Aug 11 11:23:53 2006 @@ -117,11 +117,11 @@ if MEASURE_DICT: import time, py - _dict_infos = [] class DictInfo(object): + _dict_infos = [] def __init__(self): - self.id = len(_dict_infos) + self.id = len(self._dict_infos) self.getitems = 0; self.setitems = 0; self.delitems = 0 self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 @@ -158,7 +158,7 @@ else: self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) - _dict_infos.append(self) + self._dict_infos.append(self) def __repr__(self): args = [] for k in sorted(self.__dict__): @@ -267,7 +267,7 @@ return self.content.items() _example = DictInfo() - del _dict_infos[-1] + del DictInfo._dict_infos[-1] tmpl = 'os.write(fd, "%(attr)s" + ": " + str(info.%(attr)s) + "\\n")' bodySrc = [] for attr in sorted(_example.__dict__): @@ -283,7 +283,7 @@ def report(): os.write(2, "starting to report!\n") fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) - for info in _dict_infos: + for info in DictInfo._dict_infos: os.write(fd, '------------------\n') _report_one(fd, info) os.close(fd) @@ -291,11 +291,11 @@ def reportDictInfo(): d = {} - if not _dict_infos: + if not DictInfo._dict_infos: return stillAlive = 0 totLifetime = 0.0 - for info in _dict_infos: + for info in DictInfo._dict_infos: for attr in info.__dict__: if attr == 'maxcontents': continue @@ -308,10 +308,10 @@ else: stillAlive += 1 import cPickle - cPickle.dump(_dict_infos, open('dictinfos.pickle', 'wb')) - print 'reporting on', len(_dict_infos), 'dictionaries' - if stillAlive != len(_dict_infos): - print 'average lifetime', totLifetime/(len(_dict_infos) - stillAlive), + cPickle.dump(DictInfo._dict_infos, open('dictinfos.pickle', 'wb')) + print 'reporting on', len(DictInfo._dict_infos), 'dictionaries' + if stillAlive != len(DictInfo._dict_infos): + print 'average lifetime', totLifetime/(len(DictInfo._dict_infos) - stillAlive), print '('+str(stillAlive), 'still alive)' print d From arigo at codespeak.net Fri Aug 11 12:54:45 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Aug 2006 12:54:45 +0200 (CEST) Subject: [pypy-svn] r31256 - in pypy/dist/pypy: rpython/rctypes rpython/rctypes/test translator/backendopt translator/backendopt/test translator/c translator/c/test Message-ID: <20060811105445.D86B710068@code0.codespeak.net> Author: arigo Date: Fri Aug 11 12:54:43 2006 New Revision: 31256 Modified: pypy/dist/pypy/rpython/rctypes/astruct.py pypy/dist/pypy/rpython/rctypes/rstruct.py pypy/dist/pypy/rpython/rctypes/test/test_rstruct.py pypy/dist/pypy/translator/backendopt/malloc.py pypy/dist/pypy/translator/backendopt/test/test_malloc.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/c/test/test_lltyped.py Log: Support for ctypes.Union in genc. (XXX missing llinterp support) Modified: pypy/dist/pypy/rpython/rctypes/astruct.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/astruct.py (original) +++ pypy/dist/pypy/rpython/rctypes/astruct.py Fri Aug 11 12:54:43 2006 @@ -1,14 +1,18 @@ -from ctypes import Structure +from ctypes import Structure, Union from pypy.annotation.model import SomeCTypesObject from pypy.rpython.rctypes.implementation import CTypesCallEntry, CTypesObjEntry from pypy.rpython.lltypesystem import lltype StructType = type(Structure) +UnionType = type(Union) + +# XXX this also implements Unions, but they are not properly emulated +# by the llinterpreter. They work in the generated C code, though. class CallEntry(CTypesCallEntry): "Annotation and rtyping of calls to structure types." - _type_ = StructType + _type_ = StructType, UnionType def specialize_call(self, hop, **kwds_i): from pypy.rpython.error import TyperError @@ -48,7 +52,7 @@ class ObjEntry(CTypesObjEntry): "Annotation and rtyping of structure instances." - _metatype_ = StructType + _metatype_ = StructType, UnionType def get_field_annotation(self, s_struct, fieldname): for name, ctype in self.type._fields_: @@ -59,4 +63,7 @@ def get_repr(self, rtyper, s_struct): from pypy.rpython.rctypes.rstruct import StructRepr - return StructRepr(rtyper, s_struct) + is_struct = isinstance(self.type, StructType) + is_union = isinstance(self.type, UnionType) + assert is_struct ^ is_union + return StructRepr(rtyper, s_struct, is_union) Modified: pypy/dist/pypy/rpython/rctypes/rstruct.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rstruct.py (original) +++ pypy/dist/pypy/rpython/rctypes/rstruct.py Fri Aug 11 12:54:43 2006 @@ -7,7 +7,7 @@ class StructRepr(CTypesRefRepr): - def __init__(self, rtyper, s_struct): + def __init__(self, rtyper, s_struct, is_union=False): struct_ctype = s_struct.knowntype # Find the repr and low-level type of the fields from their ctype @@ -23,6 +23,8 @@ external = getattr(struct_ctype, '_external_', False) extras = {'hints': {'c_name': struct_ctype.__name__, 'external': external}} + if is_union: + extras['hints']['union'] = True c_data_type = lltype.Struct(struct_ctype.__name__, *llfields, **extras) super(StructRepr, self).__init__(rtyper, s_struct, c_data_type) Modified: pypy/dist/pypy/rpython/rctypes/test/test_rstruct.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rstruct.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rstruct.py Fri Aug 11 12:54:43 2006 @@ -13,12 +13,17 @@ from pypy.rpython.test.test_llinterp import interpret from ctypes import c_int, c_short, Structure, POINTER, pointer, c_char_p -from ctypes import c_char, SetPointerType +from ctypes import c_char, SetPointerType, Union, c_long class tagpoint(Structure): _fields_ = [("x", c_int), ("y", c_int)] +class uvalue(Union): + _fields_ = [("c", c_char), + ("s", c_short), + ("l", c_long)] + def maketest(): class S1(Structure): _fields_ = [('x', c_int)] class S2(Structure): _fields_ = [('x', POINTER(c_int))] @@ -93,6 +98,17 @@ a.translator.view() assert s.knowntype == int + def test_annotate_union(self): + def func(n): + u = uvalue(s=n) + return u.c + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(func, [int]) + if conftest.option.view: + a.translator.view() + assert s.knowntype == str + class Test_specialization: def test_specialize_struct(self): def create_struct(): @@ -250,3 +266,12 @@ func, expected = maketest() fn = compile(func, []) assert fn() == expected + + def test_compile_union(self): + def func(n): + u = uvalue(s=n) + return u.c + fn = compile(func, [int]) + res = fn(0x4567) + expected = func(0x4567) + assert res == expected Modified: pypy/dist/pypy/translator/backendopt/malloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/malloc.py (original) +++ pypy/dist/pypy/translator/backendopt/malloc.py Fri Aug 11 12:54:43 2006 @@ -34,11 +34,22 @@ return False FIELDTYPE = S._flds[fieldname] if isinstance(FIELDTYPE, lltype.GcStruct): + if FIELDTYPE._hints.get('union'): + return False return True if len(S._names) == 1: return True return False +def union_wrapper(S): + # check if 'S' is a GcStruct containing a single inlined *union* Struct + if not isinstance(S, lltype.GcStruct): + return False + assert not S._hints.get('union') # not supported: "GcUnion" + return (len(S._names) == 1 and + isinstance(S._flds[S._names[0]], lltype.Struct) and + S._flds[S._names[0]]._hints.get('union')) + def compute_lifetimes(graph): """Compute the static data flow of the graph: returns a list of LifeTime @@ -183,6 +194,10 @@ except (ValueError, AttributeError), e: pass + # must not remove unions inlined as the only field of a GcStruct + if union_wrapper(STRUCT): + return False + # success: replace each variable with a family of variables (one per field) # 'flatnames' is a list of (STRUCTTYPE, fieldname_in_that_struct) that Modified: pypy/dist/pypy/translator/backendopt/test/test_malloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_malloc.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_malloc.py Fri Aug 11 12:54:43 2006 @@ -1,5 +1,6 @@ import py from pypy.translator.backendopt.malloc import remove_mallocs_once +from pypy.translator.backendopt.malloc import union_wrapper from pypy.translator.backendopt.inline import inline_function from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.translator import TranslationContext, graphof @@ -9,6 +10,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.conftest import option + def check_malloc_removed(graph): checkgraph(graph) count1 = count2 = 0 @@ -16,7 +18,9 @@ if isinstance(node, Block): for op in node.operations: if op.opname == 'malloc': - count1 += 1 + S = op.args[0].value + if not union_wrapper(S): # union wrappers are fine + count1 += 1 if op.opname in ('direct_call', 'indirect_call'): count2 += 1 assert count1 == 0 # number of mallocs left @@ -36,9 +40,10 @@ simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks())) if progress and option.view: t.view() - interp = LLInterpreter(t.rtyper) - res = interp.eval_graph(graph, args) - assert res == expected_result + if expected_result is not Ellipsis: + interp = LLInterpreter(t.rtyper) + res = interp.eval_graph(graph, args) + assert res == expected_result if not progress: break if must_be_removed: @@ -263,3 +268,14 @@ x.z += 3 return x.z check(fn, [], [], 12) + +def test_union(): + UNION = lltype.Struct('UNION', ('a', lltype.Signed), ('b', lltype.Signed), + hints = {'union': True}) + BIG = lltype.GcStruct('BIG', ('u1', UNION), ('u2', UNION)) + def fn(): + x = lltype.malloc(BIG) + x.u1.a = 3 + x.u2.b = 6 + return x.u1.b * x.u2.a + check(fn, [], [], Ellipsis) Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Fri Aug 11 12:54:43 2006 @@ -396,7 +396,7 @@ print >> fi, '/*** Structure definitions ***/' print >> fi for node in structdeflist: - print >> fi, 'struct %s;' % node.name + print >> fi, '%s %s;' % (node.typetag, node.name) print >> fi for node in structdeflist: for line in node.definition(): @@ -513,7 +513,7 @@ print >> f for node in structdeflist: if node.name: - print >> f, 'struct %s;' % node.name + print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: for line in node.definition(): Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Fri Aug 11 12:54:43 2006 @@ -35,6 +35,7 @@ class StructDefNode: + typetag = 'struct' def __init__(self, db, STRUCT, varlength=1): self.db = db @@ -48,6 +49,9 @@ basename = db.gettypedefnode(STRUCT).barename basename = '%s_len%d' % (basename, varlength) with_number = False + if STRUCT._hints.get('union'): + self.typetag = 'union' + assert STRUCT._gckind == 'raw' # not supported: "GcUnion" if STRUCT._hints.get('c_name'): self.barename = self.name = STRUCT._hints['c_name'] self.c_struct_field_name = self.verbatim_field_name @@ -97,7 +101,7 @@ gcinfo = defaultproperty(computegcinfo) def gettype(self): - return 'struct %s @' % self.name + return '%s %s @' % (self.typetag, self.name) def c_struct_field_name(self, name): # occasionally overridden in __init__(): @@ -141,7 +145,7 @@ def definition(self): if self.fields is None: # external definition only return - yield 'struct %s {' % self.name + yield '%s %s {' % (self.typetag, self.name) is_empty = True for name, typename in self.fields: line = '%s;' % cdecl(typename, name) @@ -151,7 +155,7 @@ is_empty = False yield '\t' + line if is_empty: - yield '\t' + 'int _dummy; /* this struct is empty */' + yield '\t' + 'char _dummy; /* this struct is empty */' yield '};' for line in self.db.gcpolicy.struct_after_definition(self): yield line @@ -178,10 +182,12 @@ except ValueError: yield '-1' else: - yield 'offsetof(struct %s, %s)' % (self.name, cname) + yield 'offsetof(%s %s, %s)' % (self.typetag, + self.name, cname) class ArrayDefNode: + typetag = 'struct' def __init__(self, db, ARRAY, varlength=1): self.db = db Modified: pypy/dist/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/dist/pypy/translator/c/test/test_lltyped.py Fri Aug 11 12:54:43 2006 @@ -238,3 +238,16 @@ fn = self.getcompiled(llf) res = fn() assert res == 27 + + def test_union(self): + U = Struct('U', ('s', Signed), ('c', Char), + hints={'union': True}) + u = malloc(U, immortal=True) + def llf(c=int): + u.s = 0x1020 + u.c = chr(c) + return u.s + + fn = self.getcompiled(llf) + res = fn(0x33) + assert res in [0x1033, 0x3320] From fijal at codespeak.net Fri Aug 11 13:34:17 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 11 Aug 2006 13:34:17 +0200 (CEST) Subject: [pypy-svn] r31257 - pypy/dist/pypy/translator/cl/test Message-ID: <20060811113417.266E010060@code0.codespeak.net> Author: fijal Date: Fri Aug 11 13:34:11 2006 New Revision: 31257 Modified: pypy/dist/pypy/translator/cl/test/test_cltrans.py pypy/dist/pypy/translator/cl/test/test_dict.py pypy/dist/pypy/translator/cl/test/test_exception.py pypy/dist/pypy/translator/cl/test/test_list.py pypy/dist/pypy/translator/cl/test/test_tuple.py Log: Skipped tests which does not pass because of lack of implementation of CDefinedInt. Modified: pypy/dist/pypy/translator/cl/test/test_cltrans.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_cltrans.py (original) +++ pypy/dist/pypy/translator/cl/test/test_cltrans.py Fri Aug 11 13:34:11 2006 @@ -131,6 +131,7 @@ assert cl_builtinusage() == 4 def test_slice(): + py.test.skip("CDefinedInt implementation") def half_of_n(n=int): """Slice test""" i = 0 @@ -154,6 +155,7 @@ assert result.val == ( '#(#() #(0) #(1) #(0 1) #(2) #(0 2) #(1 2) #(0 1 2))') def test_yast(): + py.test.skip("CDefinedInt implementation") def yast1(n): # Need this to avoid name clashes in the generated code return t.yast(range(n)) Modified: pypy/dist/pypy/translator/cl/test/test_dict.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_dict.py (original) +++ pypy/dist/pypy/translator/cl/test/test_dict.py Fri Aug 11 13:34:11 2006 @@ -9,6 +9,7 @@ assert cl_dict_length_one(42, 42) == 1 def test_dict_get(): + py.test.skip("CDefinedInt implementation") def dict_get(number): dic = {42: 43} return dic[number] @@ -16,6 +17,7 @@ assert cl_dict_get(42) == 43 def test_dict_iter(): + py.test.skip("CDefinedInt implementation") def dict_iter(): dic = {1:2, 3:4, 5:6} i = 0 @@ -26,6 +28,7 @@ assert cl_dict_iter() == 12 def test_dict_values(): + py.test.skip("CDefinedInt implementation") def dict_values(): dic = {1:2, 3:4, 5:6} i = 0 Modified: pypy/dist/pypy/translator/cl/test/test_exception.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_exception.py (original) +++ pypy/dist/pypy/translator/cl/test/test_exception.py Fri Aug 11 13:34:11 2006 @@ -21,6 +21,7 @@ assert cl_handle_exception(False) == 2 def test_indirect_call(): + py.test.skip("CDefinedInt implementation") def id(n): return n def square(n): @@ -40,6 +41,7 @@ assert cl_square_sum(5) == 55 def test_iteration(): + py.test.skip("CDefinedInt implementation") def get_last(num): last = 0 for i in range(num): Modified: pypy/dist/pypy/translator/cl/test/test_list.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_list.py (original) +++ pypy/dist/pypy/translator/cl/test/test_list.py Fri Aug 11 13:34:11 2006 @@ -21,6 +21,7 @@ assert cl_list_get(1985) == 1985 def test_list_iter(): + py.test.skip("CDefinedInt implementation") def list_iter(): a = 0 for item in [1,2,3,4,5]: @@ -46,6 +47,7 @@ assert cl_list_concat() == 8 def test_list_setitem(): + py.test.skip("CDefinedInt implementation") def list_setitem(num): a = [1,2,3,4] a[1] = num Modified: pypy/dist/pypy/translator/cl/test/test_tuple.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_tuple.py (original) +++ pypy/dist/pypy/translator/cl/test/test_tuple.py Fri Aug 11 13:34:11 2006 @@ -8,6 +8,7 @@ assert cl_tuple_get(1) == 1 def test_tuple_iter(): + py.test.skip("CDefinedInt implementation") def tuple_double(number): tuple = (number,) for item in tuple: From rhymes at codespeak.net Fri Aug 11 14:23:29 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 11 Aug 2006 14:23:29 +0200 (CEST) Subject: [pypy-svn] r31261 - pypy/dist/pypy/module/_ssl Message-ID: <20060811122329.05C9810068@code0.codespeak.net> Author: rhymes Date: Fri Aug 11 14:23:28 2006 New Revision: 31261 Modified: pypy/dist/pypy/module/_ssl/ssl.py Log: ext. compiler needs an explicit list of _fields_ also if empty. step forward to compilation Modified: pypy/dist/pypy/module/_ssl/ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/ssl.py (original) +++ pypy/dist/pypy/module/_ssl/ssl.py Fri Aug 11 14:23:28 2006 @@ -349,6 +349,7 @@ BIGNUM = bignum_st class bignum_ctx(Structure): pass +bignum_ctx._fields_ = [] BN_CTX = bignum_ctx class bn_blinding_st(Structure): pass @@ -519,6 +520,7 @@ DH_METHOD = dh_method class engine_st(Structure): pass +engine_st._fields_ = [] ENGINE = engine_st dh_st._fields_ = [ ('pad', c_int), @@ -1705,6 +1707,7 @@ X509_CERT_AUX = x509_cert_aux_st class AUTHORITY_KEYID_st(Structure): pass +AUTHORITY_KEYID_st._fields_ = [] x509_st._fields_ = [ ('cert_info', POINTER(X509_CINF)), ('sig_alg', POINTER(X509_ALGOR)), From rhymes at codespeak.net Fri Aug 11 14:40:18 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 11 Aug 2006 14:40:18 +0200 (CEST) Subject: [pypy-svn] r31262 - pypy/dist/pypy/module/bz2 Message-ID: <20060811124018.59EFD10078@code0.codespeak.net> Author: rhymes Date: Fri Aug 11 14:40:17 2006 New Revision: 31262 Modified: pypy/dist/pypy/module/bz2/bzlib.py Log: explicit empty fields for bz2lib too Modified: pypy/dist/pypy/module/bz2/bzlib.py ============================================================================== --- pypy/dist/pypy/module/bz2/bzlib.py (original) +++ pypy/dist/pypy/module/bz2/bzlib.py Fri Aug 11 14:40:17 2006 @@ -69,6 +69,7 @@ assert alignment(__sbuf) == 4, alignment(__sbuf) class __sFILEX(Structure): pass +__sFILEX._fields_ = [] class __sFILE(Structure): pass __sFILE._pack_ = 4 From rhymes at codespeak.net Fri Aug 11 15:47:24 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 11 Aug 2006 15:47:24 +0200 (CEST) Subject: [pypy-svn] r31263 - pypy/dist/pypy/module/_ssl Message-ID: <20060811134724.5F3131005A@code0.codespeak.net> Author: rhymes Date: Fri Aug 11 15:47:22 2006 New Revision: 31263 Modified: pypy/dist/pypy/module/_ssl/ssl.py Log: merged the new ssl.py generated by the svn version of the ctypes code generator. Thomas Heller fixed the explicit stuff 'bug'. See #31261 Modified: pypy/dist/pypy/module/_ssl/ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/ssl.py (original) +++ pypy/dist/pypy/module/_ssl/ssl.py Fri Aug 11 15:47:22 2006 @@ -1,20 +1,19 @@ -# generated from openssl/ssl.h with ctypes' codegen from ctypes import * STRING = c_char_p -OSBigEndian = 2 +OSUnknownByteOrder = 0 UIT_PROMPT = 1 +P_PGID = 2 P_PID = 1 +UIT_ERROR = 5 UIT_INFO = 4 +UIT_NONE = 0 +P_ALL = 0 UIT_VERIFY = 2 +OSBigEndian = 2 UIT_BOOLEAN = 3 OSLittleEndian = 1 -P_PGID = 2 -P_ALL = 0 -OSUnknownByteOrder = 0 -UIT_NONE = 0 -UIT_ERROR = 5 __darwin_nl_item = c_int __darwin_wctrans_t = c_int __darwin_wctype_t = c_ulong @@ -165,15 +164,23 @@ ASN1_STRING_TABLE = asn1_string_table_st class ASN1_TEMPLATE_st(Structure): pass +ASN1_TEMPLATE_st._fields_ = [ +] ASN1_TEMPLATE = ASN1_TEMPLATE_st class ASN1_ITEM_st(Structure): pass ASN1_ITEM = ASN1_ITEM_st +ASN1_ITEM_st._fields_ = [ +] class ASN1_TLC_st(Structure): pass ASN1_TLC = ASN1_TLC_st +ASN1_TLC_st._fields_ = [ +] class ASN1_VALUE_st(Structure): pass +ASN1_VALUE_st._fields_ = [ +] ASN1_VALUE = ASN1_VALUE_st ASN1_ITEM_EXP = ASN1_ITEM class asn1_type_st(Structure): @@ -326,6 +333,8 @@ BIO_F_BUFFER_CTX = bio_f_buffer_ctx_struct class hostent(Structure): pass +hostent._fields_ = [ +] class bf_key_st(Structure): pass bf_key_st._fields_ = [ @@ -349,7 +358,8 @@ BIGNUM = bignum_st class bignum_ctx(Structure): pass -bignum_ctx._fields_ = [] +bignum_ctx._fields_ = [ +] BN_CTX = bignum_ctx class bn_blinding_st(Structure): pass @@ -437,6 +447,8 @@ COMP_CTX = comp_ctx_st class CRYPTO_dynlock_value(Structure): pass +CRYPTO_dynlock_value._fields_ = [ +] class CRYPTO_dynlock(Structure): pass CRYPTO_dynlock._fields_ = [ @@ -464,6 +476,8 @@ class st_CRYPTO_EX_DATA_IMPL(Structure): pass CRYPTO_EX_DATA_IMPL = st_CRYPTO_EX_DATA_IMPL +st_CRYPTO_EX_DATA_IMPL._fields_ = [ +] CRYPTO_MEM_LEAK_CB = CFUNCTYPE(c_void_p, c_ulong, STRING, c_int, c_int, c_void_p) DES_cblock = c_ubyte * 8 const_DES_cblock = c_ubyte * 8 @@ -520,7 +534,6 @@ DH_METHOD = dh_method class engine_st(Structure): pass -engine_st._fields_ = [] ENGINE = engine_st dh_st._fields_ = [ ('pad', c_int), @@ -843,6 +856,8 @@ class x509_store_ctx_st(Structure): pass X509_STORE_CTX = x509_store_ctx_st +engine_st._fields_ = [ +] class PEM_Encode_Seal_st(Structure): pass PEM_Encode_Seal_st._fields_ = [ @@ -1169,21 +1184,6 @@ pass class ssl3_enc_method(Structure): pass -ssl3_enc_method._fields_ = [ - ('enc', CFUNCTYPE(POINTER(SSL), c_int)), - ('mac', CFUNCTYPE(POINTER(SSL), STRING, c_int)), - ('setup_key_block', CFUNCTYPE(POINTER(SSL))), - ('generate_master_secret', CFUNCTYPE(POINTER(SSL), STRING, STRING, c_int)), - ('change_cipher_state', CFUNCTYPE(POINTER(SSL), c_int)), - ('final_finish_mac', CFUNCTYPE(POINTER(SSL), POINTER(EVP_MD_CTX), POINTER(EVP_MD_CTX), STRING, c_int, STRING)), - ('finish_mac_length', c_int), - ('cert_verify_mac', CFUNCTYPE(POINTER(SSL), POINTER(EVP_MD_CTX), STRING)), - ('client_finished_label', STRING), - ('client_finished_label_len', c_int), - ('server_finished_label', STRING), - ('server_finished_label_len', c_int), - ('alert_value', CFUNCTYPE(c_int)), -] ssl_method_st._fields_ = [ ('version', c_int), ('ssl_new', CFUNCTYPE(c_int, POINTER(SSL))), @@ -1213,6 +1213,8 @@ ] assert sizeof(ssl_method_st) == 100, sizeof(ssl_method_st) assert alignment(ssl_method_st) == 4, alignment(ssl_method_st) +ssl3_enc_method._fields_ = [ +] SSL_METHOD = ssl_method_st class ssl_session_st(Structure): pass @@ -1245,6 +1247,8 @@ ] assert sizeof(ssl_session_st) == 200, sizeof(ssl_session_st) assert alignment(ssl_session_st) == 4, alignment(ssl_session_st) +sess_cert_st._fields_ = [ +] SSL_SESSION = ssl_session_st GEN_SESSION_CB = CFUNCTYPE(c_int, POINTER(SSL), POINTER(c_ubyte), POINTER(c_uint)) class ssl_comp_st(Structure): @@ -1324,6 +1328,8 @@ ] assert sizeof(ssl_ctx_st) == 248, sizeof(ssl_ctx_st) assert alignment(ssl_ctx_st) == 4, alignment(ssl_ctx_st) +cert_st._fields_ = [ +] class ssl2_state_st(Structure): pass class ssl3_state_st(Structure): @@ -1551,12 +1557,18 @@ assert alignment(stack_st) == 4, alignment(stack_st) class ui_st(Structure): pass +ui_st._fields_ = [ +] UI = ui_st class ui_method_st(Structure): pass +ui_method_st._fields_ = [ +] UI_METHOD = ui_method_st class ui_string_st(Structure): pass +ui_string_st._fields_ = [ +] UI_STRING = ui_string_st # values for enumeration 'UI_string_types' @@ -1707,7 +1719,6 @@ X509_CERT_AUX = x509_cert_aux_st class AUTHORITY_KEYID_st(Structure): pass -AUTHORITY_KEYID_st._fields_ = [] x509_st._fields_ = [ ('cert_info', POINTER(X509_CINF)), ('sig_alg', POINTER(X509_ALGOR)), @@ -1728,6 +1739,8 @@ ] assert sizeof(x509_st) == 84, sizeof(x509_st) assert alignment(x509_st) == 4, alignment(x509_st) +AUTHORITY_KEYID_st._fields_ = [ +] class x509_trust_st(Structure): pass x509_trust_st._fields_ = [ @@ -2012,6 +2025,8 @@ assert alignment(__sbuf) == 4, alignment(__sbuf) class __sFILEX(Structure): pass +__sFILEX._fields_ = [ +] class __sFILE(Structure): pass __sFILE._pack_ = 4 @@ -2073,8 +2088,12 @@ mode_t = __darwin_mode_t class mcontext(Structure): pass +mcontext._fields_ = [ +] class mcontext64(Structure): pass +mcontext64._fields_ = [ +] class __darwin_pthread_handler_rec(Structure): pass __darwin_pthread_handler_rec._fields_ = [ From auc at codespeak.net Fri Aug 11 15:58:48 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Fri, 11 Aug 2006 15:58:48 +0200 (CEST) Subject: [pypy-svn] r31264 - pypy/extradoc/sprintinfo/paris Message-ID: <20060811135848.58DAD10034@code0.codespeak.net> Author: auc Date: Fri Aug 11 15:58:46 2006 New Revision: 31264 Added: pypy/extradoc/sprintinfo/paris/paris-minisprint-082006.txt Log: paris minisprint report Added: pypy/extradoc/sprintinfo/paris/paris-minisprint-082006.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/paris/paris-minisprint-082006.txt Fri Aug 11 15:58:46 2006 @@ -0,0 +1,42 @@ +Minisprint in Paris from 7 to 11 August 2006 +============================================ + +Participants: +------------- + +* Anders Lehmann +* Aur?lien Camp?as + +Objectives: +----------- + +* advance WP09 + +Achievements: +------------- + +We implemented most of the necessary primitives for solving logic and +constraint problems (tell, ask, choose, commit). The missing one is +clone. + +We adapted the current constraint solver components (domains, +constraint propagators, distributors) to make it fit the requirements +of logic programming (they share the same basic primitives). + +Logic variables are used internally as a coordination mechanism +between threads. The current implementation would not support +preemptive threading -- that would need even more coordination of this +kind. + +So right now, it is possible to explore exactly one branch of the +search tree at a time. + +Next: +----- + +* translatability + +* make cloning work + +* adapt OWL reasoner to the current solver API + From mwh at codespeak.net Fri Aug 11 17:39:16 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 11 Aug 2006 17:39:16 +0200 (CEST) Subject: [pypy-svn] r31265 - pypy/dist/pypy/tool Message-ID: <20060811153916.601B41005A@code0.codespeak.net> Author: mwh Date: Fri Aug 11 17:39:15 2006 New Revision: 31265 Added: pypy/dist/pypy/tool/rundictbenchmarks.py Modified: pypy/dist/pypy/tool/readdictinfo.py Log: stuff to make my benchmarking runs more reproducible. may not be terribly usable unless you're me. Modified: pypy/dist/pypy/tool/readdictinfo.py ============================================================================== --- pypy/dist/pypy/tool/readdictinfo.py (original) +++ pypy/dist/pypy/tool/readdictinfo.py Fri Aug 11 17:39:15 2006 @@ -84,7 +84,7 @@ print >>output if __name__ == '__main__': - reportDictInfos() +# reportDictInfos() # interactive stuff: @@ -105,3 +105,10 @@ import rlcompleter readline.parse_and_bind('tab: complete') + if len(sys.argv) > 2: + attrs = sys.argv[2].split(',') + if attrs == ['all']: + attrs = slots + Rify("R.txt", *attrs) + + Added: pypy/dist/pypy/tool/rundictbenchmarks.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/rundictbenchmarks.py Fri Aug 11 17:39:15 2006 @@ -0,0 +1,26 @@ +import sys, os + +# this file runs some benchmarks with a pypy-c that is assumed to be +# built using the MeasuringDictImplementation. + +# it should be run with pypy/translator/goal as the cwd, and you'll +# need to hack a copy of rst2html for yourself (svn docutils +# required). + +try: + os.unlink("dictinfo.txt") +except os.error: + pass + +progs = [('pystone', ['-c', 'from test import pystone; pystone.main()']), + ('richards', ['richards.py']), + ('docutils', ['rst2html.py', '../../doc/coding-guide.txt', 'foo.html']), + ('translate', ['translate.py', '--backendopt', '--no-compile', '--batch', + '--text', 'targetrpystonedalone.py']) + ] + +EXE = sys.argv[1] + +for suffix, args in progs: + os.spawnv(os.P_WAIT, EXE, [EXE] + args) + os.rename('dictinfo.txt', 'dictinfo-%s.txt'%suffix) From arigo at codespeak.net Fri Aug 11 19:30:34 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Aug 2006 19:30:34 +0200 (CEST) Subject: [pypy-svn] r31266 - in pypy/dist/pypy/rpython/rctypes: . test Message-ID: <20060811173034.034501006E@code0.codespeak.net> Author: arigo Date: Fri Aug 11 19:30:32 2006 New Revision: 31266 Modified: pypy/dist/pypy/rpython/rctypes/rfunc.py pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py Log: Bug and fix for indirect calls to functions with no return value. Modified: pypy/dist/pypy/rpython/rctypes/rfunc.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rfunc.py (original) +++ pypy/dist/pypy/rpython/rctypes/rfunc.py Fri Aug 11 19:30:32 2006 @@ -32,8 +32,11 @@ r = rtyper.getrepr(SomeCTypesObject(arg_ctype, ownsmemory=False)) args_r.append(r) - r_result = rtyper.getrepr(SomeCTypesObject(self.restype, - ownsmemory=True)) + if self.restype is not None: + r_result = rtyper.getrepr(SomeCTypesObject(self.restype, + ownsmemory=True)) + else: + r_result = None if isinstance(self.ll_type.TO, lltype.ForwardReference): FUNCTYPE = get_funcptr_type(args_r, r_result) self.ll_type.TO.become(FUNCTYPE) Modified: pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c (original) +++ pypy/dist/pypy/rpython/rctypes/test/_rctypes_test.c Fri Aug 11 19:30:32 2006 @@ -38,6 +38,14 @@ p->_z++; } +EXPORT(void) _testfunc_swap2(point *p) +{ + int tmp = p->x; + p->x = p->y; + p->y = tmp; + p->_z += 2; +} + EXPORT(int) _testfunc_struct(point in) { return in.x + in.y; Modified: pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rctypes.py Fri Aug 11 19:30:32 2006 @@ -98,6 +98,17 @@ testfunc_swap.llinterp_friendly_version = ll_testfunc_swap testfunc_swap.includes = includes +# _testfunc_swap2 +testfunc_swap2 = _rctypes_test._testfunc_swap2 +testfunc_swap2.restype = None +testfunc_swap2.argtypes = [tagpointptr] + +def ll_testfunc_swap2(p): + p.c_x, p.c_y = p.c_y, p.c_x + p.c__z += 2 +testfunc_swap2.llinterp_friendly_version = ll_testfunc_swap2 +testfunc_swap2.includes = includes + # _testfunc_erase_type testfunc_erase_type = _rctypes_test._testfunc_erase_type testfunc_erase_type.restype = c_void_p @@ -199,6 +210,25 @@ res = interpret(test_testfunc_swap, []) assert res == 4 + def test_specialize_indirect_call(self): + def f(n): + pt = tagpoint() + pt.x = 5 + pt.y = 9 + pt._z = 99 + if n > 0: + f = testfunc_swap + else: + f = testfunc_swap2 + f(pointer(pt)) + assert pt.x == 9 + assert pt.y == 5 + return pt._z + res = interpret(f, [42]) + assert res == 100 + res = interpret(f, [-42]) + assert res == 101 + class Test_compile: def test_compile_byval(self): fn = compile(test_testfunc_byval, []) From arigo at codespeak.net Sat Aug 12 13:39:26 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 Aug 2006 13:39:26 +0200 (CEST) Subject: [pypy-svn] r31267 - pypy/dist/pypy/translator Message-ID: <20060812113926.616B910071@code0.codespeak.net> Author: arigo Date: Sat Aug 12 13:39:24 2006 New Revision: 31267 Modified: pypy/dist/pypy/translator/geninterplevel.py Log: Some 2.5 compatibility. Modified: pypy/dist/pypy/translator/geninterplevel.py ============================================================================== --- pypy/dist/pypy/translator/geninterplevel.py (original) +++ pypy/dist/pypy/translator/geninterplevel.py Sat Aug 12 13:39:24 2006 @@ -758,7 +758,7 @@ metaclass = "space.w_type" name = self.uniquename('gcls_' + cls.__name__) - if issubclass(cls, Exception): + if issubclass(cls, py.builtin.BaseException): # if cls.__module__ == 'exceptions': # don't rely on this, py.magic redefines AssertionError if getattr(__builtin__,cls.__name__,None) is cls: @@ -893,7 +893,7 @@ if type(ret) is tuple: ret = ret[0](self, ret[1], ret[2]) return ret - if issubclass(cls, Exception): # Python 2.5 only + if issubclass(cls, py.builtin.BaseException): # Python 2.5 only # if cls.__module__ == 'exceptions': # don't rely on this, py.magic redefines AssertionError if getattr(__builtin__,cls.__name__,None) is cls: @@ -1336,7 +1336,7 @@ yield " e.normalize_exception(space)" q = "if" for link in block.exits[1:]: - assert issubclass(link.exitcase, Exception) + assert issubclass(link.exitcase, py.builtin.BaseException) # Exeption classes come unwrapped in link.exitcase yield " %s space.is_true(space.issubtype(e.w_type, %s)):" % (q, self.nameof(link.exitcase)) From arigo at codespeak.net Sat Aug 12 17:18:11 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 Aug 2006 17:18:11 +0200 (CEST) Subject: [pypy-svn] r31272 - pypy/dist/pypy/translator/c Message-ID: <20060812151811.565D410071@code0.codespeak.net> Author: arigo Date: Sat Aug 12 17:18:09 2006 New Revision: 31272 Modified: pypy/dist/pypy/translator/c/node.py Log: Hopefully fix broken genc tests and pypy-c translation. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Sat Aug 12 17:18:09 2006 @@ -292,6 +292,7 @@ class FixedSizeArrayDefNode: gcinfo = None name = None + typetag = 'struct' def __init__(self, db, FIXEDARRAY): self.db = db @@ -354,6 +355,7 @@ class ExtTypeOpaqueDefNode: "For OpaqueTypes created by pypy.rpython.extfunctable.ExtTypeInfo." + typetag = 'struct' def __init__(self, db, T): self.db = db From rhymes at codespeak.net Sat Aug 12 18:10:38 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 12 Aug 2006 18:10:38 +0200 (CEST) Subject: [pypy-svn] r31273 - pypy/dist/pypy/module/_ssl Message-ID: <20060812161038.5C32110071@code0.codespeak.net> Author: rhymes Date: Sat Aug 12 18:10:36 2006 New Revision: 31273 Modified: pypy/dist/pypy/module/_ssl/interp_ssl.py Log: Fix to make it compile... or not. The process hangs at backendopt:malloc stage sucking hundreds of megabytes of RAM. Modified: pypy/dist/pypy/module/_ssl/interp_ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/interp_ssl.py (original) +++ pypy/dist/pypy/module/_ssl/interp_ssl.py Sat Aug 12 18:10:36 2006 @@ -158,7 +158,7 @@ libssl.X509_get_issuer_name.argtypes = [POINTER(X509)] libssl.X509_get_issuer_name.restype = POINTER(X509_NAME) libssl.X509_NAME_oneline.argtypes = [POINTER(X509_NAME), arr_x509, c_int] -libssl.X509_NAME_oneline.restype = arr_x509 +libssl.X509_NAME_oneline.restype = c_char_p libssl.X509_free.argtypes = [POINTER(X509)] libssl.X509_free.restype = c_void libssl.SSL_free.argtypes = [POINTER(SSL)] @@ -409,12 +409,12 @@ space.wrap("SSL_CTX_use_PrivateKey_file error")) ret = libssl.SSL_CTX_use_certificate_chain_file(ss.ctx, cert_file) - libssl.SSL_CTX_ctrl(ss.ctx, SSL_CTRL_OPTIONS, SSL_OP_ALL, None) + libssl.SSL_CTX_ctrl(ss.ctx, SSL_CTRL_OPTIONS, SSL_OP_ALL, c_void_p()) if ret < 1: raise OperationError(space.w_Exception, space.wrap("SSL_CTX_use_certificate_chain_file error")) - libssl.SSL_CTX_set_verify(ss.ctx, SSL_VERIFY_NONE, None) # set verify level + libssl.SSL_CTX_set_verify(ss.ctx, SSL_VERIFY_NONE, c_void_p()) # set verify level ss.ssl = libssl.SSL_new(ss.ctx) # new ssl struct libssl.SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL @@ -422,8 +422,8 @@ # to non-blocking mode (blocking is the default) if has_timeout: # Set both the read and write BIO's to non-blocking mode - libssl.BIO_ctrl(libssl.SSL_get_rbio(ss.ssl), BIO_C_SET_NBIO, 1, None) - libssl.BIO_ctrl(libssl.SSL_get_wbio(ss.ssl), BIO_C_SET_NBIO, 1, None) + libssl.BIO_ctrl(libssl.SSL_get_rbio(ss.ssl), BIO_C_SET_NBIO, 1, c_void_p()) + libssl.BIO_ctrl(libssl.SSL_get_wbio(ss.ssl), BIO_C_SET_NBIO, 1, c_void_p()) libssl.SSL_set_connect_state(ss.ssl) # Actually negotiate SSL connection @@ -461,8 +461,6 @@ errstr, errval = _ssl_seterror(space, ss, ret) raise OperationError(space.w_Exception, space.wrap("%s: %d" % (errstr, errval))) - - ss.ssl.debug = 1 ss.server_cert = libssl.SSL_get_peer_certificate(ss.ssl) if ss.server_cert: From rhymes at codespeak.net Sat Aug 12 18:31:59 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 12 Aug 2006 18:31:59 +0200 (CEST) Subject: [pypy-svn] r31274 - pypy/dist/pypy/module/bz2 Message-ID: <20060812163159.825FE10071@code0.codespeak.net> Author: rhymes Date: Sat Aug 12 18:31:57 2006 New Revision: 31274 Modified: pypy/dist/pypy/module/bz2/fileobject.py pypy/dist/pypy/module/bz2/interp_bz2.py Log: more fixes to please the ext. compiler Modified: pypy/dist/pypy/module/bz2/fileobject.py ============================================================================== --- pypy/dist/pypy/module/bz2/fileobject.py (original) +++ pypy/dist/pypy/module/bz2/fileobject.py Sat Aug 12 18:31:57 2006 @@ -111,6 +111,7 @@ assert alignment(__sbuf) == 4, alignment(__sbuf) class __sFILEX(Structure): pass +__sFILEX._fields_ = [] class __sFILE(Structure): pass __sFILE._pack_ = 4 @@ -846,10 +847,13 @@ allocfunc = CFUNCTYPE(POINTER(PyObject), POINTER(_typeobject), c_long) class PyMethodDef(Structure): pass +PyMethodDef._fields_ = [] class PyMemberDef(Structure): pass +PyMemberDef._fields_ = [] class PyGetSetDef(Structure): pass +PyGetSetDef._fields_ = [] _typeobject._fields_ = [ ('ob_refcnt', Py_ssize_t), ('ob_type', POINTER(_typeobject)), Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Sat Aug 12 18:31:57 2006 @@ -250,7 +250,7 @@ skipnextlf = obj.f_skipnextlf univ_newline = obj.f_univ_newline - total_v_size = (100, size)[size > 0] # total no. of slots in buffer + total_v_size = [100, size][size > 0] # total no. of slots in buffer buf_lst = [] buf_pos = 0 @@ -392,7 +392,7 @@ if mode_char == "": mode_char = 'r' - mode = ('wb', 'rb')[mode_char == 'r'] + mode = ['wb', 'rb'][mode_char == 'r'] self.mode_string = mode # open the file and set the buffer @@ -409,7 +409,7 @@ bzerror = c_int() if mode_char == 'r': self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, - 0, 0, None, 0) + 0, 0, c_void_p(), 0) else: self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, compresslevel, 0, 0) @@ -417,7 +417,7 @@ if bzerror.value != BZ_OK: _catch_bz2_error(self.space, bzerror.value) - self.mode = (MODE_WRITE, MODE_READ)[mode_char == 'r'] + self.mode = [MODE_WRITE, MODE_READ][mode_char == 'r'] def __del__(self): bzerror = c_int() @@ -591,7 +591,7 @@ if size == 0: return self.space.wrap("") else: - size = (size, 0)[size < 0] + size = [size, 0][size < 0] return self.space.wrap(_getline(self.space, self, size)) readline.unwrap_spec = ['self', int] @@ -609,7 +609,7 @@ raise OperationError(self.space.w_IOError, self.space.wrap("file is not ready for reading")) - bufsize = (size, _new_buffer_size(0))[size < 0] + bufsize = [size, _new_buffer_size(0)][size < 0] if bufsize > MAXINT: raise OperationError(self.space.w_OverflowError, @@ -818,9 +818,9 @@ in_buf = create_string_buffer(in_bufsize) in_buf.value = data - self.bzs.next_in = in_buf + self.bzs.next_in = cast(in_buf, POINTER(c_char)) self.bzs.avail_in = in_bufsize - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize temp = [] @@ -838,7 +838,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize if temp: @@ -861,7 +861,7 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize total_out = _bzs_total_out(self.bzs) @@ -880,7 +880,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize if temp: @@ -944,9 +944,9 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - self.bzs.next_in = in_buf + self.bzs.next_in = cast(in_buf, POINTER(c_char)) self.bzs.avail_in = in_bufsize - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize temp = [] @@ -970,7 +970,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - self.bzs.next_out = out_buf + self.bzs.next_out = cast(out_buf, POINTER(c_char)) self.bzs.avail_out = out_bufsize if temp: @@ -1014,9 +1014,9 @@ in_buf = create_string_buffer(in_bufsize) in_buf.value = data - bzs.next_in = in_buf + bzs.next_in = cast(in_buf, POINTER(c_char)) bzs.avail_in = in_bufsize - bzs.next_out = out_buf + bzs.next_out = cast(out_buf, POINTER(c_char)) bzs.avail_out = out_bufsize bzerror = libbz2.BZ2_bzCompressInit(byref(bzs), compresslevel, 0, 0) @@ -1039,7 +1039,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - bzs.next_out = out_buf + bzs.next_out = cast(out_buf, POINTER(c_char)) bzs.avail_out = out_bufsize if temp: @@ -1074,9 +1074,9 @@ out_bufsize = SMALLCHUNK out_buf = create_string_buffer(out_bufsize) - bzs.next_in = in_buf + bzs.next_in = cast(in_buf, POINTER(c_char)) bzs.avail_in = in_bufsize - bzs.next_out = out_buf + bzs.next_out = cast(out_buf, POINTER(c_char)) bzs.avail_out = out_bufsize bzerror = libbz2.BZ2_bzDecompressInit(byref(bzs), 0, 0) @@ -1103,7 +1103,7 @@ out_bufsize = _new_buffer_size(out_bufsize) out_buf = create_string_buffer(out_bufsize) - bzs.next_out = out_buf + bzs.next_out = cast(out_buf, POINTER(c_char)) bzs.avail_out = out_bufsize total_out = _bzs_total_out(bzs) From rxe at codespeak.net Mon Aug 14 04:59:11 2006 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 14 Aug 2006 04:59:11 +0200 (CEST) Subject: [pypy-svn] r31281 - pypy/dist/pypy/translator/llvm/test Message-ID: <20060814025911.5BF0010086@code0.codespeak.net> Author: rxe Date: Mon Aug 14 04:59:06 2006 New Revision: 31281 Modified: pypy/dist/pypy/translator/llvm/test/runtest.py Log: add ability to not run in isolated mode (useful for debugging) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Mon Aug 14 04:59:06 2006 @@ -9,6 +9,7 @@ # test options run_isolated_only = True +do_not_isolate = False def _cleanup(leave=0): # no test should ever need more than 5 compiled functions @@ -39,10 +40,13 @@ return True def compile_test(function, annotation, isolate=True, **kwds): - " returns module and compiled function " + " returns module and compiled function " if llvm_test(): if run_isolated_only and not isolate: py.test.skip("skipping not isolated test") + + # turn off isolation? + isolate = isolate and not do_not_isolate # maintain only 3 isolated process (if any) _cleanup(leave=3) From rxe at codespeak.net Mon Aug 14 04:59:55 2006 From: rxe at codespeak.net (rxe at codespeak.net) Date: Mon, 14 Aug 2006 04:59:55 +0200 (CEST) Subject: [pypy-svn] r31282 - in pypy/dist/pypy/translator/llvm: module test Message-ID: <20060814025955.70E9D1008B@code0.codespeak.net> Author: rxe Date: Mon Aug 14 04:59:44 2006 New Revision: 31282 Modified: pypy/dist/pypy/translator/llvm/module/protos.h pypy/dist/pypy/translator/llvm/test/test_extfunc.py Log: copy across test from genc and make a fix. Modified: pypy/dist/pypy/translator/llvm/module/protos.h ============================================================================== --- pypy/dist/pypy/translator/llvm/module/protos.h (original) +++ pypy/dist/pypy/translator/llvm/module/protos.h Mon Aug 14 04:59:44 2006 @@ -13,6 +13,7 @@ RPyMODF_RESULT *ll_modf_result(double, double); RPySTAT_RESULT *ll_stat_result(int, int, int, int, int, int, int, int, int, int); RPyPIPE_RESULT *ll_pipe_result(int, int); +RPyWAITPID_RESULT *ll_waitpid_result(long, long); void RPYTHON_RAISE_OSERROR(int error); #ifdef RPyListOfString Modified: pypy/dist/pypy/translator/llvm/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_extfunc.py Mon Aug 14 04:59:44 2006 @@ -578,4 +578,17 @@ assert res == 100110 assert os.path.islink(tmpfile2) assert not os.path.islink(tmpfile3) +if hasattr(os, 'fork'): + def test_fork(): + def does_stuff(): + pid = os.fork() + if pid == 0: # child + os._exit(4) + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + return status1 + f1 = compile_function(does_stuff, []) + status1 = f1() + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == 4 From rhymes at codespeak.net Mon Aug 14 10:31:31 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 14 Aug 2006 10:31:31 +0200 (CEST) Subject: [pypy-svn] r31284 - pypy/dist/pypy/translator/cl/test Message-ID: <20060814083131.EDB7F10090@code0.codespeak.net> Author: rhymes Date: Mon Aug 14 10:31:29 2006 New Revision: 31284 Modified: pypy/dist/pypy/translator/cl/test/test_list.py pypy/dist/pypy/translator/cl/test/test_tuple.py Log: make these tests skip for real Modified: pypy/dist/pypy/translator/cl/test/test_list.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_list.py (original) +++ pypy/dist/pypy/translator/cl/test/test_list.py Mon Aug 14 10:31:29 2006 @@ -1,4 +1,5 @@ from pypy.translator.cl.buildcl import make_cl_func +from py.test import skip def sum_(l): i = 0 @@ -21,7 +22,7 @@ assert cl_list_get(1985) == 1985 def test_list_iter(): - py.test.skip("CDefinedInt implementation") + skip("CDefinedInt implementation") def list_iter(): a = 0 for item in [1,2,3,4,5]: @@ -47,7 +48,7 @@ assert cl_list_concat() == 8 def test_list_setitem(): - py.test.skip("CDefinedInt implementation") + skip("CDefinedInt implementation") def list_setitem(num): a = [1,2,3,4] a[1] = num Modified: pypy/dist/pypy/translator/cl/test/test_tuple.py ============================================================================== --- pypy/dist/pypy/translator/cl/test/test_tuple.py (original) +++ pypy/dist/pypy/translator/cl/test/test_tuple.py Mon Aug 14 10:31:29 2006 @@ -1,4 +1,5 @@ from pypy.translator.cl.buildcl import make_cl_func +from py.test import skip def test_tuple_get(): def tuple_get(number): @@ -8,7 +9,7 @@ assert cl_tuple_get(1) == 1 def test_tuple_iter(): - py.test.skip("CDefinedInt implementation") + skip("CDefinedInt implementation") def tuple_double(number): tuple = (number,) for item in tuple: From arigo at codespeak.net Mon Aug 14 14:23:54 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 Aug 2006 14:23:54 +0200 (CEST) Subject: [pypy-svn] r31289 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20060814122354.541AF10092@code0.codespeak.net> Author: arigo Date: Mon Aug 14 14:23:52 2006 New Revision: 31289 Modified: pypy/dist/pypy/translator/backendopt/constfold.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py Log: constfold has the same bug as propagate fixed long ago: constant-folding getsubstruct might lead to the last ref to the parent GcStruct forgotten. This makes the GcStruct go away, and the back-end cannot compile it any longer. This could be fixed by making the back-ends smarter - after all, the parent GcStruct is not really needed - but that's a mess that we would have to repeat in each back-end. Instead, a quick hack does the job: in this situation, give the substructure a strong reference to its parent to keep the latter alive. Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Mon Aug 14 14:23:52 2006 @@ -43,6 +43,8 @@ log.WARNING(' %s: %s' % (e.__class__.__name__, e)) else: # success in folding this space operation + if spaceop.opname in fixup_op_result: + result = fixup_op_result[spaceop.opname](result) constants[spaceop.result] = Constant(result, RESTYPE) folded_count += 1 continue @@ -97,6 +99,25 @@ link.args = [constants.get(v, v) for v in link.args] +# +# Operations returning pointers to inlined parts of a constant object +# have to be tweaked so that the inlined part keeps the whole object alive. +# XXX This is done with a hack. (See test_keepalive_const_*()) +# +def fixup_solid(p): + container = p._obj + assert isinstance(container, lltype._parentable) + container._keepparent = container._parentstructure() + return p + +fixup_op_result = { + "getsubstruct": fixup_solid, + "getarraysubstruct": fixup_solid, + "direct_fieldptr": fixup_solid, + "direct_arrayitems": fixup_solid, + } + + def complete_constants(link, constants): # 'constants' maps some Variables of 'block' to Constants. # Some input args of 'block' may be absent from 'constants' Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Mon Aug 14 14:23:52 2006 @@ -170,3 +170,65 @@ 'int_sub': 1} check_graph(graph, [-1], 124, t) check_graph(graph, [0], 61, t) + + +def test_keepalive_const_substruct(): + S2 = lltype.Struct('S2', ('x', lltype.Signed)) + S1 = lltype.GcStruct('S1', ('sub', S2)) + s1 = lltype.malloc(S1) + s1.sub.x = 1234 + def fn(): + return s1.sub.x + graph, t = get_graph(fn, []) + + # kill all references to 's1' + s1 = fn = None + del graph.func + import gc; gc.collect() + + assert summary(graph) == {'getsubstruct': 1, 'getfield': 1} + constant_fold_graph(graph) + assert summary(graph) == {'getfield': 1} + check_graph(graph, [], 1234, t) + + +def test_keepalive_const_fieldptr(): + S1 = lltype.GcStruct('S1', ('x', lltype.Signed)) + s1 = lltype.malloc(S1) + s1.x = 1234 + def fn(): + p1 = lltype.direct_fieldptr(s1, 'x') + return p1[0] + graph, t = get_graph(fn, []) + + # kill all references to 's1' + s1 = fn = None + del graph.func + import gc; gc.collect() + + assert summary(graph) == {'direct_fieldptr': 1, 'getarrayitem': 1} + constant_fold_graph(graph) + assert summary(graph) == {'getarrayitem': 1} + check_graph(graph, [], 1234, t) + + +def test_keepalive_const_arrayitems(): + A1 = lltype.GcArray(lltype.Signed) + a1 = lltype.malloc(A1, 10) + a1[6] = 1234 + def fn(): + p1 = lltype.direct_arrayitems(a1) + p2 = lltype.direct_ptradd(p1, 6) + return p2[0] + graph, t = get_graph(fn, []) + + # kill all references to 'a1' + a1 = fn = None + del graph.func + import gc; gc.collect() + + assert summary(graph) == {'direct_arrayitems': 1, 'direct_ptradd': 1, + 'getarrayitem': 1} + constant_fold_graph(graph) + assert summary(graph) == {'getarrayitem': 1} + check_graph(graph, [], 1234, t) From arigo at codespeak.net Mon Aug 14 15:10:19 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 Aug 2006 15:10:19 +0200 (CEST) Subject: [pypy-svn] r31291 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20060814131019.D395E10092@code0.codespeak.net> Author: arigo Date: Mon Aug 14 15:10:18 2006 New Revision: 31291 Modified: pypy/dist/pypy/translator/backendopt/constfold.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py Log: Fix the constfold tests, and fix the bug shown by said fix. Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Mon Aug 14 15:10:18 2006 @@ -99,16 +99,16 @@ link.args = [constants.get(v, v) for v in link.args] -# -# Operations returning pointers to inlined parts of a constant object -# have to be tweaked so that the inlined part keeps the whole object alive. -# XXX This is done with a hack. (See test_keepalive_const_*()) -# def fixup_solid(p): + # Operations returning pointers to inlined parts of a constant object + # have to be tweaked so that the inlined part keeps the whole object alive. + # XXX This is done with a hack. (See test_keepalive_const_*()) container = p._obj assert isinstance(container, lltype._parentable) container._keepparent = container._parentstructure() - return p + # Instead of 'p', return a solid pointer, to keep the inlined part + # itself alive. + return container._as_ptr() fixup_op_result = { "getsubstruct": fixup_solid, Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Mon Aug 14 15:10:18 2006 @@ -180,14 +180,14 @@ def fn(): return s1.sub.x graph, t = get_graph(fn, []) + assert summary(graph) == {'getsubstruct': 1, 'getfield': 1} + constant_fold_graph(graph) # kill all references to 's1' s1 = fn = None del graph.func import gc; gc.collect() - assert summary(graph) == {'getsubstruct': 1, 'getfield': 1} - constant_fold_graph(graph) assert summary(graph) == {'getfield': 1} check_graph(graph, [], 1234, t) @@ -200,14 +200,14 @@ p1 = lltype.direct_fieldptr(s1, 'x') return p1[0] graph, t = get_graph(fn, []) + assert summary(graph) == {'direct_fieldptr': 1, 'getarrayitem': 1} + constant_fold_graph(graph) # kill all references to 's1' s1 = fn = None del graph.func import gc; gc.collect() - assert summary(graph) == {'direct_fieldptr': 1, 'getarrayitem': 1} - constant_fold_graph(graph) assert summary(graph) == {'getarrayitem': 1} check_graph(graph, [], 1234, t) @@ -221,14 +221,14 @@ p2 = lltype.direct_ptradd(p1, 6) return p2[0] graph, t = get_graph(fn, []) + assert summary(graph) == {'direct_arrayitems': 1, 'direct_ptradd': 1, + 'getarrayitem': 1} + constant_fold_graph(graph) # kill all references to 'a1' a1 = fn = None del graph.func import gc; gc.collect() - assert summary(graph) == {'direct_arrayitems': 1, 'direct_ptradd': 1, - 'getarrayitem': 1} - constant_fold_graph(graph) assert summary(graph) == {'getarrayitem': 1} check_graph(graph, [], 1234, t) From arigo at codespeak.net Mon Aug 14 16:56:07 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 Aug 2006 16:56:07 +0200 (CEST) Subject: [pypy-svn] r31297 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20060814145607.3A0D910092@code0.codespeak.net> Author: arigo Date: Mon Aug 14 16:56:05 2006 New Revision: 31297 Modified: pypy/dist/pypy/module/__builtin__/app_inspect.py pypy/dist/pypy/module/__builtin__/test/test_builtin.py Log: hasattr() must raise TypeError if the 2nd arg is not a string or unicode. Modified: pypy/dist/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/dist/pypy/module/__builtin__/app_inspect.py Mon Aug 14 16:56:05 2006 @@ -85,6 +85,13 @@ try: getattr(obj, attr) return True + except TypeError: + # if 'attr' was not a string or unicode, let the TypeError through, + # else eat it + if isinstance(attr, basestring): + return False + else: + raise except (KeyboardInterrupt, SystemExit): raise except: Modified: pypy/dist/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_builtin.py Mon Aug 14 16:56:05 2006 @@ -419,14 +419,19 @@ def test_hasattr(self): class X(object): - def broken(): pass + def broken(): pass # TypeError abc = property(broken) + def broken2(): raise IOError + bac = property(broken2) x = X() x.foo = 42 assert hasattr(x, '__class__') assert hasattr(x, 'foo') assert not hasattr(x, 'bar') assert not hasattr(x, 'abc') # CPython compliance + assert not hasattr(x, 'bac') # CPython compliance + raises(TypeError, hasattr, x, None) + raises(TypeError, hasattr, x, 42) class TestInternal: From arigo at codespeak.net Mon Aug 14 17:40:03 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 Aug 2006 17:40:03 +0200 (CEST) Subject: [pypy-svn] r31298 - in pypy/dist/pypy: module/posix module/sys rpython translator/goal Message-ID: <20060814154003.4ADEA1006F@code0.codespeak.net> Author: arigo Date: Mon Aug 14 17:40:00 2006 New Revision: 31298 Modified: pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/sys/__init__.py pypy/dist/pypy/module/sys/state.py pypy/dist/pypy/rpython/ros.py pypy/dist/pypy/translator/goal/app_main.py Log: issue184 testing Build a pypy-c that should be movable to a new location or a new machine. It looks for its library paths (lib-python and pypy/lib) by walking dirs upwards until it finds them. It also looks if these paths can be found in a subdirectory "share/pypy-0.9" along the way. Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Aug 14 17:40:00 2006 @@ -249,9 +249,8 @@ s = ros.environ(idx) if s is None: break - p = s.find('='); + p = s.find('=') if p >= 0: - assert p >= 0 key = s[:p] value = s[p+1:] space.setitem(w_env, space.wrap(key), space.wrap(value)) Modified: pypy/dist/pypy/module/sys/__init__.py ============================================================================== --- pypy/dist/pypy/module/sys/__init__.py (original) +++ pypy/dist/pypy/module/sys/__init__.py Mon Aug 14 17:40:00 2006 @@ -35,6 +35,7 @@ 'builtin_module_names' : 'state.w_None', 'pypy_getudir' : 'state.pypy_getudir', 'pypy_repr' : 'state.pypy_repr', + 'pypy_initial_path' : 'state.pypy_initial_path', '_getframe' : 'vm._getframe', 'setrecursionlimit' : 'vm.setrecursionlimit', Modified: pypy/dist/pypy/module/sys/state.py ============================================================================== --- pypy/dist/pypy/module/sys/state.py (original) +++ pypy/dist/pypy/module/sys/state.py Mon Aug 14 17:40:00 2006 @@ -3,8 +3,9 @@ """ import pypy from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import ObjSpace -import sys, os +import sys, os, stat, errno # ____________________________________________________________ # @@ -23,20 +24,53 @@ # Initialize the default path from pypy.interpreter import autopath srcdir = os.path.dirname(autopath.pypydir) - python_std_lib = os.path.normpath( - os.path.join(autopath.pypydir, os.pardir,'lib-python', '2.4.1')) - python_std_lib_modified = os.path.normpath( - os.path.join(autopath.pypydir, os.pardir,'lib-python', 'modified-2.4.1')) - - pypy_lib = os.path.join(autopath.pypydir, 'lib') - assert os.path.exists(python_std_lib) - assert os.path.exists(python_std_lib_modified) - importlist = [''] - for p in os.environ.get('PYTHONPATH', '').split(':'): - if p: - importlist.append(p) - importlist.extend([pypy_lib, python_std_lib_modified, python_std_lib]) - self.w_path = space.newlist([space.wrap(x) for x in importlist]) + path = getinitialpath(srcdir) + self.w_path = space.newlist([space.wrap(p) for p in path]) + +def checkdir(path): + st = os.stat(path) + if not stat.S_ISDIR(st[0]): + raise OSError(errno.ENOTDIR, path) + +def getinitialpath(srcdir): + # build the initial path from the srcdir, which is the path of + # the "dist" directory of a PyPy checkout. + from pypy.module.sys.version import CPYTHON_VERSION + from pypy.rpython import ros + + dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], + CPYTHON_VERSION[1], + CPYTHON_VERSION[2]) + lib_python = os.path.join(srcdir, 'lib-python') + + python_std_lib = os.path.join(lib_python, dirname) + checkdir(python_std_lib) + python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname) + checkdir(python_std_lib_modified) + pypydir = os.path.join(srcdir, 'pypy') + pypy_lib = os.path.join(pypydir, 'lib') + checkdir(pypy_lib) + + importlist = [''] + pythonpath = ros.getenv('PYTHONPATH') + if pythonpath: + for p in pythonpath.split(os.pathsep): + if p: + importlist.append(p) + importlist.append(pypy_lib) + importlist.append(python_std_lib_modified) + importlist.append(python_std_lib) + return importlist + +def pypy_initial_path(space, srcdir): + try: + path = getinitialpath(srcdir) + except OSError: + return space.w_None + else: + return space.newlist([space.wrap(p) for p in path]) + +pypy_initial_path.unwrap_spec = [ObjSpace, str] def get(space): return space.fromcache(State) Modified: pypy/dist/pypy/rpython/ros.py ============================================================================== --- pypy/dist/pypy/rpython/ros.py (original) +++ pypy/dist/pypy/rpython/ros.py Mon Aug 14 17:40:00 2006 @@ -16,6 +16,20 @@ if idx < len(_initial_items): return '%s=%s' % _initial_items[idx] +def getenv(name): + # slowish, ok for non-repeated use + pattern = name + '=' + idx = 0 + while 1: + s = environ(idx) + if s is None: + break + if s.startswith(pattern): + value = s[len(pattern):] + return value + idx += 1 + return None + class DIR(object): # a simulated DIR structure from C, i.e. a directory opened by Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Mon Aug 14 17:40:00 2006 @@ -1,3 +1,4 @@ +#! /usr/bin/env python # App-level version of py.py. # XXX this is probably still incomplete. """ @@ -10,7 +11,7 @@ --info print translation information about this PyPy executable """ -import sys +import sys, os originalexcepthook = sys.__excepthook__ @@ -126,8 +127,40 @@ # ____________________________________________________________ # Main entry point +AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' + def entry_point(executable, argv): - sys.executable = executable + # find the full path to the executable, assuming that if there is no '/' + # in the provided one then we must look along the $PATH + if os.sep not in executable: + path = os.getenv('PATH') + if path: + for dir in path.split(os.pathsep): + fn = os.path.join(dir, executable) + if os.path.isfile(fn): + executable = fn + break + sys.executable = os.path.abspath(executable) + + # set up a sys.path that depends on the local machine + autosubpath = AUTOSUBPATH % sys.pypy_version_info[:2] + search = executable + while 1: + dirname = resolvedirof(search) + if dirname == search: + # not found! let's hope that the compiled-in path is ok + print >> sys.stderr, ('debug: WARNING: library path not found, ' + 'using compiled-in sys.path') + break + newpath = sys.pypy_initial_path(dirname) + if newpath is None: + newpath = sys.pypy_initial_path(os.path.join(dirname, autosubpath)) + if newpath is None: + search = dirname # walk to the parent directory + continue + sys.path = newpath # found! + break + go_interactive = False i = 0 while i < len(argv): @@ -173,9 +206,7 @@ exec cmd in mainmodule.__dict__ run_toplevel(run_it) else: - import os - # XXX resolve symlinks - scriptdir = os.path.dirname(os.path.abspath(sys.argv[0])) + scriptdir = resolvedirof(sys.argv[0]) sys.path.insert(0, scriptdir) run_toplevel(execfile, sys.argv[0], mainmodule.__dict__) else: @@ -190,9 +221,36 @@ else: return 0 +def resolvedirof(filename): + try: + filename = os.path.abspath(filename) + except OSError: + pass + dirname = os.path.dirname(filename) + if os.path.islink(filename): + try: + link = os.readlink(filename) + except OSError: + pass + else: + return resolvedirof(os.path.join(dirname, link)) + return dirname + if __name__ == '__main__': # obscure! try removing the following line, see how it crashes, and # guess why... ImStillAroundDontForgetMe = sys.modules['__main__'] + # debugging only - sys.exit(entry_point(sys.argv[0], sys.argv[1:])) + def pypy_initial_path(s): + from pypy.module.sys.state import getinitialpath + try: + return getinitialpath(s) + except OSError: + return None + + from pypy.module.sys.version import PYPY_VERSION + sys.pypy_version_info = PYPY_VERSION + sys.pypy_initial_path = pypy_initial_path + #sys.exit(entry_point(sys.argv[0], sys.argv[1:])) + sys.exit(entry_point('app_main.py', sys.argv[1:])) From mwh at codespeak.net Mon Aug 14 18:47:26 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 14 Aug 2006 18:47:26 +0200 (CEST) Subject: [pypy-svn] r31300 - pypy/dist/pypy/config Message-ID: <20060814164726.E3A3210071@code0.codespeak.net> Author: mwh Date: Mon Aug 14 18:47:25 2006 New Revision: 31300 Modified: pypy/dist/pypy/config/pypyoption.py Log: add a config option for the measuring dicts. not used by anything yet... Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Mon Aug 14 18:47:25 2006 @@ -70,7 +70,12 @@ BoolOption("withmultidict", "use dictionaries optimized for flexibility", - default=False), + default=False, requires=[("withstrdict", False)]), + + BoolOption("withdictmeasurement", + "create huge files with masses of information " + "about dictionaries", + default=False, requires=[("withmultidict", True)]), BoolOption("oldstyle", "specify whether the default metaclass should be classobj", From mwh at codespeak.net Mon Aug 14 19:40:38 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 14 Aug 2006 19:40:38 +0200 (CEST) Subject: [pypy-svn] r31303 - in pypy/dist/pypy: . annotation interpreter interpreter/pyparser interpreter/test module/__builtin__ module/__builtin__/test module/_pickle_support module/posix module/sys module/thread objspace/cclp objspace/cclp/constraint objspace/constraint objspace/cpy objspace/cpy/test objspace/flow objspace/flow/test objspace/std objspace/std/test rpython tool/pytest tool/test translator translator/goal translator/pyrex Message-ID: <20060814174038.780DD10075@code0.codespeak.net> Author: mwh Date: Mon Aug 14 19:40:34 2006 New Revision: 31303 Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/conftest.py pypy/dist/pypy/interpreter/argument.py pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/eval.py pypy/dist/pypy/interpreter/executioncontext.py pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/module.py pypy/dist/pypy/interpreter/pycode.py pypy/dist/pypy/interpreter/pycompiler.py pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/interpreter/pyparser/pythonparse.py pypy/dist/pypy/interpreter/test/test_argument.py pypy/dist/pypy/interpreter/test/test_compiler.py pypy/dist/pypy/interpreter/test/test_eval.py pypy/dist/pypy/interpreter/test/test_function.py pypy/dist/pypy/interpreter/test/test_gateway.py pypy/dist/pypy/interpreter/test/test_objspace.py pypy/dist/pypy/interpreter/typedef.py pypy/dist/pypy/module/__builtin__/compiling.py pypy/dist/pypy/module/__builtin__/test/test_builtin.py pypy/dist/pypy/module/__builtin__/test/test_import.py pypy/dist/pypy/module/_pickle_support/maker.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/sys/state.py pypy/dist/pypy/module/thread/os_local.py pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/constraint/constraint.py pypy/dist/pypy/objspace/cpy/objspace.py pypy/dist/pypy/objspace/cpy/test/test_objspace.py pypy/dist/pypy/objspace/cpy/test/test_typedef.py pypy/dist/pypy/objspace/flow/objspace.py pypy/dist/pypy/objspace/flow/test/test_framestate.py pypy/dist/pypy/objspace/std/marshal_impl.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/stdtypedef.py pypy/dist/pypy/objspace/std/test/test_dictobject.py pypy/dist/pypy/objspace/std/typeobject.py pypy/dist/pypy/rpython/callparse.py pypy/dist/pypy/tool/pytest/appsupport.py pypy/dist/pypy/tool/test/test_pytestsupport.py pypy/dist/pypy/translator/geninterplevel.py pypy/dist/pypy/translator/goal/targetlogicstandalone.py pypy/dist/pypy/translator/goal/targetmultiplespaces.py pypy/dist/pypy/translator/goal/targetpypymain.py pypy/dist/pypy/translator/goal/targetpypystandalone.py pypy/dist/pypy/translator/goal/targetthunkstandalone.py pypy/dist/pypy/translator/pyrex/genpyrex.py Log: remove the extremely-rarely-not-[] argument from the newdict operation. Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Mon Aug 14 19:40:34 2006 @@ -705,12 +705,8 @@ def consider_op_newlist(self, *args): return self.bookkeeper.newlist(*args) - def consider_op_newdict(self, *args): - assert len(args) % 2 == 0 - items_s = [] - for i in range(0, len(args), 2): - items_s.append((args[i], args[i+1])) - return self.bookkeeper.newdict(*items_s) + def consider_op_newdict(self): + return self.bookkeeper.newdict() def consider_op_newslice(self, start, stop, step): self.bookkeeper.count('newslice', start, stop, step) Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Mon Aug 14 19:40:34 2006 @@ -284,14 +284,10 @@ self.dictdefs[self.position_key] = dictdef return dictdef - def newdict(self, *items_s): - """Make a SomeDict associated with the current position, general - enough to contain the given (s_key, s_value) as items.""" - dictdef = self.getdictdef() - for s_key, s_value in items_s: - dictdef.generalize_key(s_key) - dictdef.generalize_value(s_value) - return SomeDict(dictdef) + def newdict(self): + """Make a so-far empty SomeDict associated with the current + position.""" + return SomeDict(self.getdictdef()) def immutableconstant(self, const): return self.immutablevalue(const.value) @@ -801,7 +797,7 @@ else: return SomeTuple(items_s) - def newdict(self, stuff): + def newdict(self): raise CallPatternTooComplex, "'**' argument" def unpackiterable(self, s_obj, expected_length=None): Modified: pypy/dist/pypy/conftest.py ============================================================================== --- pypy/dist/pypy/conftest.py (original) +++ pypy/dist/pypy/conftest.py Mon Aug 14 19:40:34 2006 @@ -316,5 +316,5 @@ w_class = space.call_function(space.w_type, space.wrap(clsname), space.newtuple([]), - space.newdict([])) + space.newdict()) self.w_class = w_class Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Mon Aug 14 19:40:34 2006 @@ -284,7 +284,7 @@ co_argcount += 1 if has_kwarg: - scope_w[co_argcount] = self.space.newdict([]) + scope_w[co_argcount] = self.space.newdict() co_argcount += 1 return co_argcount @@ -519,7 +519,7 @@ # collect extra keyword arguments into the **kwarg if has_kwarg: - w_kwds = self.space.newdict([]) + w_kwds = self.space.newdict() if remainingkwds_w: for key, w_value in remainingkwds_w.items(): self.space.setitem(w_kwds, self.space.wrap(key), w_value) Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Mon Aug 14 19:40:34 2006 @@ -692,7 +692,7 @@ source = source.lstrip() assert source.startswith('('), "incorrect header in:\n%s" % (source,) source = py.code.Source("def anonymous%s\n" % source) - w_glob = space.newdict([]) + w_glob = space.newdict() space.exec_(source.compile(), w_glob, w_glob) return space.getitem(w_glob, space.wrap('anonymous')) @@ -833,8 +833,8 @@ # newtuple([w_1, w_2,...]) -> w_tuple # newlist([w_1, w_2,...]) -> w_list # newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) -# newunicode([i1, i2,...]) -> w_unicode from integers -# newdict([(w_key,w_value),...]) -> w_dict +# newunicode([i1, i2,...]) -> w_unicode from integers +# newdict() -> empty w_dict # newslice(w_start,w_stop,w_step) -> w_slice # call_args(w_obj,Arguments()) -> w_result Modified: pypy/dist/pypy/interpreter/eval.py ============================================================================== --- pypy/dist/pypy/interpreter/eval.py (original) +++ pypy/dist/pypy/interpreter/eval.py Mon Aug 14 19:40:34 2006 @@ -119,7 +119,7 @@ def fast2locals(self): # Copy values from self.fastlocals_w to self.w_locals if self.w_locals is None: - self.w_locals = self.space.newdict([]) + self.w_locals = self.space.newdict() varnames = self.getcode().getvarnames() fastscope_w = self.getfastscope() for i in range(min(len(varnames), len(fastscope_w))): Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Mon Aug 14 19:40:34 2006 @@ -88,7 +88,8 @@ "Create a new empty 'globals' dictionary." w_key = self.space.wrap("__builtins__") w_value = self.space.wrap(self.get_builtin()) - w_globals = self.space.newdict([(w_key, w_value)]) + w_globals = self.space.newdict() + space.setitem(w_globals, w_key, w_value) return w_globals def call_trace(self, frame): Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Mon Aug 14 19:40:34 2006 @@ -121,7 +121,7 @@ def getdict(self): if self.w_func_dict is None: - self.w_func_dict = self.space.newdict([]) + self.w_func_dict = self.space.newdict() return self.w_func_dict def setdict(self, space, w_dict): Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Mon Aug 14 19:40:34 2006 @@ -828,7 +828,7 @@ def build_applevel_dict(self, space): "NOT_RPYTHON" from pypy.interpreter.pycode import PyCode - w_glob = space.newdict([]) + w_glob = space.newdict() space.setitem(w_glob, space.wrap('__name__'), space.wrap('__builtin__')) space.exec_(self.code, w_glob, w_glob, hidden_applevel=self.hidden_applevel) Modified: pypy/dist/pypy/interpreter/module.py ============================================================================== --- pypy/dist/pypy/interpreter/module.py (original) +++ pypy/dist/pypy/interpreter/module.py Mon Aug 14 19:40:34 2006 @@ -10,7 +10,7 @@ def __init__(self, space, w_name, w_dict=None): self.space = space if w_dict is None: - w_dict = space.newdict([]) + w_dict = space.newdict() self.w_dict = w_dict self.w_name = w_name if w_name is not None: Modified: pypy/dist/pypy/interpreter/pycode.py ============================================================================== --- pypy/dist/pypy/interpreter/pycode.py (original) +++ pypy/dist/pypy/interpreter/pycode.py Mon Aug 14 19:40:34 2006 @@ -292,7 +292,7 @@ if flags & CO_OPTIMIZED: return if flags & CO_NEWLOCALS: - frame.w_locals = frame.space.newdict([]) + frame.w_locals = frame.space.newdict() else: assert frame.w_globals is not None frame.w_locals = frame.w_globals Modified: pypy/dist/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/dist/pypy/interpreter/pycompiler.py (original) +++ pypy/dist/pypy/interpreter/pycompiler.py Mon Aug 14 19:40:34 2006 @@ -156,7 +156,7 @@ w_dict = w_mod.getdict() w_reg = space.call_method(w_dict, 'setdefault', space.wrap("__warningregistry__"), - space.newdict([])) + space.newdict()) try: space.call_method(w_mod, 'warn_explicit', space.wrap(message), Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Mon Aug 14 19:40:34 2006 @@ -507,7 +507,7 @@ def BUILD_MAP(f, zero): if zero != 0: raise pyframe.BytecodeCorruption - w_dict = f.space.newdict([]) + w_dict = f.space.newdict() f.valuestack.push(w_dict) def LOAD_ATTR(f, nameindex): Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/pythonparse.py (original) +++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py Mon Aug 14 19:40:34 2006 @@ -172,7 +172,7 @@ def grammar_rules( space ): - w_rules = space.newdict([]) + w_rules = space.newdict() for key, value in PYTHON_PARSER.rules.iteritems(): space.setitem(w_rules, space.wrap(key), space.wrap(value)) return w_rules Modified: pypy/dist/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_argument.py (original) +++ pypy/dist/pypy/interpreter/test/test_argument.py Mon Aug 14 19:40:34 2006 @@ -11,8 +11,8 @@ def unpackiterable(self, it): return list(it) - def newdict(self, items): - return dict(items) + def newdict(self): + return {} def setitem(self, obj, key, value): obj[key] = value Modified: pypy/dist/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_compiler.py (original) +++ pypy/dist/pypy/interpreter/test/test_compiler.py Mon Aug 14 19:40:34 2006 @@ -13,14 +13,14 @@ def eval_string(self, string, kind='eval'): space = self.space code = self.compiler.compile(string, '<>', kind, 0) - return code.exec_code(space, space.newdict([]), space.newdict([])) + return code.exec_code(space, space.newdict(), space.newdict()) def test_compile(self): code = self.compiler.compile('6*7', '', 'eval', 0) assert isinstance(code, PyCode) assert code.co_filename == '' space = self.space - w_res = code.exec_code(space, space.newdict([]), space.newdict([])) + w_res = code.exec_code(space, space.newdict(), space.newdict()) assert space.int_w(w_res) == 42 def test_eval_unicode(self): @@ -71,7 +71,7 @@ assert isinstance(code, PyCode) assert code.co_filename == '' space = self.space - w_globals = space.newdict([]) + w_globals = space.newdict() code.exec_code(space, w_globals, w_globals) w_a = space.getitem(w_globals, space.wrap('a')) assert space.int_w(w_a) == 1 @@ -141,8 +141,8 @@ def test_toplevel_docstring(self): space = self.space code = self.compiler.compile('"spam"; "bar"; x=5', '', 'exec', 0) - w_locals = space.newdict([]) - code.exec_code(space, space.newdict([]), w_locals) + w_locals = space.newdict() + code.exec_code(space, space.newdict(), w_locals) w_x = space.getitem(w_locals, space.wrap('x')) assert space.eq_w(w_x, space.wrap(5)) w_doc = space.getitem(w_locals, space.wrap('__doc__')) @@ -150,8 +150,8 @@ # code = self.compiler.compile('"spam"; "bar"; x=5', '', 'single', 0) - w_locals = space.newdict([]) - code.exec_code(space, space.newdict([]), w_locals) + w_locals = space.newdict() + code.exec_code(space, space.newdict(), w_locals) w_x = space.getitem(w_locals, space.wrap('x')) assert space.eq_w(w_x, space.wrap(5)) w_doc = space.call_method(w_locals, 'get', space.wrap('__doc__')) @@ -331,7 +331,7 @@ ''')) code = self.compiler.compile(snippet, '', 'exec', 0) space = self.space - w_d = space.newdict([]) + w_d = space.newdict() code.exec_code(space, w_d, w_d) w_fline = space.getitem(w_d, space.wrap('fline')) w_gline = space.getitem(w_d, space.wrap('gline')) @@ -365,7 +365,7 @@ ''')) code = self.compiler.compile(snippet, '', 'exec', 0) space = self.space - w_d = space.newdict([]) + w_d = space.newdict() space.exec_(code, w_d, w_d) def test_ellipsis(self): @@ -376,7 +376,7 @@ ''')) code = self.compiler.compile(snippet, '', 'exec', 0) space = self.space - w_d = space.newdict([]) + w_d = space.newdict() space.exec_(code, w_d, w_d) def test_augassign_with_tuple_subscript(self): @@ -398,7 +398,7 @@ ''')) code = self.compiler.compile(snippet, '', 'exec', 0) space = self.space - w_d = space.newdict([]) + w_d = space.newdict() space.exec_(code, w_d, w_d) assert space.int_w(space.getitem(w_d, space.wrap('result'))) == 42 Modified: pypy/dist/pypy/interpreter/test/test_eval.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_eval.py (original) +++ pypy/dist/pypy/interpreter/test/test_eval.py Mon Aug 14 19:40:34 2006 @@ -33,18 +33,15 @@ space = self.space w = space.wrap self.f.fast2locals() - assert space.eq_w(self.f.w_locals, self.space.newdict([])) + assert space.eq_w(self.f.w_locals, self.space.wrap({})) self.f.fastlocals_w[0] = w(5) self.f.fast2locals() - assert space.eq_w(self.f.w_locals, self.space.newdict([ - (w('x'), w(5))])) + assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5})) self.f.fastlocals_w[2] = w(7) self.f.fast2locals() - assert space.eq_w(self.f.w_locals, self.space.newdict([ - (w('x'), w(5)), - (w('args'), w(7))])) + assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5, 'args': 7})) def sameList(self, l1, l2): assert len(l1) == len(l2) @@ -55,18 +52,15 @@ def test_locals2fast(self): w = self.space.wrap - self.f.w_locals = self.space.newdict([]) + self.f.w_locals = self.space.wrap({}) self.f.locals2fast() self.sameList(self.f.fastlocals_w, [None]*5) - self.f.w_locals = self.space.newdict([ - (w('x'), w(5))]) + self.f.w_locals = self.space.wrap({'x': 5}) self.f.locals2fast() self.sameList(self.f.fastlocals_w, [w(5)] + [None]*4) - self.f.w_locals = self.space.newdict([ - (w('x'), w(5)), - (w('args'), w(7))]) + self.f.w_locals = self.space.wrap({'x':5, 'args':7}) self.f.locals2fast() self.sameList(self.f.fastlocals_w, [w(5), None, w(7), None, None]) Modified: pypy/dist/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_function.py (original) +++ pypy/dist/pypy/interpreter/test/test_function.py Mon Aug 14 19:40:34 2006 @@ -260,7 +260,7 @@ def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) - self.fn = Function(self.space, code, self.space.newdict([])) + self.fn = Function(self.space, code, self.space.newdict()) def test_get(self): space = self.space @@ -287,7 +287,7 @@ # Create some function for this test only def m(self): return self func = Function(space, PyCode._from_code(self.space, m.func_code), - space.newdict([])) + space.newdict()) # Some shorthands obj1 = space.wrap(23) obj2 = space.wrap(42) Modified: pypy/dist/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_gateway.py (original) +++ pypy/dist/pypy/interpreter/test/test_gateway.py Mon Aug 14 19:40:34 2006 @@ -39,11 +39,7 @@ gateway.W_Root, 'starargs']) w = self.space.wrap - w_dict = self.space.newdict([ - (w('x'), w(123)), - (w('y'), w(23)), - (w('hello'), self.space.newtuple([w(0), w(True)])), - ]) + w_dict = w({'x': 123, 'y': 23, 'hello': (0, True)}) w_result = code.exec_code(self.space, w_dict, w_dict) assert self.space.eq_w(w_result, w(102)) @@ -59,12 +55,8 @@ gateway.W_Root, gateway.Arguments]) w = self.space.wrap - w_dict = self.space.newdict([ - (w('x'), w(123)), - (w('y'), w(23)), - (w('args'), self.space.newtuple([w(0), w(True)])), - (w('keywords'), self.space.newdict([(w('boo'), w(10))])), - ]) + w_dict = w({'x': 123, 'y': 23, 'args': (0, True), + 'keywords': {'boo': 10}}) w_result = code.exec_code(self.space, w_dict, w_dict) assert self.space.eq_w(w_result, w(1020)) @@ -115,7 +107,7 @@ assert self.space.eq_w( space.call(w_app_g3, space.newtuple([w('foo'), w('bar')]), - space.newdict([])), + space.newdict()), w('foobar')) assert self.space.eq_w( space.call_function(w_app_g3, w('foo'), w('bar')), @@ -134,7 +126,7 @@ assert self.space.eq_w( space.call(w_app_g3, space.newtuple([w('foo'), w('bar')]), - space.newdict([])), + space.newdict()), w('foobar')) assert self.space.eq_w( space.call_function(w_app_g3, w('foo'), w('bar')), @@ -152,7 +144,7 @@ assert self.space.eq_w( space.call(w_app_g3_args_w, space.newtuple([w('foo'), w('bar')]), - space.newdict([])), + space.newdict()), w('foobar')) assert self.space.eq_w( space.call_function(w_app_g3_args_w, w('foo'), w('bar')), @@ -170,7 +162,7 @@ assert self.space.eq_w( space.call(w_app_g3_ss, space.newtuple([w('foo'), w('bar')]), - space.newdict([])), + space.newdict()), w('foobar')) assert self.space.eq_w( space.call_function(w_app_g3_ss, w('foo'), w('bar')), @@ -188,7 +180,7 @@ assert self.space.eq_w( space.call(w_app_g3_if, space.newtuple([w(1), w(1.0)]), - space.newdict([])), + space.newdict()), w(2.0)) assert self.space.eq_w( space.call_function(w_app_g3_if, w(1), w(1.0)), Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Mon Aug 14 19:40:34 2006 @@ -31,12 +31,9 @@ assert self.space.eq_w(w_l, w(l)) def test_newdict(self): - w = self.space.wrap - items = [(0, 1), (3, 4)] - items_w = [(w(k), w(v)) for (k, v) in items] - d = dict(items) - w_d = self.space.newdict(items_w) - assert self.space.eq_w(w_d, w(d)) + d = {} + w_d = self.space.newdict() + assert self.space.eq_w(w_d, self.space.wrap(d)) def test_newtuple(self): w = self.space.wrap Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Mon Aug 14 19:40:34 2006 @@ -172,7 +172,7 @@ def user_setup(self, space, w_subtype, nslots): self.space = space self.w__class__ = w_subtype - self.w__dict__ = space.newdict([]) + self.w__dict__ = space.newdict() self.user_setup_slots(nslots) else: supercls = cls Modified: pypy/dist/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/compiling.py (original) +++ pypy/dist/pypy/module/__builtin__/compiling.py Mon Aug 14 19:40:34 2006 @@ -89,7 +89,7 @@ if w_globals is None or space.is_w(w_globals, space.w_None): if caller is None: - w_globals = w_locals = space.newdict([]) + w_globals = w_locals = space.newdict() else: w_globals = caller.w_globals w_locals = caller.getdictscope() Modified: pypy/dist/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_builtin.py Mon Aug 14 19:40:34 2006 @@ -450,7 +450,7 @@ w_execfile = self.get_builtin('execfile') space = self.space - w_dict = space.newdict([]) + w_dict = space.newdict() self.space.call_function(w_execfile, space.wrap(fn), w_dict, space.w_None) w_value = space.getitem(w_dict, space.wrap('i')) Modified: pypy/dist/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_import.py Mon Aug 14 19:40:34 2006 @@ -315,7 +315,7 @@ pycode = space.interpclass_w(w_code) os.close(fd) assert type(pycode) is pypy.interpreter.pycode.PyCode - w_dic = space.newdict([]) + w_dic = space.newdict() pycode.exec_code(space, w_dic, w_dic) w_ret = space.getitem(w_dic, space.wrap('x')) ret = space.int_w(w_ret) @@ -352,7 +352,7 @@ osfile.close() pycode = space.interpclass_w(w_ret) assert type(pycode) is pypy.interpreter.pycode.PyCode - w_dic = space.newdict([]) + w_dic = space.newdict() pycode.exec_code(space, w_dic, w_dic) w_ret = space.getitem(w_dic, space.wrap('x')) ret = space.int_w(w_ret) @@ -428,7 +428,7 @@ os.close(fd) # check value of load - w_dic = space.newdict([]) + w_dic = space.newdict() pycode.exec_code(space, w_dic, w_dic) w_ret = space.getitem(w_dic, space.wrap('x')) ret = space.int_w(w_ret) Modified: pypy/dist/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/dist/pypy/module/_pickle_support/maker.py (original) +++ pypy/dist/pypy/module/_pickle_support/maker.py Mon Aug 14 19:40:34 2006 @@ -26,7 +26,7 @@ def func_new(space): fu = instantiate(Function) - fu.w_func_dict = space.newdict([]) + fu.w_func_dict = space.newdict() return space.wrap(fu) func_new.unwrap_spec = [ObjSpace] Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Aug 14 19:40:34 2006 @@ -238,7 +238,7 @@ class State: def __init__(self, space): self.posix_putenv_garbage = {} - self.w_environ = space.newdict([]) + self.w_environ = space.newdict() _convertenviron(space, self.w_environ) def get(space): return space.fromcache(State) Modified: pypy/dist/pypy/module/sys/state.py ============================================================================== --- pypy/dist/pypy/module/sys/state.py (original) +++ pypy/dist/pypy/module/sys/state.py Mon Aug 14 19:40:34 2006 @@ -14,7 +14,7 @@ def __init__(self, space): self.space = space - self.w_modules = space.newdict([]) + self.w_modules = space.newdict() self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) Modified: pypy/dist/pypy/module/thread/os_local.py ============================================================================== --- pypy/dist/pypy/module/thread/os_local.py (original) +++ pypy/dist/pypy/module/thread/os_local.py Mon Aug 14 19:40:34 2006 @@ -17,7 +17,7 @@ self.space = space self.initargs = initargs.normalize() ident = thread.get_ident() - self.dicts = {ident: space.newdict([])} + self.dicts = {ident: space.newdict()} def getdict(self): ident = thread.get_ident() @@ -26,7 +26,7 @@ except KeyError: # create a new dict for this thread space = self.space - w_dict = self.dicts[ident] = space.newdict([]) + w_dict = self.dicts[ident] = space.newdict() # call __init__ try: w_self = space.wrap(self) Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Mon Aug 14 19:40:34 2006 @@ -81,8 +81,8 @@ for var in items]) func_head = 'lambda ' + var_ids + ':' expr = func_head + object_space.str_w(w_formula) - func_obj = ev(object_space, object_space.wrap(expr), object_space.newdict([]), - object_space.newdict([])) + func_obj = ev(object_space, object_space.wrap(expr), object_space.newdict(), + object_space.newdict()) assert isinstance(func_obj, Function) return func_obj @@ -112,15 +112,15 @@ def _init_result_cache(self): """key = (variable,value), value = [has_success,has_failure]""" - result_cache = self._space.newdict([]) + result_cache = self._space.newdict() for var in self._variables: assert isinstance(var, W_Variable) - result_cache.content[var.w_name()] = self._space.newdict([]) + result_cache.content[var.w_name()] = self._space.newdict() return result_cache def _assign_values(self): variables = [] - kwargs = self._space.newdict([]) + kwargs = self._space.newdict() for variable in self._variables: assert isinstance(variable, W_Variable) domain = variable.w_dom Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Mon Aug 14 19:40:34 2006 @@ -29,7 +29,7 @@ W_AbstractDomain.__init__(self, space) #XXX a pure dict used to work there (esp. in revise) assert isinstance(w_values, W_ListObject) - self._values = space.newdict([]) + self._values = space.newdict() self.set_values(w_values) Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Mon Aug 14 19:40:34 2006 @@ -305,7 +305,7 @@ def sched_info(space): sched = scheduler[0] - w_ret = space.newdict([]) + w_ret = space.newdict() if not we_are_translated(): # XXX and otherwise, WTF ??? space.setitem(w_ret, space.wrap('switches'), space.wrap(sched._switch_count)) space.setitem(w_ret, space.wrap('threads'), @@ -323,7 +323,7 @@ s = scheduler[0] si = space.setitem sw = space.wrap - w_ret = space.newdict([]) + w_ret = space.newdict() if not we_are_translated(): si(w_ret, sw('threads'), sw([id(th) for th in s.get_threads()])) Modified: pypy/dist/pypy/objspace/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/constraint/constraint.py Mon Aug 14 19:40:34 2006 @@ -182,8 +182,8 @@ for var in items]) func_head = 'lambda ' + var_ids + ':' expr = func_head + object_space.str_w(w_formula) - func_obj = ev(object_space, object_space.wrap(expr), object_space.newdict([]), - object_space.newdict([])) + func_obj = ev(object_space, object_space.wrap(expr), object_space.newdict(), + object_space.newdict()) assert isinstance(func_obj, Function) return func_obj @@ -213,15 +213,15 @@ def _init_result_cache(self): """key = (variable,value), value = [has_success,has_failure]""" - result_cache = self._space.newdict([]) + result_cache = self._space.newdict() for var in self._variables: assert isinstance(var, W_Variable) - result_cache.content[var.w_name()] = self._space.newdict([]) + result_cache.content[var.w_name()] = self._space.newdict() return result_cache def _assign_values(self, w_cs): variables = [] - kwargs = self._space.newdict([]) + kwargs = self._space.newdict() for variable in self._variables: assert isinstance(variable, W_Variable) domain = w_cs.w_dom(variable) Modified: pypy/dist/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/objspace.py Mon Aug 14 19:40:34 2006 @@ -150,8 +150,10 @@ def call_args(self, w_callable, args): args_w, kwds_w = args.unpack() w_args = self.newtuple(args_w) - w_kwds = self.newdict([(self.wrap(key), w_value) - for key, w_value in kwds_w.items()]) + w_kwds = self.newdict() + for key, w_value in kwds_w.items(): + w_key = self.wrap(key) + PyDict_SetItem(w_kwds, w_key, w_value) return PyObject_Call(w_callable, w_args, w_kwds) def new_interned_str(self, s): @@ -176,11 +178,8 @@ def newint(self, intval): return PyInt_FromLong(intval) - def newdict(self, items_w): - w_dict = PyDict_New() - for w_key, w_value in items_w: - PyDict_SetItem(w_dict, w_key, w_value) - return w_dict + def newdict(self): + return PyDict_New() def newlist(self, items_w): n = len(items_w) Modified: pypy/dist/pypy/objspace/cpy/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_objspace.py Mon Aug 14 19:40:34 2006 @@ -6,7 +6,8 @@ space = CPyObjSpace() wk1 = space.wrap('key') wone = space.wrap(1) - d = space.newdict([(space.wrap('zero'),space.wrap(0))]) + d = space.newdict() + space.setitem(d,space.wrap('zero'),space.wrap(0)) space.setitem(d,wk1,wone) wback = space.getitem(d,wk1) assert space.eq_w(wback,wone) Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_typedef.py Mon Aug 14 19:40:34 2006 @@ -330,7 +330,7 @@ w_type = space.gettypefor(W_MyType) w_obj = space.call_function(w_type, space.wrap(42)) - w_mydict = space.newdict([]) + w_mydict = space.newdict() space.setitem(w_mydict, space.wrap('hello'), w_obj) def build(): return w_mydict Modified: pypy/dist/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/objspace.py (original) +++ pypy/dist/pypy/objspace/flow/objspace.py Mon Aug 14 19:40:34 2006 @@ -80,16 +80,10 @@ self.executioncontext.recorder = previous_recorder self.concrete_mode -= 1 - def newdict(self, items_w): + def newdict(self): if self.concrete_mode: - content = [(self.unwrap(w_key), self.unwrap(w_value)) - for w_key, w_value in items_w] - return Constant(dict(content)) - flatlist_w = [] - for w_key, w_value in items_w: - flatlist_w.append(w_key) - flatlist_w.append(w_value) - return self.do_operation('newdict', *flatlist_w) + return Constant({}) + return self.do_operation('newdict') def newtuple(self, args_w): try: Modified: pypy/dist/pypy/objspace/flow/test/test_framestate.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_framestate.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_framestate.py Mon Aug 14 19:40:34 2006 @@ -19,7 +19,7 @@ pass code = func.func_code code = PyCode._from_code(self.space, code) - w_globals = Constant({}) # space.newdict([]) + w_globals = Constant({}) # space.newdict() frame = code.create_frame(space, w_globals) formalargcount = code.getformalargcount() Modified: pypy/dist/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/dist/pypy/objspace/std/marshal_impl.py (original) +++ pypy/dist/pypy/objspace/std/marshal_impl.py Mon Aug 14 19:40:34 2006 @@ -364,7 +364,7 @@ # since primitive lists are not optimized and we don't know # the dict size in advance, use the dict's setitem instead # of building a list of tuples. - w_dic = space.newdict([]) + w_dic = space.newdict() while 1: w_key = u.get_w_obj(True) if w_key is None: Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Aug 14 19:40:34 2006 @@ -247,7 +247,7 @@ def createexecutioncontext(self): # add space specific fields to execution context ec = ObjSpace.createexecutioncontext(self) - ec._py_repr = self.newdict([]) + ec._py_repr = self.newdict() return ec def gettypefor(self, cls): @@ -284,7 +284,9 @@ return W_UnicodeObject([unichr(ord(u)) for u in x]) # xxx if isinstance(x, dict): items_w = [(self.wrap(k), self.wrap(v)) for (k, v) in x.iteritems()] - return self.newdict(items_w) + r = self.newdict() + r.initialize_content(items_w) + return r if isinstance(x, float): return W_FloatObject(x) if isinstance(x, tuple): @@ -386,10 +388,8 @@ def newlist(self, list_w): return W_ListObject(list_w) - def newdict(self, list_pairs_w): - w_result = self.DictObjectCls(self) - w_result.initialize_content(list_pairs_w) - return w_result + def newdict(self): + return self.DictObjectCls(self) def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) Modified: pypy/dist/pypy/objspace/std/stdtypedef.py ============================================================================== --- pypy/dist/pypy/objspace/std/stdtypedef.py (original) +++ pypy/dist/pypy/objspace/std/stdtypedef.py Mon Aug 14 19:40:34 2006 @@ -37,7 +37,7 @@ return True def descr_del_dict(space, w_obj): # blame CPython for the existence of this one - w_obj.setdict(space, space.newdict([])) + w_obj.setdict(space, space.newdict()) std_dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict) std_dict_descr.name = '__dict__' Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictobject.py Mon Aug 14 19:40:34 2006 @@ -52,20 +52,27 @@ def test_dict_compare(self): w = self.space.wrap w0, w1, w2, w3 = map(w, range(4)) - wd1 = self.space.newdict([(w0, w1), (w2, w3)]) - wd2 = self.space.newdict([(w2, w3), (w0, w1)]) + def wd(items): + d = self.space.newdict() + d.initialize_content(items) + return d + wd1 = wd([(w0, w1), (w2, w3)]) + wd2 = wd([(w2, w3), (w0, w1)]) assert self.space.eq_w(wd1, wd2) - wd3 = self.space.newdict([(w2, w2), (w0, w1)]) + wd3 = wd([(w2, w2), (w0, w1)]) assert not self.space.eq_w(wd1, wd3) - wd4 = self.space.newdict([(w3, w3), (w0, w1)]) + wd4 = wd([(w3, w3), (w0, w1)]) assert not self.space.eq_w(wd1, wd4) - wd5 = self.space.newdict([(w3, w3)]) + wd5 = wd([(w3, w3)]) assert not self.space.eq_w(wd1, wd4) def test_dict_call(self): space = self.space w = space.wrap - wd = space.newdict + def wd(items): + d = space.newdict() + d.initialize_content(items) + return d def mydict(w_args=w(()), w_kwds=w({})): return space.call(space.w_dict, w_args, w_kwds) def deepwrap(lp): @@ -85,7 +92,6 @@ def test_dict_pop(self): space = self.space w = space.wrap - wd = space.newdict def mydict(w_args=w(()), w_kwds=w({})): return space.call(space.w_dict, w_args, w_kwds) d = mydict(w_kwds=w({"1":2, "3":4})) Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Mon Aug 14 19:40:34 2006 @@ -73,7 +73,7 @@ try: caller = space.getexecutioncontext().framestack.top() except IndexError: - w_globals = w_locals = space.newdict([]) + w_globals = w_locals = space.newdict() else: w_globals = caller.w_globals w_str_name = space.wrap('__name__') @@ -302,7 +302,9 @@ dictspec = [] for key, w_value in w_self.dict_w.items(): dictspec.append((space.wrap(key), w_value)) - return W_DictProxyObject(space.newdict(dictspec)) + newdic = space.newdict() + newdic.initialize_content(dictspec) + return W_DictProxyObject(newdic) def unwrap(w_self): if w_self.instancetypedef.fakedcpytype is not None: Modified: pypy/dist/pypy/rpython/callparse.py ============================================================================== --- pypy/dist/pypy/rpython/callparse.py (original) +++ pypy/dist/pypy/rpython/callparse.py Mon Aug 14 19:40:34 2006 @@ -167,7 +167,7 @@ def newtuple(self, items): return NewTupleHolder(items) - def newdict(self, stuff): + def newdict(self): raise CallPatternTooComplex, "'**' argument" def unpackiterable(self, it, expected_length=None): Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Mon Aug 14 19:40:34 2006 @@ -132,7 +132,7 @@ unwrap_spec=[gateway.ObjSpace, gateway.W_Root, gateway.Arguments])) - w_dict = space.newdict([]) + w_dict = space.newdict() space.setitem(w_dict, space.wrap('__init__'), w_init) return space.call_function(w_metaclass, space.wrap('AssertionError'), Modified: pypy/dist/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/dist/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/dist/pypy/tool/test/test_pytestsupport.py Mon Aug 14 19:40:34 2006 @@ -14,7 +14,7 @@ def test_AppFrame(space): import sys co = PyCode._from_code(space, somefunc.func_code) - pyframe = PyFrame(space, co, space.newdict([]), None) + pyframe = PyFrame(space, co, space.newdict(), None) runner = AppFrame(pyframe) exprinfo.run("f = lambda x: x+1", runner) msg = exprinfo.interpret("assert isinstance(f(2), float)", runner) Modified: pypy/dist/pypy/translator/geninterplevel.py ============================================================================== --- pypy/dist/pypy/translator/geninterplevel.py (original) +++ pypy/dist/pypy/translator/geninterplevel.py Mon Aug 14 19:40:34 2006 @@ -101,7 +101,7 @@ unique = self.uniquenameofprebuilt("eval_helper", eval_helper) self.initcode.append1( 'def %s(expr):\n' - ' dic = space.newdict([])\n' + ' dic = space.newdict()\n' ' if "types." in expr:\n' ' space.exec_("import types", dic, dic)\n' ' else:\n' @@ -114,7 +114,7 @@ unique = self.uniquenameofprebuilt("unpickle_helper", unpickle_helper) self.initcode.append1( 'def %s(value):\n' - ' dic = space.newdict([])\n' + ' dic = space.newdict()\n' ' space.exec_("import cPickle as pickle", dic, dic)\n' ' return space.eval("pickle.loads(%%r)" %% value, dic, dic)' % unique) self.initcode.append1('%s = %s(%r)' % ( @@ -125,7 +125,7 @@ unique = self.uniquenameofprebuilt("long_helper", long_helper) self.initcode.append1( 'def %s(value):\n' - ' dic = space.newdict([])\n' + ' dic = space.newdict()\n' ' space.exec_("", dic, dic) # init __builtins__\n' ' return space.eval(value, dic, dic)' % unique) self.initcode.append1('%s = %s(%r)' % ( @@ -136,7 +136,7 @@ unique = self.uniquenameofprebuilt("bltinmod_helper", bltinmod_helper) self.initcode.append1( 'def %s(name):\n' - ' dic = space.newdict([])\n' + ' dic = space.newdict()\n' ' space.exec_("import %%s" %% name, dic, dic)\n' ' return space.eval("%%s" %% name, dic, dic)' % (unique, )) self.initcode.append1('%s = %s(%r)' % (name, unique, mod.__name__)) @@ -183,7 +183,7 @@ # special constructors: self.has_listarg = {} - for name in "newtuple newlist newdict newstring".split(): + for name in "newtuple newlist newstring".split(): self.has_listarg[name] = name # catching all builtins in advance, to avoid problems @@ -798,7 +798,7 @@ name, self.nameof(key), self.nameof(value)) baseargs = ", ".join(basenames) - initcode.append('_dic = space.newdict([])') + initcode.append('_dic = space.newdict()') for key, value in cls.__dict__.items(): if key.startswith('__'): if key in ['__module__', '__metaclass__', '__slots__', @@ -934,7 +934,7 @@ assert dic is not __builtins__ name = self.uniquename('g%ddict' % len(dic)) self.register_early(dic, name) - self.initcode.append('%s = space.newdict([])' % (name,)) + self.initcode.append('%s = space.newdict()' % (name,)) for k in dic: if k == '__builtins__': continue Modified: pypy/dist/pypy/translator/goal/targetlogicstandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetlogicstandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetlogicstandalone.py Mon Aug 14 19:40:34 2006 @@ -97,7 +97,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict([]) + w_dict = space.newdict() space.exec_(open(filename).read(), w_dict, w_dict) w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) Modified: pypy/dist/pypy/translator/goal/targetmultiplespaces.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetmultiplespaces.py (original) +++ pypy/dist/pypy/translator/goal/targetmultiplespaces.py Mon Aug 14 19:40:34 2006 @@ -123,11 +123,11 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space1.newdict([]) + w_dict = space1.newdict() space1.exec_(open(filename).read(), w_dict, w_dict) w_entry_point_1 = space1.getitem(w_dict, space1.wrap('entry_point')) - w_dict = space2.newdict([]) + w_dict = space2.newdict() space2.exec_(open(filename).read(), w_dict, w_dict) w_entry_point_2 = space2.getitem(w_dict, space2.wrap('entry_point')) Modified: pypy/dist/pypy/translator/goal/targetpypymain.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypymain.py (original) +++ pypy/dist/pypy/translator/goal/targetpypymain.py Mon Aug 14 19:40:34 2006 @@ -104,7 +104,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict([]) + w_dict = space.newdict() space.exec_(open(filename).read(), w_dict, w_dict) w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Mon Aug 14 19:40:34 2006 @@ -134,7 +134,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict([]) + w_dict = space.newdict() space.exec_(open(filename).read(), w_dict, w_dict) entry_point = create_entry_point(space, w_dict) Modified: pypy/dist/pypy/translator/goal/targetthunkstandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetthunkstandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetthunkstandalone.py Mon Aug 14 19:40:34 2006 @@ -97,7 +97,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict([]) + w_dict = space.newdict() space.exec_(open(filename).read(), w_dict, w_dict) w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) Modified: pypy/dist/pypy/translator/pyrex/genpyrex.py ============================================================================== --- pypy/dist/pypy/translator/pyrex/genpyrex.py (original) +++ pypy/dist/pypy/translator/pyrex/genpyrex.py Mon Aug 14 19:40:34 2006 @@ -106,10 +106,7 @@ return "%s = []" % self.resultname def op_newdict(self): - pairs = [] - for i in range(0, len(self.argnames), 2): - pairs.append("%s: %s, " % (self.argnames[i], self.argnames[i+1])) - return "%s = {%s}" % (self.resultname, "".join(pairs)) + return "%s = {}" % (self.resultname,) def op_newslice(self): a = self.argnames From rhymes at codespeak.net Tue Aug 15 00:00:08 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Tue, 15 Aug 2006 00:00:08 +0200 (CEST) Subject: [pypy-svn] r31307 - pypy/dist/pypy/module/bz2/test Message-ID: <20060814220008.02F2310074@code0.codespeak.net> Author: rhymes Date: Mon Aug 14 23:59:59 2006 New Revision: 31307 Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py Log: one more test for bz2 Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py Mon Aug 14 23:59:59 2006 @@ -362,7 +362,15 @@ f = open("foo", "rb") assert self.decompress(f.read()) == self.TEXT f.close() + + def test_write_methods_on_readonly_file(self): + from bz2 import BZ2File + bz2f = BZ2File("foo", 'r') + raises(IOError, bz2f.write, "abc") + raises(IOError, bz2f.writelines, ["abc"]) + bz2f.close() + # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") # # if has_cmdline_bunzip2: From pedronis at codespeak.net Tue Aug 15 14:25:25 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 15 Aug 2006 14:25:25 +0200 (CEST) Subject: [pypy-svn] r31315 - in pypy/extradoc/talk/vancouver: . shells Message-ID: <20060815122525.9037410074@code0.codespeak.net> Author: pedronis Date: Tue Aug 15 14:25:24 2006 New Revision: 31315 Added: pypy/extradoc/talk/vancouver/ - copied from r31314, user/pedronis/vancouver/ Removed: pypy/extradoc/talk/vancouver/notes.txt pypy/extradoc/talk/vancouver/shells/ Modified: pypy/extradoc/talk/vancouver/talk.txt Log: vancouver talk Modified: pypy/extradoc/talk/vancouver/talk.txt ============================================================================== --- user/pedronis/vancouver/talk.txt (original) +++ pypy/extradoc/talk/vancouver/talk.txt Tue Aug 15 14:25:24 2006 @@ -1,14 +1,5 @@ .. include:: -.. raw:: html - - - - ================================================= PyPy Status ================================================= @@ -74,10 +65,6 @@ filling in the details (C, .NET, ...) -.. raw:: html - - - The Basic Architecture ======================= @@ -115,10 +102,6 @@ Using coroutine cloning to do backtracing (RPython example) -.. raw:: html - - - Ext-Compiler ========================= @@ -129,10 +112,6 @@ CPython - work in progress -.. raw:: html - - - PyPy/Translation overview ========================= @@ -192,15 +171,6 @@ :width: 500 :height: 500 - -.. raw:: html - - - Website =========== From arigo at codespeak.net Tue Aug 15 14:28:38 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 14:28:38 +0200 (CEST) Subject: [pypy-svn] r31316 - pypy/dist/pypy/translator Message-ID: <20060815122838.0353010075@code0.codespeak.net> Author: arigo Date: Tue Aug 15 14:28:37 2006 New Revision: 31316 Modified: pypy/dist/pypy/translator/geninterplevel.py Log: Bump geninterp revision number to rebuild the _cache with the new argument-less space.newdict(). Modified: pypy/dist/pypy/translator/geninterplevel.py ============================================================================== --- pypy/dist/pypy/translator/geninterplevel.py (original) +++ pypy/dist/pypy/translator/geninterplevel.py Tue Aug 15 14:28:37 2006 @@ -78,7 +78,7 @@ log = py.log.Producer("geninterp") py.log.setconsumer("geninterp", ansi_log) -GI_VERSION = '1.1.17' # bump this for substantial changes +GI_VERSION = '1.1.18' # bump this for substantial changes # ____________________________________________________________ try: From arigo at codespeak.net Tue Aug 15 14:30:10 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 14:30:10 +0200 (CEST) Subject: [pypy-svn] r31317 - pypy/dist/pypy/tool Message-ID: <20060815123010.2D84D10075@code0.codespeak.net> Author: arigo Date: Tue Aug 15 14:30:09 2006 New Revision: 31317 Modified: pypy/dist/pypy/tool/option.py Log: issue246 testing In py.py, stop argument parsing after the script name. (Already works in pypy-c) Modified: pypy/dist/pypy/tool/option.py ============================================================================== --- pypy/dist/pypy/tool/option.py (original) +++ pypy/dist/pypy/tool/option.py Tue Aug 15 14:30:09 2006 @@ -73,6 +73,7 @@ Options = input_options op = optparse.OptionParser() op.add_options(optionlist) + op.disable_interspersed_args() options, args = op.parse_args(argv, input_options) return args From arigo at codespeak.net Tue Aug 15 14:34:45 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 14:34:45 +0200 (CEST) Subject: [pypy-svn] r31318 - pypy/dist/pypy/rpython/rctypes/socketmodule Message-ID: <20060815123445.ACF6510075@code0.codespeak.net> Author: arigo Date: Tue Aug 15 14:34:44 2006 New Revision: 31318 Modified: pypy/dist/pypy/rpython/rctypes/socketmodule/ctypes_socket.py Log: issue243 resolved Import cleanups (thanks calderone). Modified: pypy/dist/pypy/rpython/rctypes/socketmodule/ctypes_socket.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/socketmodule/ctypes_socket.py (original) +++ pypy/dist/pypy/rpython/rctypes/socketmodule/ctypes_socket.py Tue Aug 15 14:34:44 2006 @@ -1,9 +1,16 @@ import os -import distutils + from pypy.rpython.rctypes.tool import ctypes_platform from pypy.rpython.rctypes.tool import util # ctypes.util from 0.9.9.6 + +# Not used here, but exported for other code. from pypy.rpython.rctypes.aerrno import geterrno -from ctypes import * + +from ctypes import c_ushort, c_int, c_uint, c_char_p, c_void_p +from ctypes import POINTER, ARRAY, cdll, sizeof, SetPointerType + +# Also not used here, but exported for other code. +from ctypes import cast, pointer, create_string_buffer includes = ('sys/types.h', 'sys/socket.h', From arigo at codespeak.net Tue Aug 15 14:51:23 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 14:51:23 +0200 (CEST) Subject: [pypy-svn] r31319 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060815125123.7209710074@code0.codespeak.net> Author: arigo Date: Tue Aug 15 14:51:18 2006 New Revision: 31319 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/dictstrobject.py pypy/dist/pypy/objspace/std/dicttype.py pypy/dist/pypy/objspace/std/test/test_dictobject.py Log: issue244 testing Added keyword arguments to dict.update(). Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 15 14:51:18 2006 @@ -327,8 +327,8 @@ else: w_self.implementation = EmptyDictImplementation(space) if w_otherdict is not None: - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_self, w_otherdict) + from pypy.objspace.std.dicttype import update1 + update1(space, w_self, w_otherdict) def initialize_content(w_self, list_pairs_w): impl = w_self.implementation @@ -377,11 +377,11 @@ w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__DictMulti_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Tue Aug 15 14:51:18 2006 @@ -65,11 +65,11 @@ w_dict.content[w_k] = w_v else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__Dict_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dictstrobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictstrobject.py (original) +++ pypy/dist/pypy/objspace/std/dictstrobject.py Tue Aug 15 14:51:18 2006 @@ -121,11 +121,11 @@ w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__DictStr_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/dist/pypy/objspace/std/dicttype.py (original) +++ pypy/dist/pypy/objspace/std/dicttype.py Tue Aug 15 14:51:18 2006 @@ -30,7 +30,7 @@ dict_setdefault = SMM('setdefault', 3, defaults=(None,), doc='D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d' ' if k not in D') -dict_update = SMM('update', 2, defaults=((),), +dict_update = SMM('update', 1, general__args__=True, doc='D.update(E, **F) -> None. Update D from E and F:' ' for k in E: D[k] = E[k]\n(if E has keys else: for' ' (k, v) in E: D[k] = v) then: for k in F: D[k] =' @@ -55,7 +55,7 @@ # gateway is imported in the stdtypedef module app = gateway.applevel(''' - def update(d, o): + def update1(d, o): if hasattr(o, 'keys'): for k in o.keys(): d[k] = o[k] @@ -63,6 +63,14 @@ for k,v in o: d[k] = v + def update(d, *args, **kwargs): + if len(args) == 1: + update1(d, args[0]) + elif len(args) > 1: + raise TypeError("update takes at most 1 (non-keyword) argument") + if kwargs: + update1(d, kwargs) + def popitem(d): k = d.keys() if not k: @@ -110,7 +118,7 @@ ''', filename=__file__) #XXX what about dict.fromkeys()? -dict_update__ANY_ANY = app.interphook("update") +dict_update__ANY = app.interphook("update") dict_popitem__ANY = app.interphook("popitem") dict_get__ANY_ANY_ANY = app.interphook("get") dict_setdefault__ANY_ANY_ANY = app.interphook("setdefault") @@ -118,6 +126,7 @@ dict_iteritems__ANY = app.interphook("iteritems") dict_iterkeys__ANY = app.interphook("iterkeys") dict_itervalues__ANY = app.interphook("itervalues") +update1 = app.interphook("update1") register_all(vars(), globals()) Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictobject.py Tue Aug 15 14:51:18 2006 @@ -245,7 +245,17 @@ d = {} d.update() assert d == {} - + + def test_update_kwargs(self): + d = {} + d.update(foo='bar', baz=1) + assert d == {'foo': 'bar', 'baz': 1} + + def test_update_dict_and_kwargs(self): + d = {} + d.update({'foo': 'bar'}, baz=1) + assert d == {'foo': 'bar', 'baz': 1} + def test_values(self): d = {1:2, 3:4} vals = d.values() From arigo at codespeak.net Tue Aug 15 15:03:18 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 15:03:18 +0200 (CEST) Subject: [pypy-svn] r31320 - in pypy/dist/pypy: bin interpreter interpreter/test lib lib/test2 Message-ID: <20060815130318.439B410074@code0.codespeak.net> Author: arigo Date: Tue Aug 15 15:03:06 2006 New Revision: 31320 Added: pypy/dist/pypy/lib/runpy.py (contents, props changed) pypy/dist/pypy/lib/test2/test_runpy.py (contents, props changed) Modified: pypy/dist/pypy/bin/py.py pypy/dist/pypy/interpreter/main.py pypy/dist/pypy/interpreter/test/test_main.py Log: issue145 in-progress Patch by leo.soto: add the -m command-line option to py.py. The same needs to be done for pypy-c now... Modified: pypy/dist/pypy/bin/py.py ============================================================================== --- pypy/dist/pypy/bin/py.py (original) +++ pypy/dist/pypy/bin/py.py Tue Aug 15 15:03:06 2006 @@ -22,6 +22,8 @@ interactive = 0 command = [] completer = False + module = None + module_args = [] def get_main_options(): options = option.get_standard_options() @@ -50,6 +52,18 @@ '-c', action="callback", callback=command_callback, help="program passed in as CMD (terminates option list)")) + + def runmodule_callback(option, opt, value, parser): + parser.values.module_args = parser.rargs[:] + parser.values.module = value + parser.rargs[:] = [] + + options.append(make_option( + '-m', action="callback", metavar='NAME', + callback=runmodule_callback, type="string", + help="library module to be run as a script (terminates option list)")) + + return options @@ -92,6 +106,9 @@ if Options.command: def doit(): main.run_string(Options.command[0], space=space) + elif Options.module: + def doit(): + main.run_module(Options.module, Options.module_args, space=space) elif args: scriptdir = os.path.dirname(os.path.abspath(args[0])) space.call_method(space.sys.get('path'), 'insert', Modified: pypy/dist/pypy/interpreter/main.py ============================================================================== --- pypy/dist/pypy/interpreter/main.py (original) +++ pypy/dist/pypy/interpreter/main.py Tue Aug 15 15:03:06 2006 @@ -66,6 +66,28 @@ istring = open(filename).read() run_string(istring, filename, space) +def run_module(module_name, args, space=None): + """Implements PEP 338 'Executing modules as scripts', overwriting + sys.argv[1:] using `args` and executing the module `module_name`. + sys.argv[0] always is `module_name`. + + Delegates the real work to the runpy module provided as the reference + implementation. + """ + if space is None: + from pypy.objspace.std import StdObjSpace + space = StdObjSpace() + w = space.wrap + argv = [module_name] + if args is not None: + argv.extend(args) + space.setitem(space.sys.w_dict, w('argv'), w(argv)) + w_import = space.builtin.get('__import__') + runpy = space.call_function(w_import, w('runpy')) + w_run_module = space.getitem(runpy.w_dict, w('run_module')) + return space.call_function(w_run_module, w(module_name), space.w_None, + w('__main__'), space.w_True) + # ____________________________________________________________ def run_toplevel(space, f, verbose=False): Modified: pypy/dist/pypy/interpreter/test/test_main.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_main.py (original) +++ pypy/dist/pypy/interpreter/test/test_main.py Tue Aug 15 15:03:06 2006 @@ -14,6 +14,15 @@ main() """ +# On module test we want to ensure that the called module __name__ is +# '__main__' and argv is set as expected. +testmodulecode = """ +import sys +if __name__ == '__main__': + aStr = sys.argv[1] + print len(aStr) +""" + testresultoutput = '11\n' def checkoutput(space, expected_output,f,*args): @@ -30,16 +39,38 @@ assert capturefn.read(mode='rU') == expected_output testfn = 'tmp_hello_world.py' +testmodule = 'tmp_hello_module' +testpackage = 'tmp_package' class TestMain: def setup_class(cls): ofile = open(testfn, 'w') ofile.write(testcode) ofile.close() + omodulefile = open(testmodule + '.py', 'w') + omodulefile.write(testmodulecode) + omodulefile.close() + import os + os.mkdir(testpackage) + open(os.path.join(testpackage, '__init__.py'), 'w').close() + file_name = os.path.join(testpackage, testmodule) + '.py' + omodulefile = open(file_name,'w') + omodulefile.write(testmodulecode) + omodulefile.close() + def teardown_class(cls): import os - os.remove(testfn) + def remove_if_exists(fn): + if os.path.exists(fn): + os.remove(fn) + remove_if_exists(testfn) + remove_if_exists(testmodule + '.py') + remove_if_exists(os.path.join(testpackage, '__init__.py')) + remove_if_exists(os.path.join(testpackage, '__init__.pyc')) + remove_if_exists(os.path.join(testpackage, testmodule) + '.py') + os.rmdir(testpackage) + def test_run_file(self): checkoutput(self.space, testresultoutput,main.run_file,testfn) @@ -51,3 +82,9 @@ def test_eval_string(self): w_x = main.eval_string('2+2', space=self.space) assert self.space.eq_w(w_x, self.space.wrap(4)) + + def test_run_module(self): + checkoutput(self.space, testresultoutput, main.run_module, + testmodule, ['hello world']) + checkoutput(self.space, testresultoutput, main.run_module, + testpackage + '.' + testmodule, ['hello world']) Added: pypy/dist/pypy/lib/runpy.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/runpy.py Tue Aug 15 15:03:06 2006 @@ -0,0 +1,438 @@ +"""runpy.py - locating and running Python code using the module namespace + +Provides support for locating and running Python scripts using the Python +module namespace instead of the native filesystem. Also provides support +to make it easier to execute other code without affecting the current +function namespace. + +This allows Python code to play nicely with non-filesystem based PEP 302 +importers when locating support scripts as well as when importing modules. +""" +# Written by Nick Coghlan +# to implement PEP 338 (Executing Modules as Scripts) + +import sys +import imp + +__all__ = [ + "run_module", +] + +try: + _get_loader = imp.get_loader +except AttributeError: + # get_loader() is not provided by the imp module, so emulate it + # as best we can using the PEP 302 import machinery exposed since + # Python 2.3. The emulation isn't perfect, but the differences + # in the way names are shadowed shouldn't matter in practice. + import os.path + import marshal # Handle compiled Python files + + # This helper is needed in order for the PEP 302 emulation to + # correctly handle compiled files + def _read_compiled_file(compiled_file): + magic = compiled_file.read(4) + if magic != imp.get_magic(): + return None + try: + compiled_file.read(4) # Skip timestamp + return marshal.load(compiled_file) + except Exception: + return None + + class _AbsoluteImporter(object): + """PEP 302 importer wrapper for top level import machinery""" + def find_module(self, mod_name, path=None): + if path is not None: + return None + try: + file, filename, mod_info = imp.find_module(mod_name) + except ImportError: + return None + suffix, mode, mod_type = mod_info + if mod_type == imp.PY_SOURCE: + loader = _SourceFileLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.PY_COMPILED: + loader = _CompiledFileLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.PKG_DIRECTORY: + loader = _PackageDirLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.C_EXTENSION: + loader = _FileSystemLoader(mod_name, file, + filename, mod_info) + else: + loader = _BasicLoader(mod_name, file, + filename, mod_info) + return loader + + + class _FileSystemImporter(object): + """PEP 302 importer wrapper for filesystem based imports""" + def __init__(self, path_item=None): + if path_item is not None: + if path_item != '' and not os.path.isdir(path_item): + raise ImportError("%s is not a directory" % path_item) + self.path_dir = path_item + else: + raise ImportError("Filesystem importer requires " + "a directory name") + + def find_module(self, mod_name, path=None): + if path is not None: + return None + path_dir = self.path_dir + if path_dir == '': + path_dir = os.getcwd() + sub_name = mod_name.rsplit(".", 1)[-1] + try: + file, filename, mod_info = imp.find_module(sub_name, + [path_dir]) + except ImportError: + return None + if not filename.startswith(path_dir): + return None + suffix, mode, mod_type = mod_info + if mod_type == imp.PY_SOURCE: + loader = _SourceFileLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.PY_COMPILED: + loader = _CompiledFileLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.PKG_DIRECTORY: + loader = _PackageDirLoader(mod_name, file, + filename, mod_info) + elif mod_type == imp.C_EXTENSION: + loader = _FileSystemLoader(mod_name, file, + filename, mod_info) + else: + loader = _BasicLoader(mod_name, file, + filename, mod_info) + return loader + + + class _BasicLoader(object): + """PEP 302 loader wrapper for top level import machinery""" + def __init__(self, mod_name, file, filename, mod_info): + self.mod_name = mod_name + self.file = file + self.filename = filename + self.mod_info = mod_info + + def _fix_name(self, mod_name): + if mod_name is None: + mod_name = self.mod_name + elif mod_name != self.mod_name: + raise ImportError("Loader for module %s cannot handle " + "module %s" % (self.mod_name, mod_name)) + return mod_name + + def load_module(self, mod_name=None): + mod_name = self._fix_name(mod_name) + mod = imp.load_module(mod_name, self.file, + self.filename, self.mod_info) + mod.__loader__ = self # for introspection + return mod + + def get_code(self, mod_name=None): + return None + + def get_source(self, mod_name=None): + return None + + def is_package(self, mod_name=None): + return False + + def close(self): + if self.file: + self.file.close() + + def __del__(self): + self.close() + + + class _FileSystemLoader(_BasicLoader): + """PEP 302 loader wrapper for filesystem based imports""" + def get_code(self, mod_name=None): + mod_name = self._fix_name(mod_name) + return self._get_code(mod_name) + + def get_data(self, pathname): + return open(pathname, "rb").read() + + def get_filename(self, mod_name=None): + mod_name = self._fix_name(mod_name) + return self._get_filename(mod_name) + + def get_source(self, mod_name=None): + mod_name = self._fix_name(mod_name) + return self._get_source(mod_name) + + def is_package(self, mod_name=None): + mod_name = self._fix_name(mod_name) + return self._is_package(mod_name) + + def _get_code(self, mod_name): + return None + + def _get_filename(self, mod_name): + return self.filename + + def _get_source(self, mod_name): + return None + + def _is_package(self, mod_name): + return False + + class _PackageDirLoader(_FileSystemLoader): + """PEP 302 loader wrapper for PKG_DIRECTORY directories""" + def _is_package(self, mod_name): + return True + + + class _SourceFileLoader(_FileSystemLoader): + """PEP 302 loader wrapper for PY_SOURCE modules""" + def _get_code(self, mod_name): + return compile(self._get_source(mod_name), + self.filename, 'exec') + + def _get_source(self, mod_name): + f = self.file + f.seek(0) + return f.read() + + + class _CompiledFileLoader(_FileSystemLoader): + """PEP 302 loader wrapper for PY_COMPILED modules""" + def _get_code(self, mod_name): + f = self.file + f.seek(0) + return _read_compiled_file(f) + + + def _get_importer(path_item): + """Retrieve a PEP 302 importer for the given path item + + The returned importer is cached in sys.path_importer_cache + if it was newly created by a path hook. + + If there is no importer, a wrapper around the basic import + machinery is returned. This wrapper is never inserted into + the importer cache (None is inserted instead). + + The cache (or part of it) can be cleared manually if a + rescan of sys.path_hooks is necessary. + """ + try: + importer = sys.path_importer_cache[path_item] + except KeyError: + for path_hook in sys.path_hooks: + try: + importer = path_hook(path_item) + break + except ImportError: + pass + else: + importer = None + sys.path_importer_cache[path_item] = importer + if importer is None: + try: + importer = _FileSystemImporter(path_item) + except ImportError: + pass + return importer + + + def _get_path_loader(mod_name, path=None): + """Retrieve a PEP 302 loader using a path importer""" + if path is None: + path = sys.path + absolute_loader = _AbsoluteImporter().find_module(mod_name) + if isinstance(absolute_loader, _FileSystemLoader): + # Found in filesystem, so scan path hooks + # before accepting this one as the right one + loader = None + else: + # Not found in filesystem, so use top-level loader + loader = absolute_loader + else: + loader = absolute_loader = None + if loader is None: + for path_item in path: + importer = _get_importer(path_item) + if importer is not None: + loader = importer.find_module(mod_name) + if loader is not None: + # Found a loader for our module + break + else: + # No path hook found, so accept the top level loader + loader = absolute_loader + return loader + + def _get_package(pkg_name): + """Retrieve a named package""" + pkg = __import__(pkg_name) + sub_pkg_names = pkg_name.split(".") + for sub_pkg in sub_pkg_names[1:]: + pkg = getattr(pkg, sub_pkg) + return pkg + + def _get_loader(mod_name, path=None): + """Retrieve a PEP 302 loader for the given module or package + + If the module or package is accessible via the normal import + mechanism, a wrapper around the relevant part of that machinery + is returned. + + Non PEP 302 mechanisms (e.g. the Windows registry) used by the + standard import machinery to find files in alternative locations + are partially supported, but are searched AFTER sys.path. Normally, + these locations are searched BEFORE sys.path, preventing sys.path + entries from shadowing them. + For this to cause a visible difference in behaviour, there must + be a module or package name that is accessible via both sys.path + and one of the non PEP 302 file system mechanisms. In this case, + the emulation will find the former version, while the builtin + import mechanism will find the latter. + Items of the following types can be affected by this discrepancy: + imp.C_EXTENSION + imp.PY_SOURCE + imp.PY_COMPILED + imp.PKG_DIRECTORY + """ + try: + loader = sys.modules[mod_name].__loader__ + except (KeyError, AttributeError): + loader = None + if loader is None: + imp.acquire_lock() + try: + # Module not in sys.modules, or uses an unhooked loader + parts = mod_name.rsplit(".", 1) + if len(parts) == 2: + # Sub package, so use parent package's path + pkg_name, sub_name = parts + if pkg_name and pkg_name[0] != '.': + if path is not None: + raise ImportError("Path argument must be None " + "for a dotted module name") + pkg = _get_package(pkg_name) + try: + path = pkg.__path__ + except AttributeError: + raise ImportError(pkg_name + + " is not a package") + else: + raise ImportError("Relative import syntax is not " + "supported by _get_loader()") + else: + # Top level module, so stick with default path + sub_name = mod_name + + for importer in sys.meta_path: + loader = importer.find_module(mod_name, path) + if loader is not None: + # Found a metahook to handle the module + break + else: + # Handling via the standard path mechanism + loader = _get_path_loader(mod_name, path) + finally: + imp.release_lock() + return loader + + +# This helper is needed due to a missing component in the PEP 302 +# loader protocol (specifically, "get_filename" is non-standard) +def _get_filename(loader, mod_name): + try: + get_filename = loader.get_filename + except AttributeError: + return None + else: + return get_filename(mod_name) + +# ------------------------------------------------------------ +# Done with the import machinery emulation, on with the code! + +def _run_code(code, run_globals, init_globals, + mod_name, mod_fname, mod_loader): + """Helper for _run_module_code""" + if init_globals is not None: + run_globals.update(init_globals) +# update using kwargs don't work on pypy yet +# run_globals.update(__name__=mod_name, +# __file__=mod_fname, +# __loader__=mod_loader) + run_globals.update({'__name__': mod_name, + '__file__': mod_fname, + '__loader__': mod_loader}) + + exec code in run_globals + return run_globals + +def _run_module_code(code, init_globals=None, + mod_name=None, mod_fname=None, + mod_loader=None, alter_sys=False): + """Helper for run_module""" + # Set up the top level namespace dictionary + if alter_sys: + # Modify sys.argv[0] and sys.module[mod_name] + temp_module = imp.new_module(mod_name) + mod_globals = temp_module.__dict__ + saved_argv0 = sys.argv[0] + restore_module = mod_name in sys.modules + if restore_module: + saved_module = sys.modules[mod_name] + imp.acquire_lock() + try: + sys.argv[0] = mod_fname + sys.modules[mod_name] = temp_module + try: + _run_code(code, mod_globals, init_globals, + mod_name, mod_fname, mod_loader) + finally: + sys.argv[0] = saved_argv0 + if restore_module: + sys.modules[mod_name] = saved_module + else: + del sys.modules[mod_name] + finally: + imp.release_lock() + # Copy the globals of the temporary module, as they + # may be cleared when the temporary module goes away + return mod_globals.copy() + else: + # Leave the sys module alone + return _run_code(code, {}, init_globals, + mod_name, mod_fname, mod_loader) + + +def run_module(mod_name, init_globals=None, + run_name=None, alter_sys=False): + """Execute a module's code without importing it + + Returns the resulting top level namespace dictionary + """ + loader = _get_loader(mod_name) + if loader is None: + raise ImportError("No module named " + mod_name) + code = loader.get_code(mod_name) + if code is None: + raise ImportError("No code object available for " + mod_name) + filename = _get_filename(loader, mod_name) + if run_name is None: + run_name = mod_name + return _run_module_code(code, init_globals, run_name, + filename, loader, alter_sys) + + +if __name__ == "__main__": + # Run the module specified as the next command line argument + if len(sys.argv) < 2: + print >> sys.stderr, "No module specified for execution" + else: + del sys.argv[0] # Make the requested module sys.argv[0] + run_module(sys.argv[0], run_name="__main__", alter_sys=True) Added: pypy/dist/pypy/lib/test2/test_runpy.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/test2/test_runpy.py Tue Aug 15 15:03:06 2006 @@ -0,0 +1,151 @@ +# Test the runpy module +import unittest +import os +import os.path +import sys +import tempfile +from pypy.lib.runpy import _run_module_code, run_module + + +verbose = 0 + +# Set up the test code and expected results +class TestRunModuleCode: + + expected_result = ["Top level assignment", "Lower level reference"] + test_source = ( + "# Check basic code execution\n" + "result = ['Top level assignment']\n" + "def f():\n" + " result.append('Lower level reference')\n" + "f()\n" + "# Check the sys module\n" + "import sys\n" + "run_argv0 = sys.argv[0]\n" + "if __name__ in sys.modules:\n" + " run_name = sys.modules[__name__].__name__\n" + "# Check nested operation\n" + "import pypy.lib.runpy\n" + "nested = pypy.lib.runpy._run_module_code('x=1\\n', mod_name='',\n" + " alter_sys=True)\n" + ) + + + def test_run_module_code(self): + initial = object() + name = "" + file = "Some other nonsense" + loader = "Now you're just being silly" + d1 = dict(initial=initial) + saved_argv0 = sys.argv[0] + d2 = _run_module_code(self.test_source, + d1, + name, + file, + loader, + True) + assert "result" not in d1 + assert d2["initial"] is initial + assert d2["result"] == self.expected_result + assert d2["nested"]["x"] == 1 + assert d2["__name__"] is name + assert d2["run_name"] is name + assert d2["__file__"] is file + assert d2["run_argv0"] is file + assert d2["__loader__"] is loader + assert sys.argv[0] is saved_argv0 + assert name not in sys.modules + + def test_run_module_code_defaults(self): + saved_argv0 = sys.argv[0] + d = _run_module_code(self.test_source) + assert d["result"] == self.expected_result + assert d["__name__"] is None + assert d["__file__"] is None + assert d["__loader__"] is None + assert d["run_argv0"] is saved_argv0 + assert "run_name" not in d + assert sys.argv[0] is saved_argv0 + +class TestRunModule: + + def expect_import_error(self, mod_name): + try: + run_module(mod_name) + except ImportError: + pass + else: + assert false, "Expected import error for " + mod_name + + def test_invalid_names(self): + self.expect_import_error("sys") + self.expect_import_error("sys.imp.eric") + self.expect_import_error("os.path.half") + self.expect_import_error("a.bee") + self.expect_import_error(".howard") + self.expect_import_error("..eaten") + + def test_library_module(self): + run_module("pypy.lib.runpy") + + def _make_pkg(self, source, depth): + pkg_name = "__runpy_pkg__" + init_fname = "__init__"+os.extsep+"py" + test_fname = "runpy_test"+os.extsep+"py" + pkg_dir = sub_dir = tempfile.mkdtemp() + if verbose: print " Package tree in:", sub_dir + sys.path.insert(0, pkg_dir) + if verbose: print " Updated sys.path:", sys.path[0] + for i in range(depth): + sub_dir = os.path.join(sub_dir, pkg_name) + os.mkdir(sub_dir) + if verbose: print " Next level in:", sub_dir + pkg_fname = os.path.join(sub_dir, init_fname) + pkg_file = open(pkg_fname, "w") + pkg_file.write("__path__ = ['%s']\n" % sub_dir) + pkg_file.close() + if verbose: print " Created:", pkg_fname + mod_fname = os.path.join(sub_dir, test_fname) + mod_file = open(mod_fname, "w") + mod_file.write(source) + mod_file.close() + if verbose: print " Created:", mod_fname + mod_name = (pkg_name+".")*depth + "runpy_test" + return pkg_dir, mod_fname, mod_name + + def _del_pkg(self, top, depth, mod_name): + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(top) + if verbose: print " Removed package tree" + for i in range(depth+1): # Don't forget the module itself + parts = mod_name.rsplit(".", i) + entry = parts[0] + del sys.modules[entry] + if verbose: print " Removed sys.modules entries" + del sys.path[0] + if verbose: print " Removed sys.path entry" + + def _check_module(self, depth): + pkg_dir, mod_fname, mod_name = ( + self._make_pkg("x=1\n", depth)) + try: + if verbose: print "Running from source:", mod_name + d1 = run_module(mod_name) # Read from source + __import__(mod_name) + os.remove(mod_fname) + if verbose: print "Running from compiled:", mod_name + d2 = run_module(mod_name) # Read from bytecode + finally: + self._del_pkg(pkg_dir, depth, mod_name) + assert d1["x"] == d2["x"] == 1 + if verbose: print "Module executed successfully" + + def test_run_module(self): + for depth in range(4): + if verbose: print "Testing package depth:", depth + self._check_module(depth) + From arigo at codespeak.net Tue Aug 15 15:11:21 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 15:11:21 +0200 (CEST) Subject: [pypy-svn] r31321 - pypy/dist/pypy/objspace/std Message-ID: <20060815131121.746A710074@code0.codespeak.net> Author: arigo Date: Tue Aug 15 15:11:15 2006 New Revision: 31321 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/dictstrobject.py Log: issue240 resolved Python-dev has no opinion about which of the list or dict behaviors is right. For now, let's just stick to CPython as closely as possible, with a comment in the code. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 15 15:11:15 2006 @@ -363,7 +363,8 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictMultiObject(space)]) # default argument - w_dict.implementation = w_dict.implementation.clear() + # w_dict.implementation = w_dict.implementation.clear() + # ^^^ disabled only for CPython compatibility try: space.getattr(w_src, space.wrap("keys")) except OperationError: Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Tue Aug 15 15:11:15 2006 @@ -51,7 +51,7 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictObject(space)]) # default argument - w_dict.content.clear() + # w_dict.content.clear() - disabled only for CPython compatibility try: space.getattr(w_src, space.wrap("keys")) except OperationError: Modified: pypy/dist/pypy/objspace/std/dictstrobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictstrobject.py (original) +++ pypy/dist/pypy/objspace/std/dictstrobject.py Tue Aug 15 15:11:15 2006 @@ -103,10 +103,10 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictStrObject(space)]) # default argument - if w_dict.content is None: - w_dict.content_str.clear() - else: - w_dict.content.clear() + #if w_dict.content is None: - + # w_dict.content_str.clear() - disabled only for CPython compatibility + #else: - + # w_dict.content.clear() - try: space.getattr(w_src, space.wrap("keys")) From arigo at codespeak.net Tue Aug 15 15:22:38 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Aug 2006 15:22:38 +0200 (CEST) Subject: [pypy-svn] r31322 - pypy/dist/pypy/interpreter Message-ID: <20060815132238.E41A410078@code0.codespeak.net> Author: arigo Date: Tue Aug 15 15:21:55 2006 New Revision: 31322 Modified: pypy/dist/pypy/interpreter/error.py Log: Clear OperationError.debug_excs as well from clear(). This is necessary to really break the ref cycles in typedef.Proto.__del__(). While I'm at it, hide the debug_excs list completely from a translated pypy-c. Modified: pypy/dist/pypy/interpreter/error.py ============================================================================== --- pypy/dist/pypy/interpreter/error.py (original) +++ pypy/dist/pypy/interpreter/error.py Tue Aug 15 15:21:55 2006 @@ -1,5 +1,6 @@ import autopath import os, sys +from pypy.rpython.objectmodel import we_are_translated AUTO_DEBUG = os.getenv('PYPY_DEBUG') RECORD_INTERPLEVEL_TRACEBACK = True @@ -22,13 +23,16 @@ self.w_type = w_type self.w_value = w_value self.application_traceback = tb - self.debug_excs = [] + if not we_are_translated(): + self.debug_excs = [] def clear(self, space): # for sys.exc_clear() self.w_type = space.w_None self.w_value = space.w_None self.application_traceback = None + if not we_are_translated(): + del self.debug_excs[:] def match(self, space, w_check_class): "Check if this application-level exception matches 'w_check_class'." @@ -75,7 +79,6 @@ """Records the current traceback inside the interpreter. This traceback is only useful to debug the interpreter, not the application.""" - from pypy.rpython.objectmodel import we_are_translated if not we_are_translated(): if RECORD_INTERPLEVEL_TRACEBACK: self.debug_excs.append(sys.exc_info()) From pedronis at codespeak.net Tue Aug 15 15:50:08 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 15 Aug 2006 15:50:08 +0200 (CEST) Subject: [pypy-svn] r31323 - pypy/dist/pypy/doc Message-ID: <20060815135008.7B53B10033@code0.codespeak.net> Author: pedronis Date: Tue Aug 15 15:50:06 2006 New Revision: 31323 Modified: pypy/dist/pypy/doc/extradoc.txt Log: add vpw talk Modified: pypy/dist/pypy/doc/extradoc.txt ============================================================================== --- pypy/dist/pypy/doc/extradoc.txt (original) +++ pypy/dist/pypy/doc/extradoc.txt Tue Aug 15 15:50:06 2006 @@ -5,6 +5,9 @@ Talks and Presentations ---------------------------------- +* `PyPy Status`_ talk, given by Samuele Pedroni at the Vancouner + Python Workshop 2006 (August). + * `Trouble in Paradise`_: the Open Source Project PyPy, EU-funding and Agile Practices talk, by Bea During at Agile 2006 (experience report). @@ -55,7 +58,7 @@ .. _`Sprint Driven Development`: http://codespeak.net/pypy/extradoc/talk/during-xp2006-sprints.pdf .. _`Kill -1`: http://codespeak.net/pypy/extradoc/talk/kill_1_agiletalk.pdf .. _`Open Source, EU-Funding and Agile Methods`: http://codespeak.net/pypy/extradoc/talk/agility.pdf - +.. _`PyPy Status`: http://codespeak.net/pypy/extradoc/talk/vancouver/talk.html Related projects ---------------------------------- From mwh at codespeak.net Tue Aug 15 16:30:09 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 15 Aug 2006 16:30:09 +0200 (CEST) Subject: [pypy-svn] r31324 - in pypy/dist/pypy: interpreter objspace/std Message-ID: <20060815143009.A71EA10033@code0.codespeak.net> Author: mwh Date: Tue Aug 15 16:30:08 2006 New Revision: 31324 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/objspace/std/dictmultiobject.py Log: use the config system to decide whether to use the measuring implementation. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Tue Aug 15 16:30:08 2006 @@ -163,9 +163,7 @@ w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) - # XXX ugh: - from pypy.objspace.std.dictmultiobject import MEASURE_DICT - if MEASURE_DICT: + if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report report() Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 15 16:30:08 2006 @@ -112,217 +112,215 @@ def items(self): return self.content.items() -# yeah, so this should do something clever with pypy.config -MEASURE_DICT = False +import time, py -if MEASURE_DICT: - import time, py - - class DictInfo(object): - _dict_infos = [] - def __init__(self): - self.id = len(self._dict_infos) - - self.getitems = 0; self.setitems = 0; self.delitems = 0 - self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 - self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 - self.keys = 0; self.values = 0; self.items = 0 - - self.maxcontents = 0 - - self.reads = 0 - self.hits = self.misses = 0 - self.writes = 0 - self.iterations = 0 - self.listings = 0 - - self.seen_non_string_in_write = 0 - self.seen_non_string_in_read_first = 0 - self.size_on_non_string_seen_in_read = -1 - self.size_on_non_string_seen_in_write = -1 - - self.createtime = time.time() - self.lifetime = -1.0 - - if not we_are_translated(): - # very probable stack from here: - # 0 - us - # 1 - MeasuringDictImplementation.__init__ - # 2 - W_DictMultiObject.__init__ - # 3 - space.newdict - # 4 - newdict's caller. let's look at that - try: - frame = sys._getframe(4) - except ValueError: - pass # might be at import time - else: - self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) - - self._dict_infos.append(self) - def __repr__(self): - args = [] - for k in sorted(self.__dict__): - v = self.__dict__[k] - if v != 0: - args.append('%s=%r'%(k, v)) - return ''%(', '.join(args),) - - class OnTheWayOut: - def __init__(self, info): - self.info = info - def __del__(self): - self.info.lifetime = time.time() - self.info.createtime - - class MeasuringDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - self.content = r_dict(space.eq_w, space.hash_w) - self.info = DictInfo() - self.thing_with_del = OnTheWayOut(self.info) - - def __repr__(self): - return "%s<%s>" % (self.__class__.__name__, self.content) - - def _is_str(self, w_key): - space = self.space - return space.is_true(space.isinstance(w_key, space.w_str)) - def _read(self, w_key): - self.info.reads += 1 - if not self.info.seen_non_string_in_write \ - and not self.info.seen_non_string_in_read_first \ - and not self._is_str(w_key): - self.info.seen_non_string_in_read_first = True - self.info.size_on_non_string_seen_in_read = len(self.content) - hit = w_key in self.content - if hit: - self.info.hits += 1 +class DictInfo(object): + _dict_infos = [] + def __init__(self): + self.id = len(self._dict_infos) + + self.getitems = 0; self.setitems = 0; self.delitems = 0 + self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 + self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 + self.keys = 0; self.values = 0; self.items = 0 + + self.maxcontents = 0 + + self.reads = 0 + self.hits = self.misses = 0 + self.writes = 0 + self.iterations = 0 + self.listings = 0 + + self.seen_non_string_in_write = 0 + self.seen_non_string_in_read_first = 0 + self.size_on_non_string_seen_in_read = -1 + self.size_on_non_string_seen_in_write = -1 + + self.createtime = time.time() + self.lifetime = -1.0 + + if not we_are_translated(): + # very probable stack from here: + # 0 - us + # 1 - MeasuringDictImplementation.__init__ + # 2 - W_DictMultiObject.__init__ + # 3 - space.newdict + # 4 - newdict's caller. let's look at that + try: + frame = sys._getframe(4) + except ValueError: + pass # might be at import time else: - self.info.misses += 1 + self.sig = '(%s:%s)%s'%(frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name) - def getitem(self, w_key): - self.info.getitems += 1 - self._read(w_key) - return self.content[w_key] - def setitem(self, w_key, w_value): - if not self.info.seen_non_string_in_write and not self._is_str(w_key): - self.info.seen_non_string_in_write = True - self.info.size_on_non_string_seen_in_write = len(self.content) - self.info.setitems += 1 - self.info.writes += 1 - self.content[w_key] = w_value - self.info.maxcontents = max(self.info.maxcontents, len(self.content)) - return self - def delitem(self, w_key): - if not self.info.seen_non_string_in_write \ - and not self.info.seen_non_string_in_read_first \ - and not self._is_str(w_key): - self.info.seen_non_string_in_read_first = True - self.info.size_on_non_string_seen_in_read = len(self.content) - self.info.delitems += 1 - self.info.writes += 1 - del self.content[w_key] - return self - - def length(self): - self.info.lengths += 1 - return len(self.content) - def clear(self): - self.info.clears += 1 - self.info.writes += 1 - self.content.clear() - return self - def has_key(self, w_lookup): - self.info.has_keys += 1 - self._read(w_lookup) - return w_lookup in self.content - def get(self, w_lookup, w_default): - self.info.gets += 1 - self._read(w_lookup) - return self.content.get(w_lookup, w_default) - - def iteritems(self): - self.info.iteritems += 1 - self.info.iterations += 1 - return self.content.iteritems() - def iterkeys(self): - self.info.iterkeys += 1 - self.info.iterations += 1 - return self.content.iterkeys() - def itervalues(self): - self.info.itervalues += 1 - self.info.iterations += 1 - return self.content.itervalues() - - def keys(self): - self.info.keys += 1 - self.info.listings += 1 - return self.content.keys() - def values(self): - self.info.values += 1 - self.info.listings += 1 - return self.content.values() - def items(self): - self.info.items += 1 - self.info.listings += 1 - return self.content.items() - - _example = DictInfo() - del DictInfo._dict_infos[-1] - tmpl = 'os.write(fd, "%(attr)s" + ": " + str(info.%(attr)s) + "\\n")' - bodySrc = [] - for attr in sorted(_example.__dict__): - if attr == 'sig': - continue - bodySrc.append(tmpl%locals()) - exec py.code.Source(''' - def _report_one(fd, info): - os.write(fd, "_address" + ": " + str(id(info)) + "\\n") - %s - '''%'\n '.join(bodySrc)).compile() - - def report(): - os.write(2, "starting to report!\n") - fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) - for info in DictInfo._dict_infos: - os.write(fd, '------------------\n') - _report_one(fd, info) - os.close(fd) - os.write(2, "reporting done!\n") - - def reportDictInfo(): - d = {} - if not DictInfo._dict_infos: - return - stillAlive = 0 - totLifetime = 0.0 - for info in DictInfo._dict_infos: - for attr in info.__dict__: - if attr == 'maxcontents': - continue - v = info.__dict__[attr] - if not isinstance(v, int): - continue - d[attr] = d.get(attr, 0) + v - if info.lifetime != -1.0: - totLifetime += info.lifetime - else: - stillAlive += 1 - import cPickle - cPickle.dump(DictInfo._dict_infos, open('dictinfos.pickle', 'wb')) - print 'reporting on', len(DictInfo._dict_infos), 'dictionaries' - if stillAlive != len(DictInfo._dict_infos): - print 'average lifetime', totLifetime/(len(DictInfo._dict_infos) - stillAlive), - print '('+str(stillAlive), 'still alive)' - print d + self._dict_infos.append(self) + def __repr__(self): + args = [] + for k in sorted(self.__dict__): + v = self.__dict__[k] + if v != 0: + args.append('%s=%r'%(k, v)) + return ''%(', '.join(args),) + +class OnTheWayOut: + def __init__(self, info): + self.info = info + def __del__(self): + self.info.lifetime = time.time() - self.info.createtime + +class MeasuringDictImplementation(DictImplementation): + def __init__(self, space): + self.space = space + self.content = r_dict(space.eq_w, space.hash_w) + self.info = DictInfo() + self.thing_with_del = OnTheWayOut(self.info) + + def __repr__(self): + return "%s<%s>" % (self.__class__.__name__, self.content) + + def _is_str(self, w_key): + space = self.space + return space.is_true(space.isinstance(w_key, space.w_str)) + def _read(self, w_key): + self.info.reads += 1 + if not self.info.seen_non_string_in_write \ + and not self.info.seen_non_string_in_read_first \ + and not self._is_str(w_key): + self.info.seen_non_string_in_read_first = True + self.info.size_on_non_string_seen_in_read = len(self.content) + hit = w_key in self.content + if hit: + self.info.hits += 1 + else: + self.info.misses += 1 + + def getitem(self, w_key): + self.info.getitems += 1 + self._read(w_key) + return self.content[w_key] + def setitem(self, w_key, w_value): + if not self.info.seen_non_string_in_write and not self._is_str(w_key): + self.info.seen_non_string_in_write = True + self.info.size_on_non_string_seen_in_write = len(self.content) + self.info.setitems += 1 + self.info.writes += 1 + self.content[w_key] = w_value + self.info.maxcontents = max(self.info.maxcontents, len(self.content)) + return self + def delitem(self, w_key): + if not self.info.seen_non_string_in_write \ + and not self.info.seen_non_string_in_read_first \ + and not self._is_str(w_key): + self.info.seen_non_string_in_read_first = True + self.info.size_on_non_string_seen_in_read = len(self.content) + self.info.delitems += 1 + self.info.writes += 1 + del self.content[w_key] + return self + + def length(self): + self.info.lengths += 1 + return len(self.content) + def clear(self): + self.info.clears += 1 + self.info.writes += 1 + self.content.clear() + return self + def has_key(self, w_lookup): + self.info.has_keys += 1 + self._read(w_lookup) + return w_lookup in self.content + def get(self, w_lookup, w_default): + self.info.gets += 1 + self._read(w_lookup) + return self.content.get(w_lookup, w_default) + + def iteritems(self): + self.info.iteritems += 1 + self.info.iterations += 1 + return self.content.iteritems() + def iterkeys(self): + self.info.iterkeys += 1 + self.info.iterations += 1 + return self.content.iterkeys() + def itervalues(self): + self.info.itervalues += 1 + self.info.iterations += 1 + return self.content.itervalues() + + def keys(self): + self.info.keys += 1 + self.info.listings += 1 + return self.content.keys() + def values(self): + self.info.values += 1 + self.info.listings += 1 + return self.content.values() + def items(self): + self.info.items += 1 + self.info.listings += 1 + return self.content.items() + +_example = DictInfo() +del DictInfo._dict_infos[-1] +tmpl = 'os.write(fd, "%(attr)s" + ": " + str(info.%(attr)s) + "\\n")' +bodySrc = [] +for attr in sorted(_example.__dict__): + if attr == 'sig': + continue + bodySrc.append(tmpl%locals()) +exec py.code.Source(''' +def _report_one(fd, info): + os.write(fd, "_address" + ": " + str(id(info)) + "\\n") + %s +'''%'\n '.join(bodySrc)).compile() + +def report(): + if not DictInfo._dict_infos: + return + os.write(2, "starting to report!\n") + fd = os.open('dictinfo.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) + for info in DictInfo._dict_infos: + os.write(fd, '------------------\n') + _report_one(fd, info) + os.close(fd) + os.write(2, "reporting done!\n") + +def reportDictInfo(): + d = {} + if not DictInfo._dict_infos: + return + stillAlive = 0 + totLifetime = 0.0 + for info in DictInfo._dict_infos: + for attr in info.__dict__: + if attr == 'maxcontents': + continue + v = info.__dict__[attr] + if not isinstance(v, int): + continue + d[attr] = d.get(attr, 0) + v + if info.lifetime != -1.0: + totLifetime += info.lifetime + else: + stillAlive += 1 + import cPickle + cPickle.dump(DictInfo._dict_infos, open('dictinfos.pickle', 'wb')) + print 'reporting on', len(DictInfo._dict_infos), 'dictionaries' + if stillAlive != len(DictInfo._dict_infos): + print 'average lifetime', totLifetime/(len(DictInfo._dict_infos) - stillAlive), + print '('+str(stillAlive), 'still alive)' + print d - #import atexit - #atexit.register(reportDictInfo) +#import atexit +#atexit.register(reportDictInfo) class W_DictMultiObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef def __init__(w_self, space, w_otherdict=None): - if MEASURE_DICT: + if space.config.objspace.std.withdictmeasurement: w_self.implementation = MeasuringDictImplementation(space) else: w_self.implementation = EmptyDictImplementation(space) From mwh at codespeak.net Tue Aug 15 17:44:29 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 15 Aug 2006 17:44:29 +0200 (CEST) Subject: [pypy-svn] r31325 - pypy/dist/pypy/objspace/std/test Message-ID: <20060815154429.E2A4910074@code0.codespeak.net> Author: mwh Date: Tue Aug 15 17:44:28 2006 New Revision: 31325 Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py Log: fix the stress test in test_dictmultiobject. Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictobject.py Tue Aug 15 17:44:28 2006 @@ -377,6 +377,14 @@ return type(w_obj) w_str = str +class Config: + pass + +FakeSpace.config = Config() +FakeSpace.config.objspace = Config() +FakeSpace.config.objspace.std = Config() +FakeSpace.config.objspace.std.withdictmeasurement = False + from pypy.objspace.std.dictobject import getitem__Dict_ANY, setitem__Dict_ANY_ANY class TestDictImplementation: From mwh at codespeak.net Tue Aug 15 17:48:19 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 15 Aug 2006 17:48:19 +0200 (CEST) Subject: [pypy-svn] r31326 - pypy/dist/pypy/objspace/std Message-ID: <20060815154819.6FC0410074@code0.codespeak.net> Author: mwh Date: Tue Aug 15 17:48:18 2006 New Revision: 31326 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: cleanups Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Aug 15 17:48:18 2006 @@ -287,46 +287,14 @@ os.close(fd) os.write(2, "reporting done!\n") -def reportDictInfo(): - d = {} - if not DictInfo._dict_infos: - return - stillAlive = 0 - totLifetime = 0.0 - for info in DictInfo._dict_infos: - for attr in info.__dict__: - if attr == 'maxcontents': - continue - v = info.__dict__[attr] - if not isinstance(v, int): - continue - d[attr] = d.get(attr, 0) + v - if info.lifetime != -1.0: - totLifetime += info.lifetime - else: - stillAlive += 1 - import cPickle - cPickle.dump(DictInfo._dict_infos, open('dictinfos.pickle', 'wb')) - print 'reporting on', len(DictInfo._dict_infos), 'dictionaries' - if stillAlive != len(DictInfo._dict_infos): - print 'average lifetime', totLifetime/(len(DictInfo._dict_infos) - stillAlive), - print '('+str(stillAlive), 'still alive)' - print d - -#import atexit -#atexit.register(reportDictInfo) - class W_DictMultiObject(W_Object): from pypy.objspace.std.dicttype import dict_typedef as typedef - def __init__(w_self, space, w_otherdict=None): + def __init__(w_self, space): if space.config.objspace.std.withdictmeasurement: w_self.implementation = MeasuringDictImplementation(space) else: w_self.implementation = EmptyDictImplementation(space) - if w_otherdict is not None: - from pypy.objspace.std.dicttype import update1 - update1(space, w_self, w_otherdict) def initialize_content(w_self, list_pairs_w): impl = w_self.implementation @@ -466,7 +434,10 @@ return w_res def dict_copy__DictMulti(space, w_self): - return W_DictMultiObject(space, w_self) + from pypy.objspace.std.dicttype import update1 + w_new = W_DictMultiObject(space) + update1(space, w_new, w_self) + return w_new def dict_items__DictMulti(space, w_self): return space.newlist([space.newtuple([w_k, w_v]) for w_k, w_v in w_self.implementation.iteritems()]) From mwh at codespeak.net Wed Aug 16 09:30:24 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 16 Aug 2006 09:30:24 +0200 (CEST) Subject: [pypy-svn] r31330 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060816073024.8BDDE1007F@code0.codespeak.net> Author: mwh Date: Wed Aug 16 09:30:22 2006 New Revision: 31330 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/dictstrobject.py pypy/dist/pypy/objspace/std/dicttype.py pypy/dist/pypy/objspace/std/test/test_dictobject.py Log: Revert revision 31319: "Added keyword arguments to dict.update()." It broke translation. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 09:30:22 2006 @@ -344,11 +344,11 @@ w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_src) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_kwds) def getitem__DictMulti_ANY(space, w_dict, w_lookup): try: @@ -434,9 +434,9 @@ return w_res def dict_copy__DictMulti(space, w_self): - from pypy.objspace.std.dicttype import update1 + from pypy.objspace.std.dicttype import dict_update__ANY_ANY w_new = W_DictMultiObject(space) - update1(space, w_new, w_self) + dict_update__ANY_ANY(space, w_new, w_self) return w_new def dict_items__DictMulti(space, w_self): Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Wed Aug 16 09:30:22 2006 @@ -65,11 +65,11 @@ w_dict.content[w_k] = w_v else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_src) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_kwds) def getitem__Dict_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dictstrobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictstrobject.py (original) +++ pypy/dist/pypy/objspace/std/dictstrobject.py Wed Aug 16 09:30:22 2006 @@ -121,11 +121,11 @@ w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_src) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import update1 - update1(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import dict_update__ANY_ANY + dict_update__ANY_ANY(space, w_dict, w_kwds) def getitem__DictStr_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/dist/pypy/objspace/std/dicttype.py (original) +++ pypy/dist/pypy/objspace/std/dicttype.py Wed Aug 16 09:30:22 2006 @@ -30,7 +30,7 @@ dict_setdefault = SMM('setdefault', 3, defaults=(None,), doc='D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d' ' if k not in D') -dict_update = SMM('update', 1, general__args__=True, +dict_update = SMM('update', 2, defaults=((),), doc='D.update(E, **F) -> None. Update D from E and F:' ' for k in E: D[k] = E[k]\n(if E has keys else: for' ' (k, v) in E: D[k] = v) then: for k in F: D[k] =' @@ -55,7 +55,7 @@ # gateway is imported in the stdtypedef module app = gateway.applevel(''' - def update1(d, o): + def update(d, o): if hasattr(o, 'keys'): for k in o.keys(): d[k] = o[k] @@ -63,14 +63,6 @@ for k,v in o: d[k] = v - def update(d, *args, **kwargs): - if len(args) == 1: - update1(d, args[0]) - elif len(args) > 1: - raise TypeError("update takes at most 1 (non-keyword) argument") - if kwargs: - update1(d, kwargs) - def popitem(d): k = d.keys() if not k: @@ -118,7 +110,7 @@ ''', filename=__file__) #XXX what about dict.fromkeys()? -dict_update__ANY = app.interphook("update") +dict_update__ANY_ANY = app.interphook("update") dict_popitem__ANY = app.interphook("popitem") dict_get__ANY_ANY_ANY = app.interphook("get") dict_setdefault__ANY_ANY_ANY = app.interphook("setdefault") @@ -126,7 +118,6 @@ dict_iteritems__ANY = app.interphook("iteritems") dict_iterkeys__ANY = app.interphook("iterkeys") dict_itervalues__ANY = app.interphook("itervalues") -update1 = app.interphook("update1") register_all(vars(), globals()) Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictobject.py Wed Aug 16 09:30:22 2006 @@ -245,17 +245,7 @@ d = {} d.update() assert d == {} - - def test_update_kwargs(self): - d = {} - d.update(foo='bar', baz=1) - assert d == {'foo': 'bar', 'baz': 1} - - def test_update_dict_and_kwargs(self): - d = {} - d.update({'foo': 'bar'}, baz=1) - assert d == {'foo': 'bar', 'baz': 1} - + def test_values(self): d = {1:2, 3:4} vals = d.values() From mwh at codespeak.net Wed Aug 16 11:03:04 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 16 Aug 2006 11:03:04 +0200 (CEST) Subject: [pypy-svn] r31331 - pypy/dist/pypy/objspace/std Message-ID: <20060816090304.74E8A1007F@code0.codespeak.net> Author: mwh Date: Wed Aug 16 11:02:53 2006 New Revision: 31331 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: a SmallDictImplementation for dictionaries of size 1-4. doesn't seem to help all that much performance wise :/ Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 11:02:53 2006 @@ -42,7 +42,7 @@ def getitem(self, w_key): raise KeyError def setitem(self, w_key, w_value): - return RDictImplementation(self.space).setitem(w_key, w_value) + return SmallDictImplementation(self.space).setitem(w_key, w_value) def delitem(self, w_key): raise KeyError @@ -69,6 +69,103 @@ def items(self): return [] +class Entry(object): + def __init__(self): + self.hash = 0 + self.w_key = None + self.w_value = None + def __repr__(self): + return '<%r, %r, %r>'%(self.hash, self.w_key, self.w_value) + +class SmallDictImplementation(DictImplementation): + # XXX document the invariants here! + + def __init__(self, space): + self.space = space + self.entries = [Entry(), Entry(), Entry(), Entry(), Entry()] + self.valid = 0 + + def _lookup(self, w_key): + for i in range(self.valid): + assert self.entries[i].w_value is not None + for i in range(self.valid+1, 5): + assert self.entries[i].w_value is None + hash = self.space.hash_w(w_key) + i = 0 + last = self.entries[self.valid] + last.hash = hash + last.w_key = w_key + while 1: + look_entry = self.entries[i] + if look_entry.hash == hash and self.space.eq_w(look_entry.w_key, w_key): + return look_entry + i += 1 + + def _convert_to_rdict(self): + newimpl = RDictImplementation(self.space) + i = 0 + while 1: + entry = self.entries[i] + if entry.w_value is None: + break + newimpl.setitem(entry.w_key, entry.w_value) + i += 1 + return newimpl + + def getitem(self, w_key): + entry = self._lookup(w_key) + if entry.w_value: + return entry.w_value + else: + raise KeyError + def setitem(self, w_key, w_value): + if self.valid == 4: + return self._convert_to_rdict().setitem(w_key, w_value) + entry = self._lookup(w_key) + if entry.w_value is None: + self.valid += 1 + entry.w_value = w_value + return self + def delitem(self, w_key): + entry = self._lookup(w_key) + if entry.w_value: + for i in range(self.entries.index(entry), self.valid): + self.entries[i] = self.entries[i+1] + self.entries[self.valid] = entry + entry.w_value = None + self.valid -= 1 + return self + else: + raise KeyError + + def length(self): + return self.valid + def clear(self): + return EmptyDictImplementation(self.space) + def has_key(self, w_lookup): + return self._lookup(w_lookup).w_value is not None + def get(self, w_lookup, w_default): + entry = self._lookup(w_lookup) + w_value = entry.w_value + if w_value: + return w_value + else: + return w_default + + def iteritems(self): + return self._convert_to_rdict().iteritems() + def iterkeys(self): + return self._convert_to_rdict().iterkeys() + def itervalues(self): + return self._convert_to_rdict().itervalues() + + def keys(self): + return [self.entries[i].w_key for i in range(self.valid)] + def values(self): + return [self.entries[i].w_value for i in range(self.valid)] + def items(self): + return [(e.w_key, e.w_value) for e in [self.entries[i] for i in range(self.valid)]] + class RDictImplementation(DictImplementation): def __init__(self, space): self.space = space From mwh at codespeak.net Wed Aug 16 11:26:19 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 16 Aug 2006 11:26:19 +0200 (CEST) Subject: [pypy-svn] r31332 - pypy/dist/pypy/objspace/std Message-ID: <20060816092619.D61771007F@code0.codespeak.net> Author: mwh Date: Wed Aug 16 11:26:18 2006 New Revision: 31332 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: streamline DictImplementation interface. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 11:26:18 2006 @@ -5,21 +5,15 @@ class DictImplementation(object): -## def getitem(self, w_key): -## pass +## def get(self, w_lookup): +## return w_value or None ## def setitem(self, w_key, w_value): -## pass +## return implementation ## def delitem(self, w_key): -## pass +## return implementation ## def length(self): ## pass -## def clear(self, w_dict): -## pass -## def has_key(self, w_lookup): -## pass -## def get(self, w_lookup, w_default): -## pass ## def iteritems(self): ## pass @@ -39,21 +33,15 @@ def __init__(self, space): self.space = space - def getitem(self, w_key): - raise KeyError + def get(self, w_lookup): + return None def setitem(self, w_key, w_value): - return SmallDictImplementation(self.space).setitem(w_key, w_value) + return SmallDictImplementation(self.space, w_key, w_value) def delitem(self, w_key): raise KeyError def length(self): return 0 - def clear(self): - return self - def has_key(self, w_lookup): - return False - def get(self, w_lookup, w_default): - return w_default def iteritems(self): return RDictImplementation(self.space).iteritems() @@ -80,16 +68,19 @@ class SmallDictImplementation(DictImplementation): # XXX document the invariants here! - def __init__(self, space): + def __init__(self, space, w_key, w_value): self.space = space self.entries = [Entry(), Entry(), Entry(), Entry(), Entry()] - self.valid = 0 + self.entries[0].hash = space.hash_w(w_key) + self.entries[0].w_key = w_key + self.entries[0].w_value = w_value + self.valid = 1 def _lookup(self, w_key): - for i in range(self.valid): - assert self.entries[i].w_value is not None - for i in range(self.valid+1, 5): - assert self.entries[i].w_value is None +## for i in range(self.valid): +## assert self.entries[i].w_value is not None +## for i in range(self.valid+1, 5): +## assert self.entries[i].w_value is None hash = self.space.hash_w(w_key) i = 0 last = self.entries[self.valid] @@ -112,12 +103,6 @@ i += 1 return newimpl - def getitem(self, w_key): - entry = self._lookup(w_key) - if entry.w_value: - return entry.w_value - else: - raise KeyError def setitem(self, w_key, w_value): if self.valid == 4: return self._convert_to_rdict().setitem(w_key, w_value) @@ -140,17 +125,8 @@ def length(self): return self.valid - def clear(self): - return EmptyDictImplementation(self.space) - def has_key(self, w_lookup): - return self._lookup(w_lookup).w_value is not None - def get(self, w_lookup, w_default): - entry = self._lookup(w_lookup) - w_value = entry.w_value - if w_value: - return w_value - else: - return w_default + def get(self, w_lookup): + return self._lookup(w_lookup).w_value def iteritems(self): return self._convert_to_rdict().iteritems() @@ -174,8 +150,6 @@ def __repr__(self): return "%s<%s>" % (self.__class__.__name__, self.content) - def getitem(self, w_key): - return self.content[w_key] def setitem(self, w_key, w_value): self.content[w_key] = w_value return self @@ -188,12 +162,8 @@ def length(self): return len(self.content) - def clear(self): - return EmptyDictImplementation(self.space) - def has_key(self, w_lookup): - return w_lookup in self.content - def get(self, w_lookup, w_default): - return self.content.get(w_lookup, w_default) + def get(self, w_lookup): + return self.content.get(w_lookup, None) def iteritems(self): return self.content.iteritems() @@ -216,8 +186,8 @@ def __init__(self): self.id = len(self._dict_infos) - self.getitems = 0; self.setitems = 0; self.delitems = 0 - self.lengths = 0; self.clears = 0; self.has_keys = 0; self.gets = 0 + self.setitems = 0; self.delitems = 0 + self.lengths = 0; self.gets = 0 self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 self.keys = 0; self.values = 0; self.items = 0 @@ -292,10 +262,6 @@ else: self.info.misses += 1 - def getitem(self, w_key): - self.info.getitems += 1 - self._read(w_key) - return self.content[w_key] def setitem(self, w_key, w_value): if not self.info.seen_non_string_in_write and not self._is_str(w_key): self.info.seen_non_string_in_write = True @@ -319,15 +285,6 @@ def length(self): self.info.lengths += 1 return len(self.content) - def clear(self): - self.info.clears += 1 - self.info.writes += 1 - self.content.clear() - return self - def has_key(self, w_lookup): - self.info.has_keys += 1 - self._read(w_lookup) - return w_lookup in self.content def get(self, w_lookup, w_default): self.info.gets += 1 self._read(w_lookup) @@ -414,7 +371,11 @@ return w_self.implementation.length() def get(w_dict, w_key, w_default): - return w_dict.implementation.get(w_key, w_default) + w_value = w_dict.implementation.get(w_key) + if w_value is not None: + return w_value + else: + return w_default def set_str_keyed_item(w_dict, w_key, w_value): w_dict.implementation = w_dict.implementation.setitem(w_key, w_value) @@ -426,7 +387,7 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictMultiObject(space)]) # default argument - # w_dict.implementation = w_dict.implementation.clear() + # w_dict.implementation = EmptyDictImplementation(self.space) # ^^^ disabled only for CPython compatibility try: space.getattr(w_src, space.wrap("keys")) @@ -448,10 +409,10 @@ dict_update__ANY_ANY(space, w_dict, w_kwds) def getitem__DictMulti_ANY(space, w_dict, w_lookup): - try: - return w_dict.implementation.getitem(w_lookup) - except KeyError: - raise OperationError(space.w_KeyError, w_lookup) + w_value = w_dict.implementation.get(w_lookup) + if w_value is not None: + return w_value + raise OperationError(space.w_KeyError, w_lookup) def setitem__DictMulti_ANY_ANY(space, w_dict, w_newkey, w_newvalue): w_dict.implementation = w_dict.implementation.setitem(w_newkey, w_newvalue) @@ -466,7 +427,7 @@ return space.wrap(w_dict.implementation.length()) def contains__DictMulti_ANY(space, w_dict, w_lookup): - return space.newbool(w_dict.implementation.has_key(w_lookup)) + return space.newbool(w_dict.implementation.get(w_lookup) is not None) dict_has_key__DictMulti_ANY = contains__DictMulti_ANY @@ -480,9 +441,8 @@ if w_left.implementation.length() != w_right.implementation.length(): return space.w_False for w_key, w_val in w_left.implementation.iteritems(): - try: - w_rightval = w_right.implementation.getitem(w_key) - except KeyError: + w_rightval = w_right.implementation.get(w_key) + if w_rightval is None: return space.w_False if not space.eq_w(w_val, w_rightval): return space.w_False @@ -495,9 +455,8 @@ w_its_value = None for w_key, w_val in aimpl.iteritems(): if w_smallest_diff_a_key is None or space.is_true(space.lt(w_key, w_smallest_diff_a_key)): - try: - w_bvalue = bimpl.getitem(w_key) - except KeyError: + w_bvalue = bimpl.get(w_key) + if w_bvalue is None: w_its_value = w_val w_smallest_diff_a_key = w_key else: @@ -555,10 +514,10 @@ return W_DictMultiIter_Values(space, w_self.implementation) def dict_clear__DictMulti(space, w_self): - w_self.implementation = w_self.implementation.clear() + w_self.implementation = EmptyDictImplementation(space) def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): - return w_dict.implementation.get(w_lookup, w_default) + return w_dict.get(w_lookup, w_default) app = gateway.applevel(''' def dictrepr(currently_in_repr, d): From ac at codespeak.net Wed Aug 16 11:28:35 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Wed, 16 Aug 2006 11:28:35 +0200 (CEST) Subject: [pypy-svn] r31333 - pypy/dist/pypy/module/rctime Message-ID: <20060816092835.E333F1007F@code0.codespeak.net> Author: ac Date: Wed Aug 16 11:28:34 2006 New Revision: 31333 Modified: pypy/dist/pypy/module/rctime/__init__.py pypy/dist/pypy/module/rctime/app_time.py Log: Add strptime. Modified: pypy/dist/pypy/module/rctime/__init__.py ============================================================================== --- pypy/dist/pypy/module/rctime/__init__.py (original) +++ pypy/dist/pypy/module/rctime/__init__.py Wed Aug 16 11:28:34 2006 @@ -42,4 +42,5 @@ '_check_float': 'app_time._check_float', 'struct_time': 'app_time.struct_time', '__doc__': 'app_time.__doc__', + 'strptime': 'app_time.strptime', } Modified: pypy/dist/pypy/module/rctime/app_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/app_time.py (original) +++ pypy/dist/pypy/module/rctime/app_time.py Wed Aug 16 11:28:34 2006 @@ -73,7 +73,12 @@ a floating point number for subsecond precision.""" _check_float(secs) _float_sleep(secs) + +def strptime(*args, **kw): + import _strptime + return _strptime.strptime(*args, **kw) + __doc__ = """This module provides various functions to manipulate time values. There are two standard representations of time. One is the number From auc at codespeak.net Wed Aug 16 11:29:30 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 11:29:30 +0200 (CEST) Subject: [pypy-svn] r31334 - in pypy/dist/pypy: lib module/_stackless objspace objspace/cclp objspace/cclp/constraint objspace/cclp/constraint/test objspace/test Message-ID: <20060816092930.A09961007F@code0.codespeak.net> Author: auc Date: Wed Aug 16 11:29:21 2006 New Revision: 31334 Added: pypy/dist/pypy/objspace/cclp/constraint/distributor.py pypy/dist/pypy/objspace/cclp/constraint/test/problem.py Modified: pypy/dist/pypy/lib/_exceptions.py pypy/dist/pypy/module/_stackless/clonable.py pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py pypy/dist/pypy/objspace/cclp/interp_var.py pypy/dist/pypy/objspace/cclp/scheduler.py pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thread.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/cclp/types.py pypy/dist/pypy/objspace/logic.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: constraints distributor, interp version of var utilities ; some correctness issues to be fixed later Modified: pypy/dist/pypy/lib/_exceptions.py ============================================================================== --- pypy/dist/pypy/lib/_exceptions.py (original) +++ pypy/dist/pypy/lib/_exceptions.py Wed Aug 16 11:29:21 2006 @@ -442,10 +442,14 @@ #-- Logic object space specific stuff #XXX conditionalize me on '-o logic' -class LOError(Exception): pass +class LogicError(Exception): pass -class UnificationError(LOError): pass +class UnificationError(LogicError): pass class RebindingError(UnificationError): pass -class FutureBindingError(LOError): pass +class FutureBindingError(LogicError): pass -class AllBlockedError(LOError): pass +class AllBlockedError(LogicError): pass + +# constraints + +class ConsistencyError(LogicError): pass Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Wed Aug 16 11:29:21 2006 @@ -32,7 +32,7 @@ space.getexecutioncontext().subcontext_new(self) self._dead = False self._next = self._prev = self - self._cspace = space.w_None + self._cspace = None def hello(self): if we_are_translated(): Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Wed Aug 16 11:29:21 2006 @@ -19,19 +19,6 @@ all_mms = {} -#-- Exceptions --------------------------------------- - -class ConsistencyFailure(Exception): - """The repository is not in a consistent state""" - pass - -class DomainlessVariables(Exception): - """A constraint can't be defined on variables - without a domain""" - pass - -#-- Constraints ------------------------------------------ - class W_AbstractConstraint(W_Constraint): def __init__(self, object_space, w_variables): @@ -185,9 +172,6 @@ domain.remove_values([val for val in domain._values.content.keys() if val not in keep.content]) - except ConsistencyFailure: - raise ConsistencyFailure('Inconsistency while applying %s' % \ - repr(self)) except KeyError: # There are no more value in result_cache pass @@ -207,6 +191,8 @@ def make_expression(o_space, w_variables, w_formula): """create a new constraint of type Expression or BinaryExpression The chosen class depends on the number of variables in the constraint""" + assert isinstance(w_variables, W_ListObject) + assert isinstance(w_formula, W_StringObject) assert len(w_variables.wrappeditems) > 0 return W_Expression(o_space, w_variables, w_formula) app_make_expression = gateway.interp2app(make_expression) @@ -276,6 +262,7 @@ # function bolted into the space to serve as constructor def make_alldistinct(object_space, w_variables): + assert isinstance(w_variables, W_ListObject) assert len(w_variables.wrappeditems) > 0 return object_space.wrap(W_AllDistinct(object_space, w_variables)) app_make_alldistinct = gateway.interp2app(make_alldistinct) Added: pypy/dist/pypy/objspace/cclp/constraint/distributor.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/distributor.py Wed Aug 16 11:29:21 2006 @@ -0,0 +1,150 @@ +import math +from pypy.rpython.objectmodel import we_are_translated + +from pypy.interpreter.error import OperationError +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter import baseobjspace, typedef, gateway +from pypy.interpreter.gateway import interp2app + +from pypy.objspace.std.intobject import W_IntObject +from pypy.objspace.std.listobject import W_ListObject +from pypy.objspace.std.stringobject import W_StringObject + +from pypy.objspace.cclp.types import W_AbstractDistributor, Solution +from pypy.objspace.cclp.thunk import DistributorThunk +from pypy.objspace.cclp.misc import w, ClonableCoroutine +from pypy.objspace.cclp.global_state import scheduler +from pypy.objspace.cclp.interp_var import interp_free + +def spawn_distributor(space, distributor): + thread = ClonableCoroutine(space) + thread._cspace = ClonableCoroutine.w_getcurrent(space)._cspace + thunk = DistributorThunk(space, distributor, thread) + thread.bind(thunk) + if not we_are_translated(): + w("DISTRIBUTOR THREAD", str(id(thread))) + scheduler[0].add_new_thread(thread) + scheduler[0].schedule() + +def distribute(space, w_strategy): + assert isinstance(w_strategy, W_StringObject) + strat = space.str_w(w_strategy) + if strat == 'naive': + pass + elif strat == 'dichotomy': + dist = make_split_distributor(space, space.newint(2)) + spawn_distributor(space, dist) + else: + raise OperationError(space.w_RuntimeError, + space.wrap("please pick a strategy in (naive, dichotomy)")) +app_distribute = interp2app(distribute) + + +def arrange_domains(cs, variables): + """build a data structure from var to dom + that satisfies distribute & friends""" + new_doms = {} + for var in variables: + new_doms[var] = cs.dom(var).copy() + return new_doms + +class W_Distributor(W_AbstractDistributor): + """_distribute is left unimplemented.""" + + def w_fanout(self): + return self._space.newint(self._fanout) + + def fanout(self): + return self._fanout + + def _find_smallest_domain(self): + """returns the variable having the smallest domain. + (or one of such variables if there is a tie) + """ + vars_ = [var for var in self._cspace._store.values() + if var.w_dom.size() > 1] + if len(vars_) == 0: + raise Solution + best = vars_[0] + for var in vars_: + if var.w_dom.size() < best.w_dom.size(): + best = var + return best + + def w_distribute(self, w_choice): + assert isinstance(w_choice, W_IntObject) + self.distribute(self._space.int_w(w_choice) -1) + + def distribute(self, choice): + assert isinstance(choice, int) + variable = self.find_distribution_variable() + domain = variable.w_dom + self._do_distribute(domain, choice) + + def find_distribution_variable(self): + return self._find_smallest_domain() + + def _do_distribute(self, domain, choice): + """remove values from domain depending on choice""" + raise NotImplementedError + +W_Distributor.typedef = typedef.TypeDef("W_Distributor", + W_AbstractDistributor.typedef, + fanout = interp2app(W_Distributor.w_fanout), + distribute = interp2app(W_Distributor.w_distribute)) + + +class W_NaiveDistributor(W_Distributor): + """distributes domains by splitting the smallest domain in 2 new domains + The first new domain has a size of one, + and the second has all the other values""" + + + def _do_distribute(self, domain, choice): + values = domain.get_values() + #assert len(values) > 0 + if choice == 0: + domain.remove_values(values[1:]) + else: + domain.w_remove_value(values[0]) #XXX w ? not w ? + +W_NaiveDistributor.typedef = typedef.TypeDef( + "W_NaiveDistributor", + W_Distributor.typedef) + +def make_naive_distributor(object_space, fanout=2): + if not isinstance(fanout, int): + raise OperationError(object_space.w_RuntimeError, + object_space.wrap("fanout must be a positive integer")) + return object_space.wrap(W_NaiveDistributor(object_space, fanout)) +app_make_naive_distributor = interp2app(make_naive_distributor, + unwrap_spec = [baseobjspace.ObjSpace, int]) + + +class W_SplitDistributor(W_Distributor): + """distributes domains by splitting the smallest domain in + nb_subspaces equal parts or as equal as possible. + If nb_subspaces is 0, then the smallest domain is split in + domains of size 1""" + + def _subdomains(self): + """returns the min number of partitions + for a domain to be distributed""" + to_split = self._find_smallest_domain() + if self._fanout > 0: + return min(self._fanout, to_split.w_dom.size()) + else: + return to_split.w_dom.size() + + def _do_distribute(self, domain, choice): + values = domain.get_values() + nb_elts = max(1, len(values)*1./self._subdomains()) + start, end = (int(math.floor(choice * nb_elts)), + int(math.floor((choice + 1) * nb_elts))) + domain.remove_values(values[:start]) + domain.remove_values(values[end:]) + +def make_split_distributor(space, w_fanout): + return W_SplitDistributor(space, space.int_w(w_fanout)) +app_make_split_distributor = interp2app(make_split_distributor) + Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Wed Aug 16 11:29:21 2006 @@ -7,15 +7,11 @@ from pypy.objspace.std.model import StdObjSpaceMultiMethod -from pypy.objspace.cclp.types import W_AbstractDomain, W_Var +from pypy.objspace.cclp.types import W_AbstractDomain, W_Var, ConsistencyError from pypy.objspace.cclp.interp_var import interp_bind all_mms = {} -class ConsistencyFailure(Exception): - """The repository is not in a consistent state""" - pass - class W_FiniteDomain(W_AbstractDomain): """ @@ -42,16 +38,11 @@ def _value_removed(self): """The implementation of remove_value should call this method""" - interp_bind(self._space, self._changed, True) + interp_bind(self._changed, True) self.clear_change() if self.size() == 0: - raise OperationError(self._space.w_RuntimeError, - self._space.wrap('ConsistencyFailure')) - -## def w__del__(self): -## self._space.bind(self._changed, self._space.newbool(False)) - + raise ConsistencyError, "tried to make a domain empty" def set_values(self, w_values): """Objects in the value set can't be unwrapped unless we Added: pypy/dist/pypy/objspace/cclp/constraint/test/problem.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/cclp/constraint/test/problem.py Wed Aug 16 11:29:21 2006 @@ -0,0 +1,44 @@ + + +def conference_scheduling(): + + dom_values = [(room,slot) + for room in ('room A','room B','room C') + for slot in ('day 1 AM','day 1 PM','day 2 AM', + 'day 2 PM')] + + variables = {} + for v in ('c01','c02','c03','c04','c05', 'c06','c07','c08','c09','c10'): + variables[v] = domain(dom_values, v) + + for conf in ('c03','c04','c05','c06'): + v = variables[conf] + tell(make_expression([v], "%s[0] == 'room C'" % conf)) + + for conf in ('c01','c05','c10'): + v = variables[conf] + tell(make_expression([v], "%s[1].startswith('day 1')" % conf)) + + for conf in ('c02','c03','c04','c09'): + v = variables[conf] + tell(make_expression([v], "%s[1].startswith('day 2')" % conf)) + + groups = (('c01','c02','c03','c10'), + ('c02','c06','c08','c09'), + ('c03','c05','c06','c07'), + ('c01','c03','c07','c08')) + + for group in groups: + for conf1 in group: + for conf2 in group: + if conf2 > conf1: + v1, v2 = variables[conf1], variables[conf2] + tell(make_expression([v1, v2], '%s[1] != %s[1]'% (conf1, conf2))) + + + tell(all_diff(variables.values())) + + distribute('dichotomy') + + return variables.values() + Modified: pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py Wed Aug 16 11:29:21 2006 @@ -18,31 +18,35 @@ assert cstr.knows_var(v2) def test_revise(self): - v1 = domain([1, 2], 'v1') - v2 = domain([1, 2], 'v2') - cstr = all_diff([v1, v2]) - assert cstr.revise() == False # not entailed - - v3 = domain([1], 'v3') - v4 = domain([2], 'v4') - cstr = all_diff([v3, v4]) - assert cstr.revise() == True # entailed - - v5 = domain([1], 'v5') - v6 = domain([1], 'v6') - cstr = all_diff([v5, v6]) - raises(Exception, cstr.revise) - - v7 = domain([1, 2], 'v7') - v8 = domain([1, 2], 'v8') - cstr = all_diff([v2, v7, v8]) - raises(Exception, cstr.revise) - - v9 = domain([1], 'v9') - v10= domain([1, 2], 'v10') - cstr = all_diff([v9, v10]) - assert cstr.revise() == True - assert domain_of(v10).get_values() == [2] + def in_space(): + v1 = domain([1, 2], 'v1') + v2 = domain([1, 2], 'v2') + cstr = all_diff([v1, v2]) + assert cstr.revise() == False # not entailed + + v3 = domain([1], 'v3') + v4 = domain([2], 'v4') + cstr = all_diff([v3, v4]) + assert cstr.revise() == True # entailed + + v5 = domain([1], 'v5') + v6 = domain([1], 'v6') + cstr = all_diff([v5, v6]) + raises(Exception, cstr.revise) + + v7 = domain([1, 2], 'v7') + v8 = domain([1, 2], 'v8') + cstr = all_diff([v2, v7, v8]) + raises(Exception, cstr.revise) + + v9 = domain([1], 'v9') + v10= domain([1, 2], 'v10') + cstr = all_diff([v9, v10]) + assert cstr.revise() == True + assert domain_of(v10).get_values() == [2] + + s = newspace(in_space) + class AppTest_Expression(object): Modified: pypy/dist/pypy/objspace/cclp/interp_var.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/interp_var.py (original) +++ pypy/dist/pypy/objspace/cclp/interp_var.py Wed Aug 16 11:29:21 2006 @@ -1,7 +1,11 @@ from pypy.objspace.cclp.variable import wait__Var, _assign_aliases, _entail -from pypy.objspace.cclp.types import W_Var +from pypy.objspace.cclp.types import W_Var, W_CVar +from pypy.objspace.cclp.global_state import scheduler +from pypy.objspace.cclp.misc import w +def interp_free(w_var): + return isinstance(w_var.w_bound_to, W_Var) def interp_wait(space, obj): return wait__Var(space, obj) @@ -9,24 +13,56 @@ class RebindingError(Exception): pass -def interp_bind(space, w_var, obj): - if isinstance(w_var.w_bound_to, W_Var): - return _assign_aliases(space, w_var, obj) +def interp_bind(w_var, obj): + if interp_free(w_var): + return interp_assign_aliases(w_var, obj) if w_var.w_bound_to == obj: return raise RebindingError -class EntailmentFailure(Exception): pass +class EntailmentError(Exception): pass -def interp_entail(space, w_v1, w_v2): +def interp_entail(w_v1, w_v2): w_v1val = w_v1.w_bound_to w_v2val = w_v2.w_bound_to - if not isinstance(w_v1val, W_Var): - if not isinstance(w_v2val, W_Var): + if not interp_free(w_v1): + if not interp_free(w_v2): # let's be simpler than unify if w_v1val != w_v2val: - raise EntailmentFailure - return _assign_aliases(space, w_v2, w_v1val) + raise EntailmentError + return interp_assign_aliases(w_v2, w_v1val) else: - return _entail(space, w_v1, w_v2) + w_v1.entails[w_v2] = True + +def interp_assign_aliases(w_var, w_val): + w(" :assign") + assert isinstance(w_var, W_Var) + w_curr = w_var + while 1: + w_next = w_curr.w_bound_to + _assign(w_curr, w_val) + # notify the blocked threads + scheduler[0].unblock_on(w_curr) + if w_next is w_var: + break + # switch to next + w_curr = w_next + _assign_entailed(w_var, w_val) + w(" :assigned") + +def _assign_entailed(w_var, w_val): + w(" :assign entailed") + for var in w_var.entails: + if interp_free(var): + interp_assign_aliases(var, w_val) + else: + if w_var.w_bound_to != w_val: + raise EntailmentError + +def _assign(w_var, w_val): + assert isinstance(w_var, W_Var) + if isinstance(w_var, W_CVar): + if not w_val in w_var.w_dom._values.content: + raise ValueError, "assignment out of domain" + w_var.w_bound_to = w_val Modified: pypy/dist/pypy/objspace/cclp/scheduler.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/scheduler.py (original) +++ pypy/dist/pypy/objspace/cclp/scheduler.py Wed Aug 16 11:29:21 2006 @@ -72,31 +72,6 @@ thread._prev = l thread._next = r - def remove_thread(self, thread): - assert isinstance(thread, ClonableCoroutine) - w(".. REMOVING", str(id(thread))) - assert thread not in self._blocked - try: - del self._traced[thread] - except KeyError: - w(".. removing non-traced thread") - l = thread._prev - r = thread._next - l._next = r - r._prev = l - self._head = r - if r == thread: #XXX write a test for me ! - if not we_are_translated(): - import traceback - traceback.print_exc() - self.display_head() - thread._next = thread._prev = None - # cspace/threads account mgmt - if thread._cspace is not self.space.w_None: - count = self.dec_live_thread_count(thread._cspace) - if count == 0: - del self._per_space_live_threads[thread._cspace] - #-- cspace helper def is_stable(self, cspace): @@ -122,6 +97,7 @@ self.schedule() #-- cspace -> thread_count helpers + def inc_live_thread_count(self, cspace): assert isinstance(cspace, W_CSpace) count = self._per_space_live_threads.get(cspace, 0) + 1 @@ -130,9 +106,9 @@ def dec_live_thread_count(self, cspace): assert isinstance(cspace, W_CSpace) - count = self._per_space_live_threads[cspace] -1 + count = self._per_space_live_threads[cspace] - 1 assert count >= 0 - self._per_space_live_threads[cspace] = count + self._per_space_live_threads[cspace] = count return count #-- / @@ -166,56 +142,59 @@ to_be_run = to_be_run._next assert isinstance(to_be_run, ClonableCoroutine) + # asking threads + if to_be_run in self._asking: + if self.is_stable(self._asking[to_be_run]): + del self._asking[to_be_run] + del self._blocked[to_be_run] + break if to_be_run == sentinel: if not dont_pass: return ClonableCoroutine.w_getcurrent(self.space) - self.display_head() + w(str(sched_all(self.space))) ## we RESET sched state so as to keep being usable beyond that self._init_head(self._main) self._init_blocked() w(".. SCHEDULER reinitialized") raise OperationError(self.space.w_AllBlockedError, self.space.wrap("can't schedule, possible deadlock in sight")) - # asking threads - if to_be_run in self._asking.keys(): - if self.is_stable(self._asking[to_be_run]): - del self._asking[to_be_run] - del self._blocked[to_be_run] - break self._head = to_be_run return to_be_run - #XXX call me directly for this to work translated - def __len__(self): - "count of known threads (including dead ones)" - curr = self._head - sentinel = curr - count = 1 # there is always a main thread - while curr._next != sentinel: - curr = curr._next - count += 1 - return count - - def display_head(self): - if we_are_translated(): - w("") - return - curr = self._head - v('Threads : [', '-'.join([str(id(curr)), str(curr in self._blocked)])) - while curr._next != self._head: - curr = curr._next - v('-'.join([str(id(curr)), str(curr in self._blocked)])) - w(']') - def add_new_thread(self, thread): "insert 'thread' at end of running queue" assert isinstance(thread, ClonableCoroutine) # cspace account mgmt - if thread._cspace != self.space.w_None: + if thread._cspace != None: self._per_space_live_threads.get(thread._cspace, 0) self.inc_live_thread_count(thread._cspace) self._chain_insert(thread) + def remove_thread(self, thread): + assert isinstance(thread, ClonableCoroutine) + w(".. REMOVING", str(id(thread))) + assert thread not in self._blocked + try: + del self._traced[thread] + except KeyError: + w(".. removing non-traced thread") + l = thread._prev + r = thread._next + l._next = r + r._prev = l + self._head = r + if r == thread: #XXX write a test for me ! + if not we_are_translated(): + import traceback + traceback.print_exc() + thread._next = thread._prev = None + # cspace/threads account mgmt + if thread._cspace is not None: + cspace = thread._cspace + live = self.dec_live_thread_count(cspace) + if live == 0: + del self._per_space_live_threads[cspace] + def add_to_blocked_on(self, w_var, thread): w(".. we BLOCK thread", str(id(thread)), "on var", str(w_var)) assert isinstance(w_var, W_Var) @@ -229,7 +208,7 @@ blocked.append(thread) self._blocked[thread] = True # cspace accounting - if thread._cspace is not self.space.w_None: + if thread._cspace is not None: self.dec_live_thread_count(thread._cspace) def unblock_on(self, w_var): @@ -243,7 +222,7 @@ for thr in blocked: del self._blocked[thr] # cspace accounting - if thr._cspace is not self.space.w_None: + if thr._cspace is not None: self.inc_live_thread_count(thr._cspace) def add_to_blocked_byneed(self, w_var, thread): @@ -258,7 +237,7 @@ blocked.append(thread) self._blocked[thread] = True # cspace accounting - if thread._cspace is not self.space.w_None: + if thread._cspace is not None: self.dec_live_thread_count(thread._cspace) def unblock_byneed_on(self, w_var): @@ -274,7 +253,7 @@ for thr in blocked: del self._blocked[thr] # cspace accounting - if thr._cspace is not self.space.w_None: + if thr._cspace is not None: self.inc_live_thread_count(thr._cspace) Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Wed Aug 16 11:29:21 2006 @@ -8,7 +8,8 @@ from pypy.objspace.cclp.thunk import CSpaceThunk, PropagatorThunk from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.variable import newvar - +from pypy.objspace.cclp.types import ConsistencyError, Solution, W_Var +from pypy.objspace.cclp.interp_var import interp_bind, interp_free def newspace(space, w_callable, __args__): args = __args__.normalize() @@ -34,9 +35,13 @@ assert isinstance(w_n, W_IntObject) n = space.int_w(w_n) cspace = ClonableCoroutine.w_getcurrent(space)._cspace - if cspace != space.w_None: + if cspace != None: assert isinstance(cspace, W_CSpace) - return cspace.choose(n) + try: + return space.newint(cspace.choose(w_n.intval)) + except ConsistencyError: + raise OperationError(space.w_ConsistecyError, + space.wrap("the space is failed")) raise OperationError(space.w_RuntimeError, space.wrap("choose is forbidden from the top-level space")) app_choose = gateway.interp2app(choose) @@ -54,55 +59,69 @@ def __init__(self, space, thread, parent): assert isinstance(thread, ClonableCoroutine) - assert (parent is space.w_None) or isinstance(parent, W_CSpace) + assert (parent is None) or isinstance(parent, W_CSpace) self.space = space # the object space ;-) self.parent = parent self.main_thread = thread # choice mgmt self._choice = newvar(space) self._committed = newvar(space) - # merging + # status, merging self._solution = newvar(space) - self._merged = newvar(space) + self._finished = newvar(space) + self._failed = False + # constraint store ... + self._store = {} # name -> var + def register_var(self, cvar): + self._store[cvar.name] = cvar def w_ask(self): scheduler[0].wait_stable(self) self.space.wait(self._choice) - return self._choice + choice = self._choice.w_bound_to + self._choice = newvar(self.space) + self._last_choice = choice.intval + return choice def choose(self, n): assert n > 1 scheduler[0].wait_stable(self) - assert self.space.is_true(self.space.is_free(self._choice)) - assert self.space.is_true(self.space.is_free(self._committed)) - self.space.bind(self._choice, self.space.wrap(n)) + if self._failed: #XXX set by any propagator + raise ConsistencyError + assert interp_free(self._choice) + assert interp_free(self._committed) + interp_bind(self._choice, self.space.wrap(n)) self.space.wait(self._committed) - committed = self._committed + committed = self._committed.w_bound_to self._committed = newvar(self.space) return committed def w_commit(self, w_n): - assert self.space.is_true(self.space.is_bound(self._choice)) - assert 0 < self.space.int_w(w_n) - assert self.space.int_w(w_n) <= self._choice.w_bound_to - self.space.bind(self._committed, w_n) - self._choice = newvar(self.space) - + assert isinstance(w_n, W_IntObject) + n = w_n.intval + assert interp_free(self._committed) + assert n > 0 + assert n <= self._last_choice + interp_bind(self._committed, n) def tell(self, w_constraint): space = self.space w_coro = ClonableCoroutine(space) - thunk = PropagatorThunk(space, w_constraint, w_coro, self._merged) + w_coro._cspace = self + thunk = PropagatorThunk(space, w_constraint, w_coro) w_coro.bind(thunk) if not we_are_translated(): - w("PROPAGATOR, thread", str(id(w_coro))) - w_coro._cspace = self + w("PROPAGATOR in thread", str(id(w_coro))) scheduler[0].add_new_thread(w_coro) scheduler[0].schedule() + def fail(self): + self._failed = True + self._store = {} + def w_merge(self): - self.space.bind(self._merged, self.space.w_True) + self._store = {} return self._solution Modified: pypy/dist/pypy/objspace/cclp/thread.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thread.py (original) +++ pypy/dist/pypy/objspace/cclp/thread.py Wed Aug 16 11:29:21 2006 @@ -21,6 +21,7 @@ w_Future = W_Future(space) thunk = FutureThunk(space, w_callable, args, w_Future, coro) coro.bind(thunk) + coro._cspace = ClonableCoroutine.w_getcurrent(space)._cspace if not we_are_translated(): w("FUTURE", str(id(coro)), "for", str(w_callable.name)) scheduler[0].add_new_thread(coro) @@ -40,6 +41,7 @@ #coro.cspace = ClonableCoroutine.w_getcurrent(space).cspace thunk = ProcedureThunk(space, w_callable, args, coro) coro.bind(thunk) + coro._cspace = ClonableCoroutine.w_getcurrent(space)._cspace if not we_are_translated(): w("STACKLET", str(id(coro)), "for", str(w_callable.name)) scheduler[0].add_new_thread(coro) @@ -53,5 +55,3 @@ def this_thread(space): return ClonableCoroutine.w_getcurrent(space) app_this_thread = gateway.interp2app(this_thread) - - Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 11:29:21 2006 @@ -3,8 +3,8 @@ from pypy.objspace.cclp.misc import w from pypy.objspace.cclp.global_state import scheduler -from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue -from pypy.objspace.cclp.interp_var import interp_wait, interp_entail, interp_bind +from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue, ConsistencyError, Solution +from pypy.objspace.cclp.interp_var import interp_wait, interp_entail, interp_bind, interp_free def logic_args(args): @@ -70,65 +70,96 @@ scheduler[0].remove_thread(self._coro) scheduler[0].schedule() -SPACE_FAILURE = 0 -SPACE_SOLUTION = 1 - class CSpaceThunk(_AppThunk): def __init__(self, space, w_callable, args, coro): _AppThunk.__init__(self, space, coro.costate, w_callable, args) self._coro = coro def call(self): - w(". initial (returnless) thunk CALL in", str(id(self._coro))) + w("-- initial thunk CALL in", str(id(self._coro))) scheduler[0].trace_vars(self._coro, logic_args(self.args.unpack())) cspace = self._coro._cspace try: try: _AppThunk.call(self) except Exception, exc: - w(".% exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + w("-- exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + import traceback + traceback.print_exc() scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) else: - w(".% clean (valueless) EXIT of", str(id(self._coro))) + w("-- clean (valueless) EXIT of", str(id(self._coro))) self.space.bind(cspace._solution, self.costate.w_tempval) - self.space.bind(cspace._choice, self.space.wrap(SPACE_SOLUTION)) finally: scheduler[0].remove_thread(self._coro) scheduler[0].schedule() class PropagatorThunk(AbstractThunk): - def __init__(self, space, w_constraint, coro, Merged): + def __init__(self, space, w_constraint, coro): self.space = space self.coro = coro self.const = w_constraint - self.Merged = Merged def call(self): try: + cspace = self.coro._cspace try: while 1: entailed = self.const.revise() if entailed: break Obs = W_Var(self.space) - interp_entail(self.space, self.Merged, Obs) + interp_entail(cspace._finished, Obs) for Sync in [var.w_dom.give_synchronizer() for var in self.const._variables]: - interp_entail(self.space, Sync, Obs) + interp_entail(Sync, Obs) interp_wait(self.space, Obs) + if not interp_free(cspace._finished): + break + except ConsistencyError: + self.coro._cspace.fail() except: import traceback traceback.print_exc() finally: - # all values of dom size 1 are bound - for var in self.const._variables: - if var.w_dom.size() == 1: - interp_bind(self.space, var, var.w_dom.get_values()[0]) self.coro._dead = True scheduler[0].remove_thread(self.coro) scheduler[0].schedule() +class DistributorThunk(AbstractThunk): + def __init__(self, space, w_distributor, coro): + self.space = space + self.coro = coro + self.dist = w_distributor + + def call(self): + coro = self.coro + try: + cspace = coro._cspace + dist = self.dist + try: + while 1: + choice = cspace.choose(dist.fanout()) + dist.distribute(choice) + except Solution: + w("-- DISTRIBUTOR thunk exited because a solution was found") + for var in cspace._solution.w_bound_to.wrappeditems: + assert var.w_dom.size() == 1 + interp_bind(var, var.w_dom.get_values()[0]) + interp_bind(cspace._choice, self.space.newint(1)) + except ConsistencyError, e: + w("-- DISTRIBUTOR thunk exited because", str(e)) + interp_bind(cspace._choice, self.space.newint(0)) + except: + import traceback + traceback.print_exc() + finally: + interp_bind(cspace._finished, True) + coro._dead = True + scheduler[0].remove_thread(coro) + scheduler[0].schedule() + Modified: pypy/dist/pypy/objspace/cclp/types.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/types.py (original) +++ pypy/dist/pypy/objspace/cclp/types.py Wed Aug 16 11:29:21 2006 @@ -36,18 +36,23 @@ class W_CVar(W_Var): - def __init__(w_self, space, w_dom, w_name): + def __init__(self, space, w_dom, w_name): assert isinstance(w_dom, W_AbstractDomain) - W_Var.__init__(w_self, space) - w_self.w_dom = w_dom - w_self.name = space.str_w(w_name) - w_self.w_nam = w_name + W_Var.__init__(self, space) + self.w_dom = w_dom + self.name = space.str_w(w_name) + self.w_nam = w_name + cspace = ClonableCoroutine.w_getcurrent(space)._cspace + if cspace is None: + w("-- WARNING : you are instanciating a constraint var in the top-level space") + else: + cspace.register_var(self) - def name_w(w_self): - return w_self.name + def name_w(self): + return self.name - def w_name(w_self): - return w_self.w_nam + def w_name(self): + return self.w_nam def domain_of(space, w_v): assert isinstance(w_v, W_CVar) @@ -63,6 +68,10 @@ def __init__(w_self, exc): w_self.exc = exc +class ConsistencyError(Exception): pass + +class Solution(Exception): pass + #-- Constraint --------------------------------------------- class W_Constraint(baseobjspace.Wrappable): @@ -82,6 +91,15 @@ W_AbstractDomain.typedef = typedef.TypeDef("W_AbstractDomain") +class W_AbstractDistributor(baseobjspace.Wrappable): + + def __init__(self, space, fanout): + assert isinstance(fanout, int) + self._space = space + self._fanout = fanout + self._cspace = ClonableCoroutine.w_getcurrent(space)._cspace + +W_AbstractDistributor.typedef = typedef.TypeDef("W_AbstractDistributor") #-- Misc --------------------------------------------------- Modified: pypy/dist/pypy/objspace/logic.py ============================================================================== --- pypy/dist/pypy/objspace/logic.py (original) +++ pypy/dist/pypy/objspace/logic.py Wed Aug 16 11:29:21 2006 @@ -50,13 +50,7 @@ all_mms.update(constraint.all_mms) ## #----- distributors --------------- -## from pypy.objspace.constraint import distributor - - -#-- THE SPACE --------------------------------------- - -#class UnificationError(w_RuntimeError): -# pass +from pypy.objspace.cclp.constraint import distributor #-- SPACE HELPERS ------------------------------------- @@ -243,13 +237,9 @@ space.wrap(constraint.app_make_expression)) space.setitem(space.builtin.w_dict, space.wrap('all_diff'), space.wrap(constraint.app_make_alldistinct)) -## #-- distributor -- -## space.setitem(space.builtin.w_dict, space.wrap('NaiveDistributor'), -## space.wrap(distributor.app_make_naive_distributor)) -## space.setitem(space.builtin.w_dict, space.wrap('SplitDistributor'), -## space.wrap(distributor.app_make_split_distributor)) -## space.setitem(space.builtin.w_dict, space.wrap('DichotomyDistributor'), -## space.wrap(distributor.app_make_dichotomy_distributor)) + #-- distributor -- + space.setitem(space.builtin.w_dict, space.wrap('distribute'), + space.wrap(distributor.app_distribute)) #-- threading -- space.setitem(space.builtin.w_dict, space.wrap('future'), space.wrap(app_future)) @@ -286,8 +276,8 @@ #-- path to the applevel modules -- import pypy.objspace.constraint import os - dir = os.path.dirname(pypy.objspace.constraint.__file__) - dir = os.path.join(dir, 'applevel') + dir = os.path.dirname(pypy.objspace.cclp.constraint.__file__) + dir = os.path.join(dir, 'test') space.call_method(space.sys.get('path'), 'append', space.wrap(dir)) # make sure that _stackless is imported Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 16 11:29:21 2006 @@ -1,6 +1,7 @@ from pypy.conftest import gettestobjspace from py.test import skip + class AppTest_Logic(object): def setup_class(cls): @@ -637,25 +638,6 @@ schedule() reset_scheduler() # free all the hanging threads - - def test_newspace_ask_noop(self): - - def in_space(X): return X + 42 - - def asker(): - ask() - - X = newvar() - s = newspace(in_space, X) - - assert sched_all()['space_accounting'][0][1] == 0 # live threads - assert len(sched_all()['blocked_on']) == 1 - - stacklet(asker) - - unify(X, 42) - schedule() - assert len(sched_all()['threads']) == 1 class AppTest_CompSpace(object): @@ -665,48 +647,55 @@ def test_cvar(self): - d = domain([1, 2, 4], '') - - raises(UnificationError, bind, d, 42) - bind(d, 2) - assert d == 2 + def in_space(X): + d = domain([1, 2, 4], '') - class Foo(object): - pass - - f = Foo() - d = domain([Foo(), f, Foo()], '') - raises(UnificationError, bind, d, Foo()) - bind(d, f) - assert d == f - - d1 = domain([1, 2, 3], '') - d2 = domain([2, 3, 4], '') - d3 = domain([5, 6], '') - raises(UnificationError, unify, d1, d3) - unify(d1, d2) - assert alias_of(d1, d2) - assert domain_of(d1) == domain_of(d2) == FiniteDomain([2, 3]) - - d1 = domain([1, 2, 3], '') - d4 = domain([3, 4], '') - unify(d1, d4) - assert d1 == d4 == 3 + raises(UnificationError, bind, d, 42) + bind(d, 2) + assert d == 2 + + class Foo(object): + pass + + f = Foo() + d = domain([Foo(), f, Foo()], '') + raises(UnificationError, bind, d, Foo()) + bind(d, f) + assert d == f + + d1 = domain([1, 2, 3], '') + d2 = domain([2, 3, 4], '') + d3 = domain([5, 6], '') + raises(UnificationError, unify, d1, d3) + unify(d1, d2) + assert alias_of(d1, d2) + assert domain_of(d1) == domain_of(d2) == FiniteDomain([2, 3]) + + d1 = domain([1, 2, 3], '') + d4 = domain([3, 4], '') + unify(d1, d4) + assert d1 == d4 == 3 + + d1 = domain([1, 2], '') + x = newvar() + unify(d1, x) + assert alias_of(x, d1) + raises(UnificationError, unify, x, 42) + + d1 = domain([1, 2], '') + x = newvar() + unify(d1, x) + assert alias_of(x, d1) + unify(x, 2) + assert d1 == x == 2 + #XXX and a bunch of app-level functions + #raises(TypeError, domain_of, x) - d1 = domain([1, 2], '') - x = newvar() - unify(d1, x) - assert alias_of(x, d1) - raises(UnificationError, unify, x, 42) + bind(X, True) - d1 = domain([1, 2], '') - x = newvar() - unify(d1, x) - assert alias_of(x, d1) - unify(x, 2) - assert d1 == x == 2 - #XXX and a bunch of app-level functions - #raises(TypeError, domain_of, x) + X = newvar() + newspace(in_space, X) + wait(X) def test_newspace_ask_wait(self): @@ -739,13 +728,12 @@ cspace.commit(2) X = newvar() - s = newspace(chooser, X) stacklet(asker, s) schedule() + wait(X) assert X == 2 - def test_more_ask_choose(self): def chooser(vec, X): @@ -757,9 +745,9 @@ def asker(cspace): while 1: choices = cspace.ask() - if choices == 1: # success ! - break cspace.commit(choices) + if choices == 8: # success ! + break # choices >= 1 v = range(2, 9) @@ -774,28 +762,34 @@ assert X == 'done' schedule() - assert len(sched_all()['threads']) == 1 + #XXX + #assert len(sched_all()['threads']) == 1 - def test_tell(self): - def problem(): - X, Y = domain([1, 2], 'X'), domain([1, 2, 3], 'Y') - tell(make_expression([X, Y], 'X + Y > 4')) - return (X, Y) + def test_tell_ask_choose_commit(self): + from problem import conference_scheduling - def solve(spc, X): + def solve(spc, Sol): while 1: status = spc.ask() - if status == 1: + if status > 1: + spc.commit(1) + elif status in (0, 1): break - unify(spc.merge(), X) + if status: + unify(Sol, spc.merge()) + else: + unify(Sol, False) - s = newspace(problem) + s = newspace(conference_scheduling) Solution = newvar() stacklet(solve, s, Solution) - schedule() - - assert Solution == (2, 3) + assert Solution == [('room B', 'day 1 PM'), ('room A', 'day 1 PM'), + ('room B', 'day 2 AM'), ('room B', 'day 1 AM'), + ('room A', 'day 2 PM'), ('room C', 'day 2 AM'), + ('room C', 'day 2 PM'), ('room C', 'day 1 PM'), + ('room C', 'day 1 AM'), ('room B', 'day 2 PM')] - assert len(sched_all()['threads']) == 1 + #XXX + #assert len(sched_all()['threads']) == 1 From mwh at codespeak.net Wed Aug 16 11:38:54 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 16 Aug 2006 11:38:54 +0200 (CEST) Subject: [pypy-svn] r31335 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060816093854.013E210083@code0.codespeak.net> Author: mwh Date: Wed Aug 16 11:38:51 2006 New Revision: 31335 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: just allocate one EmptyDictImplementation per space. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 11:38:51 2006 @@ -158,7 +158,7 @@ if self.content: return self else: - return EmptyDictImplementation(self.space) + return self.space.emptydictimpl def length(self): return len(self.content) @@ -348,7 +348,7 @@ if space.config.objspace.std.withdictmeasurement: w_self.implementation = MeasuringDictImplementation(space) else: - w_self.implementation = EmptyDictImplementation(space) + w_self.implementation = space.emptydictimpl def initialize_content(w_self, list_pairs_w): impl = w_self.implementation @@ -387,7 +387,7 @@ w_src, w_kwds = __args__.parse('dict', (['seq_or_map'], None, 'kwargs'), # signature [W_DictMultiObject(space)]) # default argument - # w_dict.implementation = EmptyDictImplementation(self.space) + # w_dict.implementation = space.emptydictimpl # ^^^ disabled only for CPython compatibility try: space.getattr(w_src, space.wrap("keys")) @@ -514,7 +514,7 @@ return W_DictMultiIter_Values(space, w_self.implementation) def dict_clear__DictMulti(space, w_self): - w_self.implementation = EmptyDictImplementation(space) + w_self.implementation = space.emptydictimpl def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): return w_dict.get(w_lookup, w_default) Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Wed Aug 16 11:38:51 2006 @@ -62,6 +62,7 @@ elif self.config.objspace.std.withmultidict: from pypy.objspace.std import dictmultiobject self.DictObjectCls = dictmultiobject.W_DictMultiObject + self.emptydictimpl = dictmultiobject.EmptyDictImplementation(self) else: from pypy.objspace.std import dictobject self.DictObjectCls = dictobject.W_DictObject Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Wed Aug 16 11:38:51 2006 @@ -1,5 +1,7 @@ import autopath -from pypy.objspace.std.dictmultiobject import W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY +from pypy.objspace.std.dictmultiobject import \ + W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ + EmptyDictImplementation from pypy.conftest import gettestobjspace from pypy.objspace.std.test import test_dictobject @@ -14,6 +16,7 @@ class TestDictImplementation: def setup_method(self,method): self.space = test_dictobject.FakeSpace() + self.space.emptydictimpl = EmptyDictImplementation(self.space) self.space.DictObjectCls = W_DictMultiObject def test_stressdict(self): From rhymes at codespeak.net Wed Aug 16 11:52:53 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Wed, 16 Aug 2006 11:52:53 +0200 (CEST) Subject: [pypy-svn] r31336 - in pypy/dist/pypy/module/rctime: . test Message-ID: <20060816095253.962CE1007F@code0.codespeak.net> Author: rhymes Date: Wed Aug 16 11:52:49 2006 New Revision: 31336 Modified: pypy/dist/pypy/module/rctime/app_time.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: beautify strptime() and add the test for it, thanks arre Modified: pypy/dist/pypy/module/rctime/app_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/app_time.py (original) +++ pypy/dist/pypy/module/rctime/app_time.py Wed Aug 16 11:52:49 2006 @@ -75,9 +75,15 @@ _float_sleep(secs) -def strptime(*args, **kw): +def strptime(string, format="%a %b %d %H:%M:%S %Y"): + """strptime(string, format) -> struct_time + + Parse a string to a time tuple according to a format specification. + See the library reference manual for formatting codes + (same as strftime()).""" + import _strptime - return _strptime.strptime(*args, **kw) + return _strptime.strptime(string, format) __doc__ = """This module provides various functions to manipulate time values. Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Wed Aug 16 11:52:49 2006 @@ -247,7 +247,6 @@ raises(ValueError, rctime.strftime, '', (1900, 1, 1, 0, 0, 0, 0, 1, 2)) def test_strptime(self): - skip("strptime is not implemented right now") import rctime t = rctime.time() @@ -256,7 +255,4 @@ 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = ' %' + directive - # try: rctime.strptime(rctime.strftime(format, tt), format) - # except ValueError: - # raise ValueError, "conversion specifier: %r failed.' % format" From auc at codespeak.net Wed Aug 16 11:53:36 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 11:53:36 +0200 (CEST) Subject: [pypy-svn] r31337 - pypy/dist/pypy/objspace/cclp/constraint Message-ID: <20060816095336.2757510080@code0.codespeak.net> Author: auc Date: Wed Aug 16 11:53:33 2006 New Revision: 31337 Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/distributor.py Log: rm dead code Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Wed Aug 16 11:53:33 2006 @@ -88,15 +88,6 @@ # self.filter_func is a function taking keyword arguments and returning a boolean self.filter_func = self._space.make_filter(w_variables, w_formula) - def test_solution(self, sol_dict): - """test a solution against this constraint - accept a mapping of variable names to value""" - args = [] - for var in self._variables: - assert isinstance(var, W_Variable) - args.append(sol_dict[var.w_name()]) - return self.filter_func(*args) - def _init_result_cache(self): """key = (variable,value), value = [has_success,has_failure]""" result_cache = self._space.newdict() @@ -216,11 +207,6 @@ variables = ord_vars.values() -## variables = [(_spc.int_w(w_cs.w_dom(variable).w_size()), -## variable, w_cs.w_dom(variable)) -## for variable in self._variables] -## variables.sort() - # if a domain has a size of 1, # then the value must be removed from the other domains for var, dom in variables: Modified: pypy/dist/pypy/objspace/cclp/constraint/distributor.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/distributor.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/distributor.py Wed Aug 16 11:53:33 2006 @@ -40,14 +40,6 @@ app_distribute = interp2app(distribute) -def arrange_domains(cs, variables): - """build a data structure from var to dom - that satisfies distribute & friends""" - new_doms = {} - for var in variables: - new_doms[var] = cs.dom(var).copy() - return new_doms - class W_Distributor(W_AbstractDistributor): """_distribute is left unimplemented.""" From arigo at codespeak.net Wed Aug 16 12:16:04 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 16 Aug 2006 12:16:04 +0200 (CEST) Subject: [pypy-svn] r31338 - in pypy/dist/pypy/objspace/cclp: . constraint constraint/test Message-ID: <20060816101604.94BFC10082@code0.codespeak.net> Author: arigo Date: Wed Aug 16 12:16:02 2006 New Revision: 31338 Modified: pypy/dist/pypy/objspace/cclp/constraint/ (props changed) pypy/dist/pypy/objspace/cclp/constraint/__init__.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/constraint.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/distributor.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/domain.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/test/ (props changed) pypy/dist/pypy/objspace/cclp/constraint/test/problem.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py (props changed) pypy/dist/pypy/objspace/cclp/constraint/variable.py (props changed) pypy/dist/pypy/objspace/cclp/interp_var.py (props changed) Log: fixeol From arigo at codespeak.net Wed Aug 16 12:51:03 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 16 Aug 2006 12:51:03 +0200 (CEST) Subject: [pypy-svn] r31339 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20060816105103.6103110082@code0.codespeak.net> Author: arigo Date: Wed Aug 16 12:51:01 2006 New Revision: 31339 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rpython/rtuple.py pypy/dist/pypy/rpython/test/test_rtuple.py Log: RPython support for slicing tuples (constant indices only). Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Wed Aug 16 12:51:01 2006 @@ -456,6 +456,15 @@ return unionof(*tup1.items) getitem.can_only_throw = [IndexError] +class __extend__(pairtype(SomeTuple, SomeSlice)): + + def getitem((tup, slic)): + if not slic.is_immutable_constant(): + raise Exception("not supported: " + "tuple slicing with non-constant indexes") + return SomeTuple(tup.items[slic.const]) + getitem.can_only_throw = [] + class __extend__(pairtype(SomeList, SomeInteger)): Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Wed Aug 16 12:51:01 2006 @@ -231,6 +231,10 @@ self.start = start self.stop = stop self.step = step + if (start.is_immutable_constant() and + stop .is_immutable_constant() and + step .is_immutable_constant()): + self.const = slice(start.const, stop.const, step.const) def can_be_none(self): return False Modified: pypy/dist/pypy/rpython/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/rtuple.py (original) +++ pypy/dist/pypy/rpython/rtuple.py Wed Aug 16 12:51:01 2006 @@ -6,6 +6,7 @@ from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst from pypy.rpython.rmodel import IteratorRepr from pypy.rpython.rmodel import externalvsinternal +from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem.lltype import Void, Signed from pypy.rpython.rarithmetic import intmask @@ -153,6 +154,21 @@ index = v_index.value return r_tup.getitem(hop.llops, v_tuple, index) +class __extend__(pairtype(AbstractTupleRepr, AbstractSliceRepr)): + + def rtype_getitem((r_tup, r_slice), hop): + s_slice = hop.args_s[1] + if not s_slice.is_immutable_constant(): + raise TyperError("non-constant tuple slicing index") + v_tup = hop.inputarg(r_tup, arg=0) + + indices = range(len(r_tup.items_r))[s_slice.const] + assert len(indices) == len(hop.r_result.items_r) + + items_v = [r_tup.getitem_internal(hop.llops, v_tup, i) + for i in indices] + return hop.r_result.newtuple(hop.llops, hop.r_result, items_v) + class __extend__(pairtype(AbstractTupleRepr, Repr)): def rtype_contains((r_tup, r_item), hop): s_tup = hop.args_s[0] Modified: pypy/dist/pypy/rpython/test/test_rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rtuple.py (original) +++ pypy/dist/pypy/rpython/test/test_rtuple.py Wed Aug 16 12:51:01 2006 @@ -249,6 +249,21 @@ assert res[0] == res[1] == res[2] == [] self.interpret(f, []) + def test_slice(self): + def g(n): + t = (1.5, "hello", n) + return t[1:] + t[:-1] + t[12:] + t[0:2] + def f(n): + res = g(n) + assert len(res) == 6 + assert res[0] == "hello" + assert res[1] == n + assert res[2] == 1.5 + assert res[3] == "hello" + assert res[4] == 1.5 + assert res[5] == "hello" + self.interpret(f, [9]) + class TestLLtype(BaseTestRtuple, LLRtypeMixin): pass From benyoung at codespeak.net Wed Aug 16 13:24:35 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Wed, 16 Aug 2006 13:24:35 +0200 (CEST) Subject: [pypy-svn] r31340 - pypy/dist/pypy/objspace/std Message-ID: <20060816112435.9775010080@code0.codespeak.net> Author: benyoung Date: Wed Aug 16 13:24:24 2006 New Revision: 31340 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: First cut of moving string dicts into the multidict style Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 13:24:24 2006 @@ -179,6 +179,82 @@ def items(self): return self.content.items() +class StrDictImplementation(DictImplementation): + def __init__(self, space): + self.space = space + self.content = {} + + def setitem(self, w_key, w_value): + space = self.space + if space.is_w(space.type(w_key), space.w_str): + self.content[space.str_w(w_key)] = w_value + return self + else: + return self._as_rdict().setitem(w_key, w_value) + + def delitem(self, w_key): + space = self.space + if space.is_w(space.type(w_key), space.w_str): + del self.content[space.str_w(w_key)] + if self.content: + return self + else: + return space.emptydictimpl + elif self._is_sane_hash(w_lookup_type): + raise KeyError + else: + return self._as_rdict().delitem(w_key) + + def length(self): + return len(self.content) + + def get(self, w_lookup): + space = self.space + w_lookup_type = space.type(w_lookup) + if space.is_w(w_lookup_type, space.w_str): + return self.content.get(space.str_w(w_lookup), None) + elif self._is_sane_hash(w_lookup_type): + return None + else: + return self._as_rdict().get(w_lookup) + + def iteritems(self): + return self._as_rdict().iteritems() + + def iterkeys(self): + return self._as_rdict().iterkeys() + + def itervalues(self): + return self._as_rdict().itervalues() + + def keys(self): + space = self.space + return [space.wrap(key) for key in self.content.iterkeys()] + + def values(self): + return self.content.values() + + def items(self): + space = self.space + return [(space.wrap(key), w_value) for (key, w_value) in self.content.iterkeys()] + + def _is_sane_hash(w_self, w_lookup_type): + """ Handles the case of a non string key lookup. + Types that have a sane hash/eq function should allow us to return True + directly to signal that the key is not in the dict in any case. + XXX The types should provide such a flag. """ + + space = self.space + # XXX there are much more types + return space.is_w(w_lookup_type, space.w_NoneType) or space.is_w(w_lookup_type, space.w_int) + + def _as_rdict(self): + newimpl = RDictImplementation(self.space) + for k, w_v in w_self.content.items(): + newimpl.setitem[w_self.space.wrap(k)] = w_v + return newimpl + + import time, py class DictInfo(object): From benyoung at codespeak.net Wed Aug 16 13:27:53 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Wed, 16 Aug 2006 13:27:53 +0200 (CEST) Subject: [pypy-svn] r31341 - pypy/dist/pypy/objspace/std Message-ID: <20060816112753.CA19010083@code0.codespeak.net> Author: benyoung Date: Wed Aug 16 13:27:37 2006 New Revision: 31341 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: Immediate fix :( Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 13:27:37 2006 @@ -250,7 +250,7 @@ def _as_rdict(self): newimpl = RDictImplementation(self.space) - for k, w_v in w_self.content.items(): + for k, w_v in self.content.items(): newimpl.setitem[w_self.space.wrap(k)] = w_v return newimpl From benyoung at codespeak.net Wed Aug 16 13:36:55 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Wed, 16 Aug 2006 13:36:55 +0200 (CEST) Subject: [pypy-svn] r31342 - pypy/dist/pypy/objspace/std Message-ID: <20060816113655.322F310083@code0.codespeak.net> Author: benyoung Date: Wed Aug 16 13:36:45 2006 New Revision: 31342 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py Log: More fixes Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 13:36:45 2006 @@ -238,7 +238,7 @@ space = self.space return [(space.wrap(key), w_value) for (key, w_value) in self.content.iterkeys()] - def _is_sane_hash(w_self, w_lookup_type): + def _is_sane_hash(self, w_lookup_type): """ Handles the case of a non string key lookup. Types that have a sane hash/eq function should allow us to return True directly to signal that the key is not in the dict in any case. @@ -251,7 +251,7 @@ def _as_rdict(self): newimpl = RDictImplementation(self.space) for k, w_v in self.content.items(): - newimpl.setitem[w_self.space.wrap(k)] = w_v + newimpl.setitem(self.space.wrap(k), w_v) return newimpl From arigo at codespeak.net Wed Aug 16 13:46:26 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 16 Aug 2006 13:46:26 +0200 (CEST) Subject: [pypy-svn] r31343 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20060816114626.311B610080@code0.codespeak.net> Author: arigo Date: Wed Aug 16 13:46:24 2006 New Revision: 31343 Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py Log: Temporary fix: the _ssl module produces GcStructs whose repr is so incredibly large it doesn't fit in standard RAM sizes. Clearly we need to fix the reprs, but for now let's avoid computing the repr just to put it into an exception message (which is caught and ignored anyway). Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Aug 16 13:46:24 2006 @@ -1562,7 +1562,8 @@ if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT if GCSTRUCT._runtime_type_info is None: - raise ValueError, "no attached runtime type info for %s" % GCSTRUCT + raise ValueError, ("no attached runtime type info for GcStruct %s" % + GCSTRUCT._name) return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) def runtime_type_info(p): From arigo at codespeak.net Wed Aug 16 13:59:28 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 16 Aug 2006 13:59:28 +0200 (CEST) Subject: [pypy-svn] r31344 - in pypy/dist/pypy: annotation rpython Message-ID: <20060816115928.314BA1007F@code0.codespeak.net> Author: arigo Date: Wed Aug 16 13:59:25 2006 New Revision: 31344 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/rpython/rtuple.py Log: Oups! I broke translation again with this tuple slicing (more precisely, list slicing was broken). Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Wed Aug 16 13:59:25 2006 @@ -459,10 +459,8 @@ class __extend__(pairtype(SomeTuple, SomeSlice)): def getitem((tup, slic)): - if not slic.is_immutable_constant(): - raise Exception("not supported: " - "tuple slicing with non-constant indexes") - return SomeTuple(tup.items[slic.const]) + start, stop, step = slic.constant_indices() + return SomeTuple(tup.items[start:stop:step]) getitem.can_only_throw = [] Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Wed Aug 16 13:59:25 2006 @@ -231,10 +231,14 @@ self.start = start self.stop = stop self.step = step - if (start.is_immutable_constant() and - stop .is_immutable_constant() and - step .is_immutable_constant()): - self.const = slice(start.const, stop.const, step.const) + + def constant_indices(self): + if (self.start.is_immutable_constant() and + self.stop .is_immutable_constant() and + self.step .is_immutable_constant()): + return self.start.const, self.stop.const, self.step.const + else: + raise Exception("need constant indices for this slice") def can_be_none(self): return False Modified: pypy/dist/pypy/rpython/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/rtuple.py (original) +++ pypy/dist/pypy/rpython/rtuple.py Wed Aug 16 13:59:25 2006 @@ -157,12 +157,10 @@ class __extend__(pairtype(AbstractTupleRepr, AbstractSliceRepr)): def rtype_getitem((r_tup, r_slice), hop): - s_slice = hop.args_s[1] - if not s_slice.is_immutable_constant(): - raise TyperError("non-constant tuple slicing index") v_tup = hop.inputarg(r_tup, arg=0) - - indices = range(len(r_tup.items_r))[s_slice.const] + s_slice = hop.args_s[1] + start, stop, step = s_slice.constant_indices() + indices = range(len(r_tup.items_r))[start:stop:step] assert len(indices) == len(hop.r_result.items_r) items_v = [r_tup.getitem_internal(hop.llops, v_tup, i) From auc at codespeak.net Wed Aug 16 14:09:58 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 14:09:58 +0200 (CEST) Subject: [pypy-svn] r31345 - in pypy/dist/pypy/objspace/cclp: . constraint Message-ID: <20060816120958.E1BCE1007F@code0.codespeak.net> Author: auc Date: Wed Aug 16 14:09:53 2006 New Revision: 31345 Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py pypy/dist/pypy/objspace/cclp/constraint/distributor.py pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/thunk.py Log: proper distributability check Modified: pypy/dist/pypy/objspace/cclp/constraint/constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/constraint.py Wed Aug 16 14:09:53 2006 @@ -39,10 +39,6 @@ def affected_variables(self): return self._variables - def w_knows_var(self, w_variable): - assert isinstance(w_variable, W_Variable) - return self._space.newbool(w_variable in self._variables) - def w_revise(self): return self._space.newbool(self.revise()) @@ -51,7 +47,6 @@ "W_AbstractConstraint", W_Constraint.typedef, affected_variables = interp2app(W_AbstractConstraint.w_affected_variables), - knows_var = interp2app(W_AbstractConstraint.w_knows_var), revise = interp2app(W_AbstractConstraint.w_revise)) @@ -130,6 +125,7 @@ else: # it's over raise StopIteration + #XXX this smells return kwargs def revise(self): Modified: pypy/dist/pypy/objspace/cclp/constraint/distributor.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/distributor.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/distributor.py Wed Aug 16 14:09:53 2006 @@ -48,21 +48,25 @@ def fanout(self): return self._fanout - + def _find_smallest_domain(self): """returns the variable having the smallest domain. (or one of such variables if there is a tie) """ vars_ = [var for var in self._cspace._store.values() if var.w_dom.size() > 1] - if len(vars_) == 0: - raise Solution best = vars_[0] for var in vars_: if var.w_dom.size() < best.w_dom.size(): best = var return best + def distributable(self): + for var in self._cspace._store.values(): + if var.w_dom.size() > 1: + return True + return False + def w_distribute(self, w_choice): assert isinstance(w_choice, W_IntObject) self.distribute(self._space.int_w(w_choice) -1) Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Wed Aug 16 14:09:53 2006 @@ -8,7 +8,7 @@ from pypy.objspace.std.model import StdObjSpaceMultiMethod from pypy.objspace.cclp.types import W_AbstractDomain, W_Var, ConsistencyError -from pypy.objspace.cclp.interp_var import interp_bind +from pypy.objspace.cclp.interp_var import interp_bind, interp_free all_mms = {} @@ -30,23 +30,26 @@ def clear_change(self): - assert not isinstance(self._changed.w_bound_to, W_Var) + "create a fresh change synchonizer" + assert not interp_free(self._changed) self._changed = W_Var(self._space) def give_synchronizer(self): return self._changed def _value_removed(self): - """The implementation of remove_value should call this method""" + "The implementation of remove_value should call this method" + #atomic interp_bind(self._changed, True) self.clear_change() + #/atomic if self.size() == 0: raise ConsistencyError, "tried to make a domain empty" def set_values(self, w_values): - """Objects in the value set can't be unwrapped unless we - specialize on specific types - this might need speccialization + """XXX Objects in the value set can't be unwrapped unless we + specialize on specific types - this will need specialization of revise & friends """ for w_v in w_values.wrappeditems: Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 14:09:53 2006 @@ -33,11 +33,11 @@ try: _AppThunk.call(self) except Exception, exc: - w(".! exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + w(".! exceptional EXIT of procedure", str(id(self._coro)), "with", str(exc)) scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True else: - w(".! clean (valueless) EXIT of", str(id(self._coro))) + w(".! clean EXIT of procedure", str(id(self._coro))) finally: scheduler[0].remove_thread(self._coro) scheduler[0].schedule() @@ -56,13 +56,13 @@ try: _AppThunk.call(self) except Exception, exc: - w(".! exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + w(".! exceptional EXIT of future", str(id(self._coro)), "with", str(exc)) failed_val = W_FailedValue(exc) self.space.bind(self.w_Result, failed_val) scheduler[0].dirty_traced_vars(self._coro, failed_val) self._coro._dead = True else: - w(".! clean EXIT of", str(id(self._coro)), + w(".! clean EXIT of future", str(id(self._coro)), "-- setting future result", str(self.w_Result), "to", str(self.costate.w_tempval)) self.space.unify(self.w_Result, self.costate.w_tempval) @@ -83,14 +83,14 @@ try: _AppThunk.call(self) except Exception, exc: - w("-- exceptional EXIT of", str(id(self._coro)), "with", str(exc)) + w("-- exceptional EXIT of cspace", str(id(self._coro)), "with", str(exc)) import traceback traceback.print_exc() scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) else: - w("-- clean (valueless) EXIT of", str(id(self._coro))) + w("-- clean (valueless) EXIT of cspace", str(id(self._coro))) self.space.bind(cspace._solution, self.costate.w_tempval) finally: scheduler[0].remove_thread(self._coro) @@ -120,7 +120,7 @@ if not interp_free(cspace._finished): break except ConsistencyError: - self.coro._cspace.fail() + cspace.fail() except: import traceback traceback.print_exc() @@ -142,18 +142,20 @@ cspace = coro._cspace dist = self.dist try: - while 1: + while dist.distributable(): choice = cspace.choose(dist.fanout()) dist.distribute(choice) - except Solution: w("-- DISTRIBUTOR thunk exited because a solution was found") for var in cspace._solution.w_bound_to.wrappeditems: assert var.w_dom.size() == 1 interp_bind(var, var.w_dom.get_values()[0]) + assert interp_free(cspace._choice) interp_bind(cspace._choice, self.space.newint(1)) except ConsistencyError, e: w("-- DISTRIBUTOR thunk exited because", str(e)) interp_bind(cspace._choice, self.space.newint(0)) + import traceback + traceback.print_exc() except: import traceback traceback.print_exc() From benyoung at codespeak.net Wed Aug 16 14:30:51 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Wed, 16 Aug 2006 14:30:51 +0200 (CEST) Subject: [pypy-svn] r31346 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060816123051.D70E71007F@code0.codespeak.net> Author: benyoung Date: Wed Aug 16 14:30:36 2006 New Revision: 31346 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: Added some tests for the implementations + some dict fixes Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 14:30:36 2006 @@ -111,14 +111,17 @@ self.valid += 1 entry.w_value = w_value return self + def delitem(self, w_key): entry = self._lookup(w_key) - if entry.w_value: + if entry.w_value is not None: for i in range(self.entries.index(entry), self.valid): self.entries[i] = self.entries[i+1] self.entries[self.valid] = entry entry.w_value = None self.valid -= 1 + if self.valid == 0: + return self.space.emptydictimpl return self else: raise KeyError @@ -236,7 +239,7 @@ def items(self): space = self.space - return [(space.wrap(key), w_value) for (key, w_value) in self.content.iterkeys()] + return [(space.wrap(key), w_value) for (key, w_value) in self.content.iteritems()] def _is_sane_hash(self, w_lookup_type): """ Handles the case of a non string key lookup. Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Wed Aug 16 14:30:36 2006 @@ -1,7 +1,7 @@ import autopath from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ - EmptyDictImplementation + EmptyDictImplementation, RDictImplementation, StrDictImplementation, SmallDictImplementation from pypy.conftest import gettestobjspace from pypy.objspace.std.test import test_dictobject @@ -13,9 +13,17 @@ def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) +class FakeSpace(test_dictobject.FakeSpace): + def str_w(self, string): + assert isinstance(string, str) + return string + + def wrap(self, obj): + return obj + class TestDictImplementation: def setup_method(self,method): - self.space = test_dictobject.FakeSpace() + self.space = FakeSpace() self.space.emptydictimpl = EmptyDictImplementation(self.space) self.space.DictObjectCls = W_DictMultiObject @@ -30,3 +38,65 @@ pydict[x] = i for x in pydict: assert pydict[x] == getitem__DictMulti_ANY(self.space, d, x) + +class TestRDictImplementation: + ImplementionClass = RDictImplementation + + def setup_method(self,method): + self.space = FakeSpace() + self.space.emptydictimpl = EmptyDictImplementation(self.space) + self.space.DictObjectCls = W_DictMultiObject + self.string = self.space.str_w("fish") + self.string2 = self.space.str_w("fish2") + self.impl = self.get_impl() + + def get_impl(self): + return self.ImplementionClass(self.space) + + def test_setitem(self): + assert self.impl.setitem(self.string, 1000) is self.impl + assert self.impl.get(self.string) == 1000 + + def test_delitem(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + assert self.impl.delitem(self.string) is self.impl + assert self.impl.delitem(self.string2) is self.space.emptydictimpl + + def test_keys(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + keys = self.impl.keys() + keys.sort() + assert keys == [self.string, self.string2] + + def test_values(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + values = self.impl.values() + values.sort() + assert values == [1000, 2000] + + def test_items(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + items = self.impl.items() + items.sort() + assert items == zip([self.string, self.string2], [1000, 2000]) + + def test_devolve(self): + impl = self.impl + for x in xrange(100): + impl = impl.setitem(self.space.str_w(str(x)), x) + impl = impl.setitem(x, x) + assert isinstance(impl, RDictImplementation) + +class TestStrDictImplementation(TestRDictImplementation): + ImplementionClass = StrDictImplementation + +class TestSmallDictImplementation(TestRDictImplementation): + ImplementionClass = SmallDictImplementation + + def get_impl(self): + return self.ImplementionClass(self.space, self.string, self.string2) + From arigo at codespeak.net Wed Aug 16 14:41:17 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 16 Aug 2006 14:41:17 +0200 (CEST) Subject: [pypy-svn] r31347 - in pypy/dist/pypy: interpreter objspace/std objspace/std/test Message-ID: <20060816124117.68B5D10080@code0.codespeak.net> Author: arigo Date: Wed Aug 16 14:41:13 2006 New Revision: 31347 Modified: pypy/dist/pypy/interpreter/argument.py pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/dictobject.py pypy/dist/pypy/objspace/std/dictstrobject.py pypy/dist/pypy/objspace/std/dicttype.py pypy/dist/pypy/objspace/std/test/test_dictobject.py Log: issue244 testing Second try at adding keyword args to dict.update(). This time pypy-c seems to translate successfully. Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Wed Aug 16 14:41:13 2006 @@ -63,7 +63,10 @@ return ArgumentsPrepended(self, w_firstarg) def popfirst(self): - return None, None + """For optimization only: might return (w_firstarg, args_with_rest), + or might just raise IndexError. + """ + raise IndexError def match_signature(self, signature, defaults_w): """Parse args and kwargs according to the signature of a code object, @@ -210,7 +213,7 @@ def popfirst(self): if self.nargs <= 0: - return None, None + raise IndexError valuestack = self.valuestack newnargs = self.nargs-1 return valuestack.top(newnargs), ArgumentsFromValuestack(self.space, valuestack, newnargs) Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Wed Aug 16 14:41:13 2006 @@ -551,8 +551,11 @@ def funcrun(self, func, args): space = func.space - w_obj, newargs = args.popfirst() - if w_obj is not None: + try: + w_obj, newargs = args.popfirst() + except IndexError: + return BuiltinCode.funcrun(self, func, args) + else: try: w_result = self.func__args__(space, w_obj, newargs) except KeyboardInterrupt: @@ -565,8 +568,6 @@ if w_result is None: w_result = space.w_None return w_result - else: - return BuiltinCode.funcrun(self, func, args) class BuiltinCode0(BuiltinCode): def fastcall_0(self, space, w_func): @@ -789,15 +790,20 @@ if ret_w is not None: # it was RPython return ret_w # the last argument can be an Arguments - if args_w and isinstance(args_w[-1], AbstractArguments): - # ...which is merged with the previous arguments, if any - args = args_w[-1] - if len(args_w) > 1: - more_args_w, more_kwds_w = args.unpack() - args = Arguments(space, list(args_w[:-1]) + more_args_w, - more_kwds_w) + if not args_w: + args = Arguments(space, []) else: - args = Arguments(space, list(args_w)) + args = args_w[-1] + assert args is not None + if not isinstance(args, AbstractArguments): + args = Arguments(space, list(args_w)) + else: + # ...which is merged with the previous arguments, if any + if len(args_w) > 1: + more_args_w, more_kwds_w = args.unpack() + args = Arguments(space, + list(args_w[:-1]) + more_args_w, + more_kwds_w) w_func = self.wget(space, name) return space.call_args(w_func, args) def get_function(space): Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 14:41:13 2006 @@ -481,11 +481,11 @@ w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__DictMulti_ANY(space, w_dict, w_lookup): w_value = w_dict.implementation.get(w_lookup) @@ -569,9 +569,9 @@ return w_res def dict_copy__DictMulti(space, w_self): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY + from pypy.objspace.std.dicttype import update1 w_new = W_DictMultiObject(space) - dict_update__ANY_ANY(space, w_new, w_self) + update1(space, w_new, w_self) return w_new def dict_items__DictMulti(space, w_self): Modified: pypy/dist/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictobject.py (original) +++ pypy/dist/pypy/objspace/std/dictobject.py Wed Aug 16 14:41:13 2006 @@ -65,11 +65,11 @@ w_dict.content[w_k] = w_v else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__Dict_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dictstrobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictstrobject.py (original) +++ pypy/dist/pypy/objspace/std/dictstrobject.py Wed Aug 16 14:41:13 2006 @@ -121,11 +121,11 @@ w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_src) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_src) if space.is_true(w_kwds): - from pypy.objspace.std.dicttype import dict_update__ANY_ANY - dict_update__ANY_ANY(space, w_dict, w_kwds) + from pypy.objspace.std.dicttype import update1 + update1(space, w_dict, w_kwds) def getitem__DictStr_ANY(space, w_dict, w_lookup): try: Modified: pypy/dist/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/dist/pypy/objspace/std/dicttype.py (original) +++ pypy/dist/pypy/objspace/std/dicttype.py Wed Aug 16 14:41:13 2006 @@ -30,7 +30,7 @@ dict_setdefault = SMM('setdefault', 3, defaults=(None,), doc='D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d' ' if k not in D') -dict_update = SMM('update', 2, defaults=((),), +dict_update = SMM('update', 1, general__args__=True, doc='D.update(E, **F) -> None. Update D from E and F:' ' for k in E: D[k] = E[k]\n(if E has keys else: for' ' (k, v) in E: D[k] = v) then: for k in F: D[k] =' @@ -55,7 +55,7 @@ # gateway is imported in the stdtypedef module app = gateway.applevel(''' - def update(d, o): + def update1(d, o): if hasattr(o, 'keys'): for k in o.keys(): d[k] = o[k] @@ -63,6 +63,14 @@ for k,v in o: d[k] = v + def update(d, *args, **kwargs): + if len(args) == 1: + update1(d, args[0]) + elif len(args) > 1: + raise TypeError("update takes at most 1 (non-keyword) argument") + if kwargs: + update1(d, kwargs) + def popitem(d): k = d.keys() if not k: @@ -110,7 +118,7 @@ ''', filename=__file__) #XXX what about dict.fromkeys()? -dict_update__ANY_ANY = app.interphook("update") +dict_update__ANY = app.interphook("update") dict_popitem__ANY = app.interphook("popitem") dict_get__ANY_ANY_ANY = app.interphook("get") dict_setdefault__ANY_ANY_ANY = app.interphook("setdefault") @@ -118,6 +126,7 @@ dict_iteritems__ANY = app.interphook("iteritems") dict_iterkeys__ANY = app.interphook("iterkeys") dict_itervalues__ANY = app.interphook("itervalues") +update1 = app.interphook("update1") register_all(vars(), globals()) Modified: pypy/dist/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictobject.py Wed Aug 16 14:41:13 2006 @@ -245,7 +245,17 @@ d = {} d.update() assert d == {} - + + def test_update_kwargs(self): + d = {} + d.update(foo='bar', baz=1) + assert d == {'foo': 'bar', 'baz': 1} + + def test_update_dict_and_kwargs(self): + d = {} + d.update({'foo': 'bar'}, baz=1) + assert d == {'foo': 'bar', 'baz': 1} + def test_values(self): d = {1:2, 3:4} vals = d.values() From auc at codespeak.net Wed Aug 16 14:46:30 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 14:46:30 +0200 (CEST) Subject: [pypy-svn] r31348 - pypy/dist/pypy/objspace/cclp Message-ID: <20060816124630.B571A10080@code0.codespeak.net> Author: auc Date: Wed Aug 16 14:46:28 2006 New Revision: 31348 Modified: pypy/dist/pypy/objspace/cclp/interp_var.py pypy/dist/pypy/objspace/cclp/thunk.py Log: wait_or Modified: pypy/dist/pypy/objspace/cclp/interp_var.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/interp_var.py (original) +++ pypy/dist/pypy/objspace/cclp/interp_var.py Wed Aug 16 14:46:28 2006 @@ -66,3 +66,11 @@ if not w_val in w_var.w_dom._values.content: raise ValueError, "assignment out of domain" w_var.w_bound_to = w_val + + +def interp_wait_or(space, lvars): + assert isinstance(lvars, list) + O = W_Var(space) + for V in lvars: + interp_entail(V, O) + return interp_wait(space, O) Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 14:46:28 2006 @@ -4,7 +4,7 @@ from pypy.objspace.cclp.misc import w from pypy.objspace.cclp.global_state import scheduler from pypy.objspace.cclp.types import W_Var, W_Future, W_FailedValue, ConsistencyError, Solution -from pypy.objspace.cclp.interp_var import interp_wait, interp_entail, interp_bind, interp_free +from pypy.objspace.cclp.interp_var import interp_wait, interp_entail, interp_bind, interp_free, interp_wait_or def logic_args(args): @@ -71,6 +71,7 @@ scheduler[0].schedule() class CSpaceThunk(_AppThunk): + "for a constraint script/logic program" def __init__(self, space, w_callable, args, coro): _AppThunk.__init__(self, space, coro.costate, w_callable, args) self._coro = coro @@ -84,8 +85,6 @@ _AppThunk.call(self) except Exception, exc: w("-- exceptional EXIT of cspace", str(id(self._coro)), "with", str(exc)) - import traceback - traceback.print_exc() scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) @@ -111,12 +110,12 @@ entailed = self.const.revise() if entailed: break - Obs = W_Var(self.space) - interp_entail(cspace._finished, Obs) - for Sync in [var.w_dom.give_synchronizer() - for var in self.const._variables]: - interp_entail(Sync, Obs) - interp_wait(self.space, Obs) + # we will block on domains being pruned + wait_list = [var.w_dom.give_synchronizer() + for var in self.const._variables] + #or the cspace being dead + wait_list.append(cspace._finished) + interp_wait_or(self.space, wait_list) if not interp_free(cspace._finished): break except ConsistencyError: @@ -146,6 +145,7 @@ choice = cspace.choose(dist.fanout()) dist.distribute(choice) w("-- DISTRIBUTOR thunk exited because a solution was found") + #XXX assert that all propagators are entailed for var in cspace._solution.w_bound_to.wrappeditems: assert var.w_dom.size() == 1 interp_bind(var, var.w_dom.get_values()[0]) From auc at codespeak.net Wed Aug 16 15:11:17 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 15:11:17 +0200 (CEST) Subject: [pypy-svn] r31349 - in pypy/dist/pypy/objspace: cclp test Message-ID: <20060816131117.05E6A1007F@code0.codespeak.net> Author: auc Date: Wed Aug 16 15:11:15 2006 New Revision: 31349 Modified: pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: also test no solution case Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 15:11:15 2006 @@ -154,8 +154,6 @@ except ConsistencyError, e: w("-- DISTRIBUTOR thunk exited because", str(e)) interp_bind(cspace._choice, self.space.newint(0)) - import traceback - traceback.print_exc() except: import traceback traceback.print_exc() Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 16 15:11:15 2006 @@ -395,7 +395,7 @@ # outside this thread ... But this can't # happen right now because we keep # references also of this thread in the - # scheduler. + # scheduler. wait_needed(L) Tail = newvar() bind(L, (n, Tail)) @@ -766,14 +766,14 @@ #assert len(sched_all()['threads']) == 1 - def test_tell_ask_choose_commit(self): + def test_tell_ask_choose_commit_success(self): from problem import conference_scheduling - def solve(spc, Sol): + def solve(spc, commitment, Sol): while 1: status = spc.ask() if status > 1: - spc.commit(1) + spc.commit(commitment) elif status in (0, 1): break if status: @@ -781,15 +781,19 @@ else: unify(Sol, False) - s = newspace(conference_scheduling) - Solution = newvar() - stacklet(solve, s, Solution) - - assert Solution == [('room B', 'day 1 PM'), ('room A', 'day 1 PM'), - ('room B', 'day 2 AM'), ('room B', 'day 1 AM'), - ('room A', 'day 2 PM'), ('room C', 'day 2 AM'), - ('room C', 'day 2 PM'), ('room C', 'day 1 PM'), - ('room C', 'day 1 AM'), ('room B', 'day 2 PM')] + for commit_to in (1, 2): + s = newspace(conference_scheduling) + Solution = newvar() + stacklet(solve, s, commit_to, Solution) + if commit_to == 1: + assert Solution == [('room B', 'day 1 PM'), ('room A', 'day 1 PM'), + ('room B', 'day 2 AM'), ('room B', 'day 1 AM'), + ('room A', 'day 2 PM'), ('room C', 'day 2 AM'), + ('room C', 'day 2 PM'), ('room C', 'day 1 PM'), + ('room C', 'day 1 AM'), ('room B', 'day 2 PM')] + else: + assert Solution == False - #XXX + #XXX who's still stuck there ? #assert len(sched_all()['threads']) == 1 + From mwh at codespeak.net Wed Aug 16 17:55:52 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 16 Aug 2006 17:55:52 +0200 (CEST) Subject: [pypy-svn] r31350 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060816155552.B77931007F@code0.codespeak.net> Author: mwh Date: Wed Aug 16 17:55:51 2006 New Revision: 31350 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: tweaks, and a SmallStrDictImplementation which doesn't work, probably for stupid reasons. but now i have to ru for a train, literally. Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Wed Aug 16 17:55:51 2006 @@ -3,10 +3,15 @@ from pypy.rpython.objectmodel import r_dict, we_are_translated +def _is_str(space, w_key): + return space.is_w(space.type(w_key), space.w_str) + class DictImplementation(object): ## def get(self, w_lookup): ## return w_value or None +## def setitem_str(self, w_key, w_value): +## return implementation ## def setitem(self, w_key, w_value): ## return implementation ## def delitem(self, w_key): @@ -36,7 +41,15 @@ def get(self, w_lookup): return None def setitem(self, w_key, w_value): - return SmallDictImplementation(self.space, w_key, w_value) + if _is_str(self.space, w_key): + return StrDictImplementation(self.space, w_key, w_value) + #return SmallStrDictImplementation(self.space, w_key, w_value) + else: + return RDictImplementation(self.space).setitem(w_key, w_value) + #return SmallDictImplementation(self.space, w_key, w_value) + def setitem_str(self, w_key, w_value): + return StrDictImplementation(self.space, w_key, w_value) + #return SmallStrDictImplementation(self.space, w_key, w_value) def delitem(self, w_key): raise KeyError @@ -77,10 +90,6 @@ self.valid = 1 def _lookup(self, w_key): -## for i in range(self.valid): -## assert self.entries[i].w_value is not None -## for i in range(self.valid+1, 5): -## assert self.entries[i].w_value is None hash = self.space.hash_w(w_key) i = 0 last = self.entries[self.valid] @@ -145,6 +154,123 @@ def items(self): return [(e.w_key, e.w_value) for e in [self.entries[i] for i in range(self.valid)]] +class StrEntry(object): + def __init__(self): + self.hash = 0 + self.key = None + self.w_value = None + def __repr__(self): + return '<%r, %r, %r>'%(self.hash, self.key, self.w_value) + +class SmallStrDictImplementation(DictImplementation): + # XXX document the invariants here! + + def __init__(self, space, w_key, w_value): + self.space = space + self.entries = [StrEntry(), StrEntry(), StrEntry(), StrEntry(), StrEntry()] + key = space.str_w(w_key) + self.entries[0].key = key + self.entries[0].hash = hash(key) + self.entries[0].w_value = w_value + self.valid = 1 + + def _lookup(self, key): + _hash = hash(key) + i = 0 + last = self.entries[self.valid] + last.hash = _hash + last.key = key + while 1: + look_entry = self.entries[i] + if look_entry.hash == _hash and look_entry.key == key: + return look_entry + i += 1 + + def _convert_to_rdict(self): + newimpl = RDictImplementation(self.space) + i = 0 + while 1: + entry = self.entries[i] + if entry.w_value is None: + break + newimpl.content[self.space.wrap(entry.key)] = entry.w_value + i += 1 + return newimpl + + def _convert_to_sdict(self, w_key, w_value): + newimpl = StrDictImplementation(self.space, w_key, w_value) + i = 0 + while 1: + entry = self.entries[i] + if entry.w_value is None: + break + newimpl.content[entry.key] = entry.w_value + i += 1 + return newimpl + + def setitem(self, w_key, w_value): + if not _is_str(self.space, w_key): + return self._convert_to_rdict().setitem(w_key, w_value) + return self.setitem_str(w_key, w_value) + + def setitem_str(self, w_key, w_value): + print w_key, w_value + if self.valid == 4: + return self._convert_to_sdict(w_key, w_value) + entry = self._lookup(self.space.str_w(w_key)) + if entry.w_value is None: + self.valid += 1 + entry.w_value = w_value + return self + + def delitem(self, w_key): + space = self.space + if space.is_w(space.type(w_key), space.w_str): + entry = self._lookup(space.str_w(w_key)) + if entry.w_value is not None: + for i in range(self.entries.index(entry), self.valid): + self.entries[i] = self.entries[i+1] + self.entries[self.valid] = entry + entry.w_value = None + self.valid -= 1 + if self.valid == 0: + return self.space.emptydictimpl + return self + else: + raise KeyError + elif self._is_sane_hash(space.type(w_key)): + raise KeyError + else: + return self._convert_to_rdict().delitem(w_key) + + def length(self): + return self.valid + + def get(self, w_lookup): + space = self.space + w_lookup_type = space.type(w_lookup) + if space.is_w(w_lookup_type, space.w_str): + return self._lookup(w_lookup).w_value + elif self._is_sane_hash(w_lookup_type): + return None + else: + return self._convert_to_rdict().get(w_lookup) + + def iteritems(self): + return self._convert_to_rdict().iteritems() + def iterkeys(self): + return self._convert_to_rdict().iterkeys() + def itervalues(self): + return self._convert_to_rdict().itervalues() + + def keys(self): + return [self.space.wrap(self.entries[i].key) for i in range(self.valid)] + def values(self): + return [self.entries[i].w_value for i in range(self.valid)] + def items(self): + return [(self.space.wrap(e.key), e.w_value) + for e in [self.entries[i] for i in range(self.valid)]] + class RDictImplementation(DictImplementation): def __init__(self, space): self.space = space @@ -156,6 +282,7 @@ def setitem(self, w_key, w_value): self.content[w_key] = w_value return self + setitem_str = setitem def delitem(self, w_key): del self.content[w_key] if self.content: @@ -183,11 +310,12 @@ return self.content.items() class StrDictImplementation(DictImplementation): - def __init__(self, space): + def __init__(self, space, w_key, w_value): self.space = space - self.content = {} + self.content = {space.str_w(w_key): w_value} def setitem(self, w_key, w_value): + print 's', w_key, w_value space = self.space if space.is_w(space.type(w_key), space.w_str): self.content[space.str_w(w_key)] = w_value @@ -195,6 +323,10 @@ else: return self._as_rdict().setitem(w_key, w_value) + def setitem_str(self, w_key, w_value): + self.content[self.space.str_w(w_key)] = w_value + return self + def delitem(self, w_key): space = self.space if space.is_w(space.type(w_key), space.w_str): @@ -203,7 +335,7 @@ return self else: return space.emptydictimpl - elif self._is_sane_hash(w_lookup_type): + elif self._is_sane_hash(space.type(w_key)): raise KeyError else: return self._as_rdict().delitem(w_key) @@ -265,7 +397,7 @@ def __init__(self): self.id = len(self._dict_infos) - self.setitems = 0; self.delitems = 0 + self.setitem_strs = 0; self.setitems = 0; self.delitems = 0 self.lengths = 0; self.gets = 0 self.iteritems = 0; self.iterkeys = 0; self.itervalues = 0 self.keys = 0; self.values = 0; self.items = 0 @@ -350,6 +482,9 @@ self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) return self + def setitem(self, w_key, w_value): + self.info.setitem_strs += 1 + return self.setitem(w_key, w_value) def delitem(self, w_key): if not self.info.seen_non_string_in_write \ and not self.info.seen_non_string_in_read_first \ @@ -457,7 +592,7 @@ return w_default def set_str_keyed_item(w_dict, w_key, w_value): - w_dict.implementation = w_dict.implementation.setitem(w_key, w_value) + w_dict.implementation = w_dict.implementation.setitem_str(w_key, w_value) registerimplementation(W_DictMultiObject) Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Wed Aug 16 17:55:51 2006 @@ -1,7 +1,8 @@ import autopath from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ - EmptyDictImplementation, RDictImplementation, StrDictImplementation, SmallDictImplementation + EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ + SmallDictImplementation, SmallStrDictImplementation from pypy.conftest import gettestobjspace from pypy.objspace.std.test import test_dictobject @@ -60,8 +61,10 @@ def test_delitem(self): self.impl.setitem(self.string, 1000) self.impl.setitem(self.string2, 2000) - assert self.impl.delitem(self.string) is self.impl - assert self.impl.delitem(self.string2) is self.space.emptydictimpl + newimpl = self.impl.delitem(self.string) + assert newimpl is self.impl + newimpl = self.impl.delitem(self.string2) + assert newimpl is self.space.emptydictimpl def test_keys(self): self.impl.setitem(self.string, 1000) @@ -94,9 +97,18 @@ class TestStrDictImplementation(TestRDictImplementation): ImplementionClass = StrDictImplementation + def get_impl(self): + return self.ImplementionClass(self.space, self.string, self.string2) + class TestSmallDictImplementation(TestRDictImplementation): ImplementionClass = SmallDictImplementation def get_impl(self): return self.ImplementionClass(self.space, self.string, self.string2) +class TestSmallStrDictImplementation(TestRDictImplementation): + ImplementionClass = SmallStrDictImplementation + + def get_impl(self): + return self.ImplementionClass(self.space, self.string, self.string2) + From auc at codespeak.net Wed Aug 16 18:06:54 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 18:06:54 +0200 (CEST) Subject: [pypy-svn] r31351 - pypy/dist/pypy/objspace/test Message-ID: <20060816160654.3B47C10080@code0.codespeak.net> Author: auc Date: Wed Aug 16 18:06:52 2006 New Revision: 31351 Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: distribution by a 'relational' program Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 16 18:06:52 2006 @@ -766,7 +766,7 @@ #assert len(sched_all()['threads']) == 1 - def test_tell_ask_choose_commit_success(self): + def test_tell_ask_choose_commit(self): from problem import conference_scheduling def solve(spc, commitment, Sol): @@ -797,3 +797,61 @@ #XXX who's still stuck there ? #assert len(sched_all()['threads']) == 1 + def test_logic_program(self): + skip("coming soon") + + def soft(): + choice = choose(2) + if choice == 1: + return 'beige' + else: + return 'coral' + + def hard(): + choice = choose(2) + if choice == 1: + return 'mauve' + else: + return 'ochre' + + def contrast(C1, C2): + choice = choose(2) + if choice == 1: + unify(C1, soft()) + unify(C2, hard()) + else: + unify(C1, hard()) + unify(C2, soft()) + + def suit(): + Shirt, Pants, Socks = newvar(), newvar(), newvar() + contrast(Shirt, Pants) + contrast(Pants, Socks) + print Shirt, Socks, Pants + if Shirt == Socks: fail() + return (Shirt, Pants, Socks) + + def solve(spc, commitment, Sol): + while 1: + status = spc.ask() + if status > 1: + spc.commit(commitment.next()) + elif status in (0, 1): + break + if status: + unify(Sol, spc.merge()) + else: + unify(Sol, False) + + def lazily(lst): + def _(): + for e in lst: + yield e + return _ + + for commit_to in (lazily([1, 1, 1, 1, 1, 1]), + lazily([1, 1, 1, 2, 1, 2])): + s = newspace(suit) + Solution = newvar() + stacklet(solve, s, commit_to(), Solution) + print Solution From auc at codespeak.net Wed Aug 16 18:31:16 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 18:31:16 +0200 (CEST) Subject: [pypy-svn] r31352 - pypy/dist/pypy/objspace/cclp Message-ID: <20060816163116.579DB10082@code0.codespeak.net> Author: auc Date: Wed Aug 16 18:31:14 2006 New Revision: 31352 Modified: pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thunk.py Log: setting the distributor Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Wed Aug 16 18:31:14 2006 @@ -40,7 +40,7 @@ try: return space.newint(cspace.choose(w_n.intval)) except ConsistencyError: - raise OperationError(space.w_ConsistecyError, + raise OperationError(space.w_ConsistencyError, space.wrap("the space is failed")) raise OperationError(space.w_RuntimeError, space.wrap("choose is forbidden from the top-level space")) @@ -62,7 +62,7 @@ assert (parent is None) or isinstance(parent, W_CSpace) self.space = space # the object space ;-) self.parent = parent - self.main_thread = thread + self.distributor = thread # choice mgmt self._choice = newvar(space) self._committed = newvar(space) Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 18:31:14 2006 @@ -76,10 +76,14 @@ _AppThunk.__init__(self, space, coro.costate, w_callable, args) self._coro = coro + def is_distributor(self): + return self._coro == self._coro._cspace.distributor + def call(self): w("-- initial thunk CALL in", str(id(self._coro))) scheduler[0].trace_vars(self._coro, logic_args(self.args.unpack())) cspace = self._coro._cspace + cspace.distributor = self._coro try: try: _AppThunk.call(self) @@ -87,10 +91,15 @@ w("-- exceptional EXIT of cspace", str(id(self._coro)), "with", str(exc)) scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) self._coro._dead = True - self.space.bind(cspace._choice, self.space.wrap(SPACE_FAILURE)) + if self.is_distributor(): + cspace.fail() + import traceback + traceback.print_exc() else: w("-- clean (valueless) EXIT of cspace", str(id(self._coro))) - self.space.bind(cspace._solution, self.costate.w_tempval) + interp_bind(cspace._solution, self.costate.w_tempval) + if self.is_distributor(): + interp_bind(cspace._choice, self.space.newint(1)) finally: scheduler[0].remove_thread(self._coro) scheduler[0].schedule() @@ -139,6 +148,7 @@ coro = self.coro try: cspace = coro._cspace + cspace.distributor = coro dist = self.dist try: while dist.distributable(): From auc at codespeak.net Wed Aug 16 18:48:49 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Wed, 16 Aug 2006 18:48:49 +0200 (CEST) Subject: [pypy-svn] r31353 - in pypy/dist/pypy/objspace: cclp test Message-ID: <20060816164849.B853E1007E@code0.codespeak.net> Author: auc Date: Wed Aug 16 18:48:47 2006 New Revision: 31353 Modified: pypy/dist/pypy/objspace/cclp/space.py pypy/dist/pypy/objspace/cclp/thunk.py pypy/dist/pypy/objspace/test/test_logicobjspace.py Log: fixes for fail, thunks -- can run a successful or failing relational program (still not non-deterministically) Modified: pypy/dist/pypy/objspace/cclp/space.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/space.py (original) +++ pypy/dist/pypy/objspace/cclp/space.py Wed Aug 16 18:48:47 2006 @@ -118,6 +118,8 @@ def fail(self): self._failed = True + interp_bind(self._finished, True) + interp_bind(self._choice, self.space.newint(0)) self._store = {} def w_merge(self): Modified: pypy/dist/pypy/objspace/cclp/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/thunk.py (original) +++ pypy/dist/pypy/objspace/cclp/thunk.py Wed Aug 16 18:48:47 2006 @@ -88,13 +88,14 @@ try: _AppThunk.call(self) except Exception, exc: + # maybe app_level sent something ... w("-- exceptional EXIT of cspace", str(id(self._coro)), "with", str(exc)) - scheduler[0].dirty_traced_vars(self._coro, W_FailedValue(exc)) + failed_value = W_FailedValue(exc) + scheduler[0].dirty_traced_vars(self._coro, failed_value) self._coro._dead = True if self.is_distributor(): cspace.fail() - import traceback - traceback.print_exc() + interp_bind(cspace._solution, failed_value) else: w("-- clean (valueless) EXIT of cspace", str(id(self._coro))) interp_bind(cspace._solution, self.costate.w_tempval) Modified: pypy/dist/pypy/objspace/test/test_logicobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_logicobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_logicobjspace.py Wed Aug 16 18:48:47 2006 @@ -768,7 +768,7 @@ def test_tell_ask_choose_commit(self): from problem import conference_scheduling - + def solve(spc, commitment, Sol): while 1: status = spc.ask() @@ -798,7 +798,6 @@ #assert len(sched_all()['threads']) == 1 def test_logic_program(self): - skip("coming soon") def soft(): choice = choose(2) @@ -827,7 +826,6 @@ Shirt, Pants, Socks = newvar(), newvar(), newvar() contrast(Shirt, Pants) contrast(Pants, Socks) - print Shirt, Socks, Pants if Shirt == Socks: fail() return (Shirt, Pants, Socks) @@ -854,4 +852,4 @@ s = newspace(suit) Solution = newvar() stacklet(solve, s, commit_to(), Solution) - print Solution + assert Solution in (False, ('beige', 'mauve', 'coral')) From benyoung at codespeak.net Thu Aug 17 09:23:58 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Thu, 17 Aug 2006 09:23:58 +0200 (CEST) Subject: [pypy-svn] r31354 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060817072358.2DEE610082@code0.codespeak.net> Author: benyoung Date: Thu Aug 17 09:23:43 2006 New Revision: 31354 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: Some fixes for mwhs changes + more tests Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Thu Aug 17 09:23:43 2006 @@ -214,18 +214,18 @@ return self.setitem_str(w_key, w_value) def setitem_str(self, w_key, w_value): - print w_key, w_value - if self.valid == 4: - return self._convert_to_sdict(w_key, w_value) entry = self._lookup(self.space.str_w(w_key)) if entry.w_value is None: + if self.valid == 4: + return self._convert_to_sdict(w_key, w_value) self.valid += 1 entry.w_value = w_value return self def delitem(self, w_key): space = self.space - if space.is_w(space.type(w_key), space.w_str): + w_key_type = space.type(w_key) + if space.is_w(w_key_type, space.w_str): entry = self._lookup(space.str_w(w_key)) if entry.w_value is not None: for i in range(self.entries.index(entry), self.valid): @@ -238,7 +238,7 @@ return self else: raise KeyError - elif self._is_sane_hash(space.type(w_key)): + elif self._is_sane_hash(w_key_type): raise KeyError else: return self._convert_to_rdict().delitem(w_key) @@ -271,51 +271,12 @@ return [(self.space.wrap(e.key), e.w_value) for e in [self.entries[i] for i in range(self.valid)]] -class RDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - self.content = r_dict(space.eq_w, space.hash_w) - - def __repr__(self): - return "%s<%s>" % (self.__class__.__name__, self.content) - - def setitem(self, w_key, w_value): - self.content[w_key] = w_value - return self - setitem_str = setitem - def delitem(self, w_key): - del self.content[w_key] - if self.content: - return self - else: - return self.space.emptydictimpl - - def length(self): - return len(self.content) - def get(self, w_lookup): - return self.content.get(w_lookup, None) - - def iteritems(self): - return self.content.iteritems() - def iterkeys(self): - return self.content.iterkeys() - def itervalues(self): - return self.content.itervalues() - - def keys(self): - return self.content.keys() - def values(self): - return self.content.values() - def items(self): - return self.content.items() - class StrDictImplementation(DictImplementation): def __init__(self, space, w_key, w_value): self.space = space self.content = {space.str_w(w_key): w_value} def setitem(self, w_key, w_value): - print 's', w_key, w_value space = self.space if space.is_w(space.type(w_key), space.w_str): self.content[space.str_w(w_key)] = w_value @@ -329,13 +290,14 @@ def delitem(self, w_key): space = self.space - if space.is_w(space.type(w_key), space.w_str): + w_key_type = space.type(w_key) + if space.is_w(w_key_type, space.w_str): del self.content[space.str_w(w_key)] if self.content: return self else: return space.emptydictimpl - elif self._is_sane_hash(space.type(w_key)): + elif self._is_sane_hash(w_key_type): raise KeyError else: return self._as_rdict().delitem(w_key) @@ -389,6 +351,43 @@ newimpl.setitem(self.space.wrap(k), w_v) return newimpl +class RDictImplementation(DictImplementation): + def __init__(self, space): + self.space = space + self.content = r_dict(space.eq_w, space.hash_w) + + def __repr__(self): + return "%s<%s>" % (self.__class__.__name__, self.content) + + def setitem(self, w_key, w_value): + self.content[w_key] = w_value + return self + setitem_str = setitem + def delitem(self, w_key): + del self.content[w_key] + if self.content: + return self + else: + return self.space.emptydictimpl + + def length(self): + return len(self.content) + def get(self, w_lookup): + return self.content.get(w_lookup, None) + + def iteritems(self): + return self.content.iteritems() + def iterkeys(self): + return self.content.iterkeys() + def itervalues(self): + return self.content.itervalues() + + def keys(self): + return self.content.keys() + def values(self): + return self.content.values() + def items(self): + return self.content.items() import time, py @@ -482,7 +481,7 @@ self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) return self - def setitem(self, w_key, w_value): + def setitem_str(self, w_key, w_value): self.info.setitem_strs += 1 return self.setitem(w_key, w_value) def delitem(self, w_key): @@ -499,10 +498,10 @@ def length(self): self.info.lengths += 1 return len(self.content) - def get(self, w_lookup, w_default): + def get(self, w_lookup): self.info.gets += 1 self._read(w_lookup) - return self.content.get(w_lookup, w_default) + return self.content.get(w_lookup, None) def iteritems(self): self.info.iteritems += 1 Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Thu Aug 17 09:23:43 2006 @@ -2,7 +2,7 @@ from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ - SmallDictImplementation, SmallStrDictImplementation + SmallDictImplementation, SmallStrDictImplementation, MeasuringDictImplementation from pypy.conftest import gettestobjspace from pypy.objspace.std.test import test_dictobject @@ -22,6 +22,9 @@ def wrap(self, obj): return obj + def isinstance(self, obj, klass): + return isinstance(obj, klass) + class TestDictImplementation: def setup_method(self,method): self.space = FakeSpace() @@ -42,11 +45,13 @@ class TestRDictImplementation: ImplementionClass = RDictImplementation + DevolvedClass = RDictImplementation + EmptyClass = EmptyDictImplementation def setup_method(self,method): self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) self.space.DictObjectCls = W_DictMultiObject + self.space.emptydictimpl = EmptyDictImplementation(self.space) self.string = self.space.str_w("fish") self.string2 = self.space.str_w("fish2") self.impl = self.get_impl() @@ -64,7 +69,7 @@ newimpl = self.impl.delitem(self.string) assert newimpl is self.impl newimpl = self.impl.delitem(self.string2) - assert newimpl is self.space.emptydictimpl + assert isinstance(newimpl, self.EmptyClass) def test_keys(self): self.impl.setitem(self.string, 1000) @@ -92,7 +97,7 @@ for x in xrange(100): impl = impl.setitem(self.space.str_w(str(x)), x) impl = impl.setitem(x, x) - assert isinstance(impl, RDictImplementation) + assert isinstance(impl, self.DevolvedClass) class TestStrDictImplementation(TestRDictImplementation): ImplementionClass = StrDictImplementation @@ -106,9 +111,13 @@ def get_impl(self): return self.ImplementionClass(self.space, self.string, self.string2) +class TestMeasuringDictImplementation(TestRDictImplementation): + ImplementionClass = MeasuringDictImplementation + DevolvedClass = MeasuringDictImplementation + EmptyClass = MeasuringDictImplementation + class TestSmallStrDictImplementation(TestRDictImplementation): ImplementionClass = SmallStrDictImplementation def get_impl(self): return self.ImplementionClass(self.space, self.string, self.string2) - From benyoung at codespeak.net Thu Aug 17 09:30:54 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Thu, 17 Aug 2006 09:30:54 +0200 (CEST) Subject: [pypy-svn] r31355 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060817073054.CFE5210082@code0.codespeak.net> Author: benyoung Date: Thu Aug 17 09:30:44 2006 New Revision: 31355 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: More fixes + more tests for dict impls Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Thu Aug 17 09:30:44 2006 @@ -113,14 +113,16 @@ return newimpl def setitem(self, w_key, w_value): - if self.valid == 4: - return self._convert_to_rdict().setitem(w_key, w_value) entry = self._lookup(w_key) if entry.w_value is None: + if self.valid == 4: + return self._convert_to_rdict().setitem(w_key, w_value) self.valid += 1 entry.w_value = w_value return self + setitem_str = setitem + def delitem(self, w_key): entry = self._lookup(w_key) if entry.w_value is not None: Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Thu Aug 17 09:30:44 2006 @@ -63,6 +63,10 @@ assert self.impl.setitem(self.string, 1000) is self.impl assert self.impl.get(self.string) == 1000 + def test_setitem_str(self): + assert self.impl.setitem_str(self.string, 1000) is self.impl + assert self.impl.get(self.string) == 1000 + def test_delitem(self): self.impl.setitem(self.string, 1000) self.impl.setitem(self.string2, 2000) @@ -92,6 +96,27 @@ items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + def test_iterkeys(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + keys = list(self.impl.iterkeys()) + keys.sort() + assert keys == [self.string, self.string2] + + def test_itervalues(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + values = list(self.impl.itervalues()) + values.sort() + assert values == [1000, 2000] + + def test_iteritems(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + items = list(self.impl.iteritems()) + items.sort() + assert items == zip([self.string, self.string2], [1000, 2000]) + def test_devolve(self): impl = self.impl for x in xrange(100): From auc at codespeak.net Thu Aug 17 09:58:52 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 17 Aug 2006 09:58:52 +0200 (CEST) Subject: [pypy-svn] r31356 - in pypy/dist/pypy/objspace/cclp/constraint: . test Message-ID: <20060817075852.16C1810084@code0.codespeak.net> Author: auc Date: Thu Aug 17 09:58:49 2006 New Revision: 31356 Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py Log: fix two test failures Modified: pypy/dist/pypy/objspace/cclp/constraint/domain.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/domain.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/domain.py Thu Aug 17 09:58:49 2006 @@ -54,6 +54,13 @@ """ for w_v in w_values.wrappeditems: self._space.setitem(self._values, w_v, self._space.w_True) + + def w_remove_value(self, w_value): + try: + self.remove_value(w_value) + except ConsistencyError: + raise OperationError(self._space.w_ConsistencyError, + self._space.wrap("tried to empty a domain")) def remove_value(self, w_value): """Remove value of domain and check for consistency""" @@ -135,7 +142,7 @@ W_FiniteDomain.typedef = typedef.TypeDef( "W_FiniteDomain", W_AbstractDomain.typedef, - remove_value = interp2app(W_FiniteDomain.remove_value), + remove_value = interp2app(W_FiniteDomain.w_remove_value), remove_values = interp2app(W_FiniteDomain.w_remove_values), get_values = interp2app(W_FiniteDomain.w_get_values), __eq__ = interp2app(W_FiniteDomain.__eq__), Modified: pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_constraint.py Thu Aug 17 09:58:49 2006 @@ -14,8 +14,6 @@ variables = cstr.affected_variables() assert variables is not None assert len(variables) == 2 - assert cstr.knows_var(v1) - assert cstr.knows_var(v2) def test_revise(self): def in_space(): Modified: pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py ============================================================================== --- pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py (original) +++ pypy/dist/pypy/objspace/cclp/constraint/test/test_fd.py Thu Aug 17 09:58:49 2006 @@ -19,8 +19,7 @@ def test_remove_all_values(self): fd = FiniteDomain([3]) - # FIXME: check this is a ConsistencyFailure - raises(Exception, fd.remove_value, 3) + raises(ConsistencyError, fd.remove_value, 3) def test_remove_values(self): fd = FiniteDomain([1, 2, 3]) From arigo at codespeak.net Thu Aug 17 10:33:04 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 17 Aug 2006 10:33:04 +0200 (CEST) Subject: [pypy-svn] r31357 - pypy/dist/pypy/translator/js Message-ID: <20060817083304.344D910084@code0.codespeak.net> Author: arigo Date: Thu Aug 17 10:33:01 2006 New Revision: 31357 Modified: pypy/dist/pypy/translator/js/database.py Log: Fixed the __eq__ methods to check the type of the 'other' argument. (Crashed for me in js/demo/jsdemo/example.py) Modified: pypy/dist/pypy/translator/js/database.py ============================================================================== --- pypy/dist/pypy/translator/js/database.py (original) +++ pypy/dist/pypy/translator/js/database.py Thu Aug 17 10:33:01 2006 @@ -204,7 +204,14 @@ self.cts = db.type_system_class(db) self.depends = set() self.depends_on = set() - + + def __hash__(self): + return hash(self.get_key()) + + def __eq__(self, other): + return (other.__class__ is self.__class__ and + other.get_key() == self.get_key()) + def __ne__(self, other): return not (self == other) @@ -259,11 +266,8 @@ self.static_type = static_type self.cts.lltype_to_cts(obj._TYPE) # force scheduling of obj's class - def __hash__(self): - return hash(self.obj) - - def __eq__(self, other): - return self.obj == other.obj + def get_key(self): + return self.obj def get_name(self): return self.obj._TYPE._name.replace('.', '_') @@ -334,11 +338,8 @@ self.depends.add(name) name.depends_on.add(self) - def __hash__(self): - return hash(self.const) - - def __eq__(self, other): - return self.const == other.const + def get_key(self): + return self.const def init_fields(self, ilasm, const_var, name): if not self.const: @@ -373,11 +374,8 @@ self.depends.add(name) name.depends_on.add(self) - def __hash__(self): - return hash(self.const) - - def __eq__(self, other): - return self.const == other.const + def get_key(self): + return self.const def init_fields(self, ilasm, const_var, name): #import pdb;pdb.set_trace() @@ -396,11 +394,8 @@ def get_name(self): return "const_str" - def __hash__(self): - return hash(self.const._str) - - def __eq__(self, other): - return self.const._str == other.const._str + def get_key(self): + return self.const._str def init(self, ilasm): s = self.const._str @@ -416,11 +411,8 @@ def __init__(self, name): self.name = name - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - return self.name == other.name + def get_key(self): + return self.name def get_name(self): return self.name @@ -468,11 +460,8 @@ self.depends = set() self.depends_on = set() - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return hash(self.name) + def get_key(self): + return self.name def get_name(self): return self.const._TYPE._name.split('.')[-1][:-2] From benyoung at codespeak.net Thu Aug 17 10:46:10 2006 From: benyoung at codespeak.net (benyoung at codespeak.net) Date: Thu, 17 Aug 2006 10:46:10 +0200 (CEST) Subject: [pypy-svn] r31358 - pypy/dist/pypy/objspace/std/test Message-ID: <20060817084610.73D3010084@code0.codespeak.net> Author: benyoung Date: Thu Aug 17 10:45:58 2006 New Revision: 31358 Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: More testing for dict impls Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Thu Aug 17 10:45:58 2006 @@ -57,22 +57,28 @@ self.impl = self.get_impl() def get_impl(self): + "Needs to be empty, or one entry with key self.string" return self.ImplementionClass(self.space) def test_setitem(self): assert self.impl.setitem(self.string, 1000) is self.impl + assert self.impl.length() == 1 assert self.impl.get(self.string) == 1000 def test_setitem_str(self): assert self.impl.setitem_str(self.string, 1000) is self.impl + assert self.impl.length() == 1 assert self.impl.get(self.string) == 1000 def test_delitem(self): self.impl.setitem(self.string, 1000) self.impl.setitem(self.string2, 2000) + assert self.impl.length() == 2 newimpl = self.impl.delitem(self.string) + assert self.impl.length() == 1 assert newimpl is self.impl newimpl = self.impl.delitem(self.string2) + assert self.impl.length() == 0 assert isinstance(newimpl, self.EmptyClass) def test_keys(self): From pedronis at codespeak.net Thu Aug 17 11:14:28 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 11:14:28 +0200 (CEST) Subject: [pypy-svn] r31360 - pypy/dist/pypy/interpreter Message-ID: <20060817091428.6D4101008E@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 11:14:27 2006 New Revision: 31360 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: this should have been part of 26659 "setattr doesn't anymore force attribute names to be string in 2.4" Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Thu Aug 17 11:14:27 2006 @@ -386,7 +386,6 @@ return self.int_w(self.hash(w_obj)) def set_str_keyed_item(self, w_obj, w_key, w_value): - w_strkey = self.wrap(self.str_w(w_key)) # Force the key to a space.w_str return self.setitem(w_obj, w_strkey, w_value) def finditem(self, w_obj, w_key): From pedronis at codespeak.net Thu Aug 17 11:16:46 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 11:16:46 +0200 (CEST) Subject: [pypy-svn] r31362 - pypy/dist/pypy/interpreter Message-ID: <20060817091646.E427E1008E@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 11:16:46 2006 New Revision: 31362 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: oops, this code is not exercised at all. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Thu Aug 17 11:16:46 2006 @@ -386,7 +386,7 @@ return self.int_w(self.hash(w_obj)) def set_str_keyed_item(self, w_obj, w_key, w_value): - return self.setitem(w_obj, w_strkey, w_value) + return self.setitem(w_obj, w_key, w_value) def finditem(self, w_obj, w_key): try: From ac at codespeak.net Thu Aug 17 13:38:35 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 17 Aug 2006 13:38:35 +0200 (CEST) Subject: [pypy-svn] r31365 - in pypy/dist/pypy: rpython translator/c/src Message-ID: <20060817113835.32A6D10088@code0.codespeak.net> Author: ac Date: Thu Aug 17 13:38:34 2006 New Revision: 31365 Modified: pypy/dist/pypy/rpython/rcpy.py pypy/dist/pypy/translator/c/src/module.h Log: (samuele, arre) Evil hack to support __x__ methods. Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 13:38:34 2006 @@ -242,15 +242,18 @@ # the llsetup function that will store the 'objects' into the # type's tp_dict + Py_TPFLAGS_HEAPTYPE = CDefinedIntSymbolic('Py_TPFLAGS_HEAPTYPE') if cpytype.objects: objects = [(lltype.pyobjectptr(name), value) - for name, value in cpytype.objects.items()] + for name, value in cpytype.objects.items() if name != '__new__'] def ll_type_setup(p): tp = lltype.cast_pointer(lltype.Ptr(PY_TYPE_OBJECT), p) - tp_dict = tp.c_tp_dict + old_flags = tp.c_tp_flags + tp.c_tp_flags |= Py_TPFLAGS_HEAPTYPE for name, value in objects: - llop.setitem(PyObjPtr, tp_dict, name, value) + llop.setattr(PyObjPtr, tp, name, value) + tp.c_tp_flags = old_flags result._obj.setup_fnptr = rtyper.annotate_helper_fn(ll_type_setup, [PyObjPtr]) Modified: pypy/dist/pypy/translator/c/src/module.h ============================================================================== --- pypy/dist/pypy/translator/c/src/module.h (original) +++ pypy/dist/pypy/translator/c/src/module.h Thu Aug 17 13:38:34 2006 @@ -121,7 +121,7 @@ *def->p = obj; /* store the object ref in the global var */ } /* All objects should be valid at this point. Loop again and - make sure all types are ready, and call the user-defined setups. + make sure all types are ready. */ for (cpydef = cpyheadtable; cpydef->name != NULL; cpydef++) { obj = cpydef->cpyobj; @@ -131,6 +131,11 @@ if (PyType_Ready((PyTypeObject*) obj) < 0) return -1; } + } + /* call the user-defined setups *after* all types are ready + * in case of dependencies */ + for (cpydef = cpyheadtable; cpydef->name != NULL; cpydef++) { + obj = cpydef->cpyobj; if (cpydef->setupfn) { cpydef->setupfn(obj); if (RPyExceptionOccurred()) { From ac at codespeak.net Thu Aug 17 13:40:04 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 17 Aug 2006 13:40:04 +0200 (CEST) Subject: [pypy-svn] r31366 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060817114004.9CE1B10034@code0.codespeak.net> Author: ac Date: Thu Aug 17 13:40:04 2006 New Revision: 31366 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2_file.py Log: Fixing some wrapping issues. Add iteration over BZ2File objects. Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 13:40:04 2006 @@ -227,7 +227,7 @@ if shortread: # if this is EOF, update type flags. - if skipnextlf and (bzerror == BZ_STREAM_END): + if skipnextlf and (bzerror.value == BZ_STREAM_END): newlinetypes |= NEWLINE_CR break @@ -294,7 +294,7 @@ buf_lst.append(ch.value) buf_pos += 1 - if not (bzerror == BZ_OK and ch.value != '\n' and buf_pos != end_pos): + if not (bzerror.value == BZ_OK and ch.value != '\n' and buf_pos != end_pos): break obj.f_newlinetypes = newlinetypes @@ -461,7 +461,7 @@ if bzerror.value != BZ_OK: return _catch_bz2_error(self.space, bzerror.value) - return ret + return self.space.wrap(ret) close.unwrap_spec = ['self'] def tell(self): @@ -687,7 +687,7 @@ self.pos += bufsize if bzerror.value != BZ_OK: - _catch_bz2_error(self.space, bzerror) + _catch_bz2_error(self.space, bzerror.value) write.unwrap_spec = ['self', str] def writelines(self, w_sequence_of_strings): @@ -709,33 +709,36 @@ elif self.f_newlinetypes == NEWLINE_LF: return space.wrap('\n') elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_LF: - return space.wrap(('\r', '\n')) + return space.newtuple([space.wrap('\r'), space.wrap('\n')]) elif self.f_newlinetypes == NEWLINE_CRLF: return space.wrap("\r\n") elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_CRLF: - return space.wrap(('\r', "\r\n")) + return space.newtuple([space.wrap('\r'), space.wrap("\r\n")]) elif self.f_newlinetypes == NEWLINE_LF|NEWLINE_CRLF: - return space.wrap(('\n', "\r\n")) + return space.newtuple([space.wrap('\n'), space.wrap("\r\n")]) elif self.f_newlinetypes == NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF: - return space.wrap(('\r', '\n', "\r\n")) + return space.newtuple([space.wrap('\r'), space.wrap('\n'), + space.wrap("\r\n")]) else: raise OperationError(space.w_SystemError, space.wrap( - "Unknown newlines value 0x%d\n" % hex(self.f_newlinetypes))) + "Unknown newlines value 0x%s\n" % hex(self.f_newlinetypes))) def fget_closed(space, self): return space.wrap(self.mode == MODE_CLOSED) - # XXX: I have to hack-in this UGLY thing because there's no support of - # special methods outside so I can't use the iterator protocol (no __iter__) - # next() method is not implemented, it's useless right now - # XXX no 2: unwrap() is the evil itself! XXX - def get_iterator(self): - w_lines = self.readlines() - lines = self.space.unwrap(w_lines) - return self.space.wrap(iter(lines)) - get_iterator.unwrap_spec = ['self'] - + def next(self): + space = self.space + w_line = self.readline() + if space.int_w(space.len(w_line)) == 0: # EOF + raise OperationError(space.w_StopIteration, space.w_None) + return w_line + next.unwrap_spec = ['self'] + + def __iter__(self): + return self.space.wrap(self) + __iter__.unwrap_spec = ['self'] + def xreadlines(self): """xreadlines() -> self @@ -743,7 +746,7 @@ optimizations previously implemented in the xreadlines module.""" # this method should use the iterator protocol one day... - return self.get_iterator() + return self.space.wrap(self) xreadlines.unwrap_spec = ['self'] @@ -758,8 +761,8 @@ readlines = interp2app(_BZ2File.readlines, unwrap_spec=_BZ2File.readlines.unwrap_spec), read = interp2app(_BZ2File.read, unwrap_spec=_BZ2File.read.unwrap_spec), - get_iterator = interp2app(_BZ2File.get_iterator, - unwrap_spec=_BZ2File.get_iterator.unwrap_spec), + __iter__ = interp2app(_BZ2File.__iter__), + next = interp2app(_BZ2File.next), xreadlines = interp2app(_BZ2File.xreadlines, unwrap_spec=_BZ2File.xreadlines.unwrap_spec), write = interp2app(_BZ2File.write, unwrap_spec=_BZ2File.write.unwrap_spec), @@ -1126,7 +1129,7 @@ compress() function instead. The compresslevel parameter, if given, must be a number between 1 and 9.""" - return _BZ2Comp(space, compresslevel) + return space.wrap(_BZ2Comp(space, compresslevel)) BZ2Compressor.unwrap_spec = [ObjSpace, int] def BZ2Decompressor(space): @@ -1136,7 +1139,7 @@ data sequentially. If you want to decompress data in one shot, use the decompress() function instead.""" - return _BZ2Decomp(space) + return space.wrap(_BZ2Decomp(space)) BZ2Decompressor.unwrap_spec = [ObjSpace] def BZ2File(space, filename, mode='r', buffering=-1, compresslevel=9): @@ -1155,6 +1158,6 @@ '\\r\\n' or a tuple containing all the newline types seen. Universal newlines are available only when reading.""" - return _BZ2File(space, filename, mode, buffering, compresslevel) + return space.wrap(_BZ2File(space, filename, mode, buffering, compresslevel)) BZ2File.unwrap_spec = [ObjSpace, str, str, int, int] Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py Thu Aug 17 13:40:04 2006 @@ -288,7 +288,7 @@ bz2f = BZ2File("foo") sio = StringIO(self.TEXT) - assert list(bz2f.get_iterator()) == sio.readlines() + assert list(bz2f.__iter__()) == sio.readlines() bz2f.close() def test_xreadlines(self): From ac at codespeak.net Thu Aug 17 15:39:21 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 17 Aug 2006 15:39:21 +0200 (CEST) Subject: [pypy-svn] r31367 - in pypy/dist/pypy: objspace/cpy/test rpython Message-ID: <20060817133921.7375710083@code0.codespeak.net> Author: ac Date: Thu Aug 17 15:39:20 2006 New Revision: 31367 Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py pypy/dist/pypy/rpython/rcpy.py Log: Enable special methods for the CPython abstract protocols. Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_typedef.py Thu Aug 17 15:39:20 2006 @@ -27,7 +27,6 @@ def fset_x(space, self, w_value): self.x = space.int_w(w_value) - def test_direct(): W_MyType.typedef = TypeDef("MyType") space = CPyObjSpace() @@ -148,6 +147,21 @@ assert type(res).__name__ == 'MyType' assert res.multiply(3) == 369 +def test_special_method(): + W_MyType.typedef = TypeDef("MyType", + __mul__ = interp2app(W_MyType.multiply)) + space = CPyObjSpace() + assert space.int_w(W_MyType(space, 6).multiply(space.wrap(7))) == 42 + + def make_mytype(): + return space.wrap(W_MyType(space, 123)) + fn = compile(make_mytype, [], + annotatorpolicy = CPyAnnotatorPolicy(space)) + + res = fn(expected_extra_mallocs=1) + assert type(res).__name__ == 'MyType' + assert res * 3 == 369 + def test_interp_attrproperty(): W_MyType.typedef = TypeDef("MyType", Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 15:39:20 2006 @@ -127,7 +127,15 @@ PyObjPtr = lltype.Ptr(lltype.PyObject) - +PyNumberMethods = lltype.Struct('PyNumberMethods', + ('data', lltype.FixedSizeArray(lltype.Signed, 38)) +) +PyMappingMethods = lltype.Struct('PyMappingMethods', + ('data', lltype.FixedSizeArray(lltype.Signed, 3)) +) +PySequenceMethods = lltype.Struct('PySequenceMethods', + ('data', lltype.FixedSizeArray(lltype.Signed, 10)) +) PY_TYPE_OBJECT = lltype.PyForwardReference() PY_TYPE_OBJECT.become(lltype.PyStruct( 'PyTypeObject', @@ -143,9 +151,9 @@ ('c_tp_setattr', lltype.Signed), # in ('c_tp_compare', lltype.Signed), ('c_tp_repr', lltype.Signed), # progress - ('c_tp_as_number', lltype.Signed), - ('c_tp_as_sequence',lltype.Signed), - ('c_tp_as_mapping',lltype.Signed), + ('c_tp_as_number', lltype.Ptr(PyNumberMethods)), + ('c_tp_as_sequence',lltype.Ptr(PySequenceMethods)), + ('c_tp_as_mapping',lltype.Ptr(PyMappingMethods)), ('c_tp_hash', lltype.Signed), ('c_tp_call', lltype.Signed), ('c_tp_str', lltype.Signed), @@ -234,10 +242,14 @@ p[len(name)] = '\x00' pytypeobj.c_tp_name = lltype.direct_arrayitems(p) pytypeobj.c_tp_basicsize = llmemory.sizeof(r_inst.lowleveltype.TO) - pytypeobj.c_tp_flags = CDefinedIntSymbolic('Py_TPFLAGS_DEFAULT') + pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_CHECKTYPES)''') pytypeobj.c_tp_new = rtyper.type_system.getcallable(tp_new_graph) pytypeobj.c_tp_dealloc = rtyper.annotate_helper_fn(ll_tp_dealloc, [PyObjPtr]) + pytypeobj.c_tp_as_number = lltype.malloc(PyNumberMethods, immortal=True) + pytypeobj.c_tp_as_sequence = lltype.malloc(PySequenceMethods, immortal=True) + pytypeobj.c_tp_as_mapping = lltype.malloc(PyMappingMethods, immortal=True) result = lltype.cast_pointer(PyObjPtr, pytypeobj) # the llsetup function that will store the 'objects' into the From pedronis at codespeak.net Thu Aug 17 15:40:50 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 15:40:50 +0200 (CEST) Subject: [pypy-svn] r31368 - in pypy/dist/pypy: rpython/memory translator translator/backendopt/test translator/c translator/stackless translator/stackless/test Message-ID: <20060817134050.00F0210083@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 15:40:48 2006 New Revision: 31368 Modified: pypy/dist/pypy/rpython/memory/gctransform.py pypy/dist/pypy/translator/backendopt/test/test_support.py pypy/dist/pypy/translator/c/exceptiontransform.py pypy/dist/pypy/translator/stackless/test/test_transform.py pypy/dist/pypy/translator/stackless/transform.py pypy/dist/pypy/translator/unsimplify.py Log: one common version of varoftype Modified: pypy/dist/pypy/rpython/memory/gctransform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform.py Thu Aug 17 15:40:48 2006 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem.lloperation import llop, LL_OPERATIONS from pypy.objspace.flow.model import SpaceOperation, Variable, Constant, \ c_last_exception, FunctionGraph, Block, Link, checkgraph +from pypy.translator.unsimplify import varoftype from pypy.translator.unsimplify import insert_empty_block from pypy.translator.unsimplify import insert_empty_startblock from pypy.translator.unsimplify import starts_with_empty_block @@ -649,11 +650,6 @@ v_addr], varoftype(lltype.Void))] -def varoftype(concretetype): - var = Variable() - var.concretetype = concretetype - return var - def find_gc_ptrs_in_type(TYPE): if isinstance(TYPE, lltype.Array): return find_gc_ptrs_in_type(TYPE.OF) Modified: pypy/dist/pypy/translator/backendopt/test/test_support.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_support.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_support.py Thu Aug 17 15:40:48 2006 @@ -1,3 +1,4 @@ +from pypy.translator.unsimplify import varoftype from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.support import \ needs_conservative_livevar_calculation, split_block_with_keepalive, \ @@ -10,11 +11,6 @@ NonGcB = lltype.Struct("B", ('x', lltype.Signed)) GcA = lltype.GcStruct("A", ('b', NonGcB), ('c', lltype.Ptr(lltype.FuncType([], lltype.Void)))) -def varoftype(concretetype): - var = model.Variable() - var.concretetype = concretetype - return var - def test_nclc_should_be_true(): # this is testing a block like: # +--- inputargs: pointer_to_gc Modified: pypy/dist/pypy/translator/c/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/c/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/c/exceptiontransform.py Thu Aug 17 15:40:48 2006 @@ -1,12 +1,11 @@ from pypy.translator.simplify import join_blocks, cleanup_graph -from pypy.translator.unsimplify import copyvar +from pypy.translator.unsimplify import copyvar, varoftype from pypy.translator.backendopt import canraise, inline, support, removenoops from pypy.objspace.flow.model import Block, Constant, Variable, Link, \ c_last_exception, SpaceOperation, checkgraph, FunctionGraph from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem import lloperation from pypy.rpython.memory.lladdress import NULL -from pypy.rpython.memory.gctransform import varoftype from pypy.rpython import rtyper from pypy.rpython import rclass from pypy.rpython.rarithmetic import r_uint, r_longlong, r_ulonglong Modified: pypy/dist/pypy/translator/stackless/test/test_transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/test/test_transform.py (original) +++ pypy/dist/pypy/translator/stackless/test/test_transform.py Thu Aug 17 15:40:48 2006 @@ -3,7 +3,7 @@ from pypy.translator.stackless.transform import StacklessTransformer, FrameTyper from pypy.translator.c.genc import CStandaloneBuilder from pypy.translator.c import gc -from pypy.rpython.memory.gctransform import varoftype +from pypy.translator.unsimplify import varoftype from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import llinterp, rstack from pypy.translator.translator import TranslationContext, graphof Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Thu Aug 17 15:40:48 2006 @@ -3,8 +3,8 @@ from pypy.rpython import rarithmetic, rclass, rmodel from pypy.translator.backendopt import support from pypy.objspace.flow import model -from pypy.rpython.memory.gctransform import varoftype from pypy.translator import unsimplify, simplify +from pypy.translator.unsimplify import varoftype from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.translator.stackless import code, frame Modified: pypy/dist/pypy/translator/unsimplify.py ============================================================================== --- pypy/dist/pypy/translator/unsimplify.py (original) +++ pypy/dist/pypy/translator/unsimplify.py Thu Aug 17 15:40:48 2006 @@ -10,6 +10,11 @@ newvar.concretetype = v.concretetype return newvar +def varoftype(concretetype, name=None): + var = Variable(name) + var.concretetype = concretetype + return var + def insert_empty_block(annotator, link, newops=[]): """Insert and return a new block along the given link.""" vars = {} From auc at codespeak.net Thu Aug 17 15:46:48 2006 From: auc at codespeak.net (auc at codespeak.net) Date: Thu, 17 Aug 2006 15:46:48 +0200 (CEST) Subject: [pypy-svn] r31369 - pypy/dist/pypy/module/_stackless Message-ID: <20060817134648.A338510083@code0.codespeak.net> Author: auc Date: Thu Aug 17 15:46:47 2006 New Revision: 31369 Modified: pypy/dist/pypy/module/_stackless/clonable.py Log: kill crap Modified: pypy/dist/pypy/module/_stackless/clonable.py ============================================================================== --- pypy/dist/pypy/module/_stackless/clonable.py (original) +++ pypy/dist/pypy/module/_stackless/clonable.py Thu Aug 17 15:46:47 2006 @@ -48,22 +48,6 @@ ec = self.space.getexecutioncontext() ec.subcontext_leave(self) - def w_clone(self): - if not we_are_translated(): - raise NotImplementedError - if ClonableCoroutine.w_getcurrent(self.space) is self: - raise RuntimeError("clone() cannot clone the current coroutine; " - "use fork() instead") - if self.local_pool is None: # force it now - self.local_pool = gc_swap_pool(gc_swap_pool(None)) - # cannot gc_clone() directly self, because it is not in its own - # local_pool. Moreover, it has a __del__, which cloning doesn't - # support properly at the moment. - copy = ClonableCoroutine(self.space, costate=self.costate) - copy.parent = self.parent - copy.frame, copy.local_pool = gc_clone(self.frame, self.local_pool) - return copy - def w_getcurrent(space): return space.wrap(ClonableCoroutine._get_state(space).current) @@ -243,7 +227,6 @@ __new__ = interp2app(ClonableCoroutine.descr_method__new__.im_func), _framestack = GetSetProperty(w_descr__framestack), getcurrent = interp2app(ClonableCoroutine.w_getcurrent), - clone = interp2app(ClonableCoroutine.w_clone), __reduce__ = interp2app(ClonableCoroutine.descr__reduce__, unwrap_spec=['self', ObjSpace]), __setstate__ = interp2app(ClonableCoroutine.descr__setstate__, From rhymes at codespeak.net Thu Aug 17 16:06:02 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 16:06:02 +0200 (CEST) Subject: [pypy-svn] r31370 - pypy/dist/pypy/module/bz2 Message-ID: <20060817140602.2552910083@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 16:06:00 2006 New Revision: 31370 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: fix some compilation issues Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 16:06:00 2006 @@ -117,7 +117,7 @@ libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE), c_int, POINTER(c_uint), POINTER(c_uint)] libbz2.BZ2_bzWriteClose.restype = c_void -libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] +libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), POINTER(c_char), c_int] libbz2.BZ2_bzRead.restype = c_int libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int] libbz2.BZ2_bzWrite.restype = c_void @@ -183,6 +183,8 @@ newlinetypes = obj.f_newlinetypes skipnextlf = obj.f_skipnextlf + dst_lst = [] + src_pos = dst_pos = 0 while n: src = dst @@ -191,9 +193,9 @@ shortread = n != 0 # True iff EOF or error # needed to operate with "pointers" - src_lst = list(src.value) + src_lst = [c for c in src.value] src_pos = 0 - dst_lst = list(dst.value) + dst_lst = [c for c in dst.value] dst_pos = 0 while nread: nread -= 1 @@ -236,8 +238,8 @@ data = "".join(dst_lst) buf = create_string_buffer(len(data)) - for i, ch in enumerate(data): - buf[i] = ch + for i in range(len(data)): + buf[i] = data[i] return dst_pos, buf @@ -637,9 +639,12 @@ else: break - buf_lst = list(buf.value) + buf_lst = [c for c in buf.value] if bytesread != bufsize: - return self.space.wrap("".join(buf_lst[:bytesread])) + start = 0 + assert bytesread >= 0 + assert start >= 0 + return self.space.wrap("".join(buf_lst[start:bytesread])) return self.space.wrap("".join(buf_lst)) read.unwrap_spec = ['self', int] @@ -703,7 +708,7 @@ # accessors for properties def fget_newlines(space, self): if self.f_newlinetypes == NEWLINE_UNKNOWN: - return space.wrap(None) + return space.w_None elif self.f_newlinetypes == NEWLINE_CR: return space.wrap('\r') elif self.f_newlinetypes == NEWLINE_LF: From rhymes at codespeak.net Thu Aug 17 16:08:46 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 16:08:46 +0200 (CEST) Subject: [pypy-svn] r31371 - in pypy/dist/pypy/module/bz2: . test Message-ID: <20060817140846.1DD6A10084@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 16:08:44 2006 New Revision: 31371 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py pypy/dist/pypy/module/bz2/test/test_bz2_file.py Log: use iter() Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 16:08:44 2006 @@ -749,9 +749,8 @@ For backward compatibility. BZ2File objects now include the performance optimizations previously implemented in the xreadlines module.""" - - # this method should use the iterator protocol one day... - return self.space.wrap(self) + + return self.__iter__() xreadlines.unwrap_spec = ['self'] Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/dist/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py Thu Aug 17 16:08:44 2006 @@ -281,16 +281,15 @@ bz2f.close() def test_iterator(self): - # XXX: to fix when the actual iterator protocol will be available from bz2 import BZ2File from cStringIO import StringIO self.create_temp_file() bz2f = BZ2File("foo") sio = StringIO(self.TEXT) - assert list(bz2f.__iter__()) == sio.readlines() + assert list(iter(bz2f)) == sio.readlines() bz2f.close() - + def test_xreadlines(self): from bz2 import BZ2File from cStringIO import StringIO From pedronis at codespeak.net Thu Aug 17 16:33:11 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 16:33:11 +0200 (CEST) Subject: [pypy-svn] r31372 - in pypy/dist/pypy: objspace/cpy/test rpython Message-ID: <20060817143311.9AC5710083@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 16:33:09 2006 New Revision: 31372 Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py pypy/dist/pypy/rpython/rcpy.py Log: (arre, pedronis) use a __new__ in ext-compiled types if it is there Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_typedef.py Thu Aug 17 16:33:09 2006 @@ -273,7 +273,8 @@ mytype_new.unwrap_spec = [ObjSpace, W_Root, int] W_MyType.typedef = TypeDef("MyType", - __new__ = interp2app(mytype_new)) + __new__ = interp2app(mytype_new), + x = interp_attrproperty("x", W_MyType)) space = CPyObjSpace() def build(): @@ -283,11 +284,13 @@ w_obj = build() w_name = space.getattr(space.type(w_obj), space.wrap('__name__')) assert space.unwrap(w_name) == 'MyType' + assert space.int_w(space.getattr(w_obj, space.wrap('x'))) == 42 fn = compile(build, [], annotatorpolicy = CPyAnnotatorPolicy(space)) res = fn(expected_extra_mallocs=1) assert type(res).__name__ == 'MyType' + assert res.x == 42 def test_prebuilt_type(): def mytype_new(space, w_subtype, x): Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 16:33:09 2006 @@ -258,6 +258,10 @@ if cpytype.objects: objects = [(lltype.pyobjectptr(name), value) for name, value in cpytype.objects.items() if name != '__new__'] + if '__new__' in cpytype.objects: + new = cpytype.objects['__new__']._obj.value + objects.append((lltype.pyobjectptr('__new__'), + lltype.pyobjectptr(staticmethod(new)))) def ll_type_setup(p): tp = lltype.cast_pointer(lltype.Ptr(PY_TYPE_OBJECT), p) From bea at codespeak.net Thu Aug 17 17:04:15 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 17 Aug 2006 17:04:15 +0200 (CEST) Subject: [pypy-svn] r31374 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060817150415.186E11007F@code0.codespeak.net> Author: bea Date: Thu Aug 17 17:04:13 2006 New Revision: 31374 Modified: pypy/extradoc/sprintinfo/ireland-2006/announce.txt Log: updated with changes regarding workshop and tutorial timing Modified: pypy/extradoc/sprintinfo/ireland-2006/announce.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/announce.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/announce.txt Thu Aug 17 17:04:13 2006 @@ -22,25 +22,31 @@ .. _`Summer of PyPy`: http://codespeak.net/pypy/dist/pypy/doc/summer-of-pypy -First day: introduction and workshop (possible to attend only this day) +Workshop parallell to the first day of the sprint, 21st of August ------------------------------------------------------------------------ -During the first day (21st of August) there will be talks on various subjects -related to PyPy: +During the first day (21st of August) there will be in parallell to the sprint: -* A tutorial and technical introduction to the PyPy codebase - (suited for people interested in getting an overview of PyPy?s architecture - and/or contributing to PyPy) - -* a workshop covering more in-depth technical aspects of PyPy and what PyPy - can do for you. The workshop will also cover methodology, aiming at explaining - the pros and cons of sprint-driven development. +* a workshop covering more in-depth methodology aspects of PyPy + aiming at explaining the pros and cons of sprint-driven development. (suited for sprint attendants, students, staff and other interested parties from/around the University and the local area) -The tutorial will be part of the sprint introduction - the workshop will take place if -there is enough interest raised before the 21st of August from people planning to -attend. You are of course welcome to attend just for this first day of the sprint. +The workshop participants (mostly University of Limerick researchers) will +observe the sprint during parts of the first day, film an take pictures as well. + + +Tutorial, PyPy introduction and dinner on friday, 25th of August +--------------------------------------------------------------------------- + +During friday we will arrange a tutorial and technical +introduction to the PyPy codebase, followed by a Q&A session. This session was originally +planned for the first day but we moved it to friday afternoon in order to +make it easier for people from the irish python group to participate. +This tutorial session starts at 5pm. At 8pm we go for a nice dinner together. + +This friday afternoon tutorial session and dinner is open for anyone interested, +you do not have to be a sprint participant to join us for this event. If you want to come ... From hpk at codespeak.net Thu Aug 17 17:41:10 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 17 Aug 2006 17:41:10 +0200 (CEST) Subject: [pypy-svn] r31375 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060817154110.4B15110083@code0.codespeak.net> Author: hpk Date: Thu Aug 17 17:41:08 2006 New Revision: 31375 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: i re-scheduled my flights and attendance for personal reasons, aim to share a room with bea sunday-friday. Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Thu Aug 17 17:41:08 2006 @@ -12,9 +12,9 @@ Samuele Pedroni 20-28/8 on-campus Michael Hudson 20-27 Brookfield Hall, Castletroy Armin Rigo 21-28 ? -Holger Krekel 18-28 v/dublin on-campus (faxed form) +Holger Krekel 20-25/8 aiming for brookfield Maciej Fijalkowski 21-28?/8 Brookfield Hall, Castletroy -Bea During 20-25/8 ? +Bea During 20-25/8 aiming for brookfield ==================== ============== ======================== People on the following list were present at previous sprints: From pedronis at codespeak.net Thu Aug 17 18:11:16 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 18:11:16 +0200 (CEST) Subject: [pypy-svn] r31376 - in pypy/dist/pypy: objspace/cpy objspace/cpy/test rpython Message-ID: <20060817161116.A6D3910080@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 18:11:14 2006 New Revision: 31376 Modified: pypy/dist/pypy/objspace/cpy/objspace.py pypy/dist/pypy/objspace/cpy/test/test_typedef.py pypy/dist/pypy/objspace/cpy/typedef.py pypy/dist/pypy/rpython/rcpy.py Log: (arre, pedronis) support in the ext-compiler __new__ using allocate_instance. Subclassing doesn't work properly because deallocation is wrongly done resulting in a segfault for now. Modified: pypy/dist/pypy/objspace/cpy/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/objspace.py (original) +++ pypy/dist/pypy/objspace/cpy/objspace.py Thu Aug 17 18:11:14 2006 @@ -8,7 +8,7 @@ from pypy.interpreter.function import Function from pypy.interpreter.typedef import GetSetProperty from pypy.rpython.rarithmetic import r_uint, r_longlong, r_ulonglong -from pypy.rpython.objectmodel import we_are_translated +from pypy.rpython.objectmodel import we_are_translated, instantiate class CPyObjSpace(baseobjspace.ObjSpace): @@ -115,6 +115,16 @@ else: return None + def allocate_instance(self, cls, w_subtype): + if we_are_translated(): + from pypy.objspace.cpy.typedef import cpython_allocate + obj = cpython_allocate(cls, w_subtype) + else: + assert self.is_w(self.gettypefor(cls), w_subtype) + obj = object.__new__(cls) + return self.wrap(obj) + + # __________ operations with a direct CPython equivalent __________ getattr = staticmethod(PyObject_GetAttr) Modified: pypy/dist/pypy/objspace/cpy/test/test_typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/test/test_typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/test/test_typedef.py Thu Aug 17 18:11:14 2006 @@ -292,6 +292,67 @@ assert type(res).__name__ == 'MyType' assert res.x == 42 +def test_with_new_with_allocate_instance(): + def mytype_new(space, w_subtype, x): + w_obj = space.allocate_instance(W_MyType, w_subtype) + W_MyType.__init__(space.interp_w(W_MyType, w_obj), space, x) + return w_obj + mytype_new.unwrap_spec = [ObjSpace, W_Root, int] + + W_MyType.typedef = TypeDef("MyType", + __new__ = interp2app(mytype_new), + x = interp_attrproperty("x", W_MyType)) + space = CPyObjSpace() + + def build(): + w_type = space.gettypefor(W_MyType) + return space.call_function(w_type, space.wrap(42)) + + w_obj = build() + w_name = space.getattr(space.type(w_obj), space.wrap('__name__')) + assert space.unwrap(w_name) == 'MyType' + assert space.int_w(space.getattr(w_obj, space.wrap('x'))) == 42 + + fn = compile(build, [], + annotatorpolicy = CPyAnnotatorPolicy(space)) + res = fn(expected_extra_mallocs=1) + assert type(res).__name__ == 'MyType' + assert res.x == 42 + +def test_with_new_with_allocate_instance_subclass(): + py.test.skip("dealloction for now segfaults") + def mytype_new(space, w_subtype, x): + w_obj = space.allocate_instance(W_MyType, w_subtype) + W_MyType.__init__(space.interp_w(W_MyType, w_obj), space, x) + return w_obj + mytype_new.unwrap_spec = [ObjSpace, W_Root, int] + + W_MyType.typedef = TypeDef("MyType", + __new__ = interp2app(mytype_new), + x = interp_attrproperty("x", W_MyType)) + space = CPyObjSpace() + + def build(): + w_type = space.gettypefor(W_MyType) + return space.call_function(w_type, space.wrap(42)) + + fn = compile(build, [], + annotatorpolicy = CPyAnnotatorPolicy(space)) + res = fn(expected_extra_mallocs=1) + assert type(res).__name__ == 'MyType' + assert res.x == 42 + + class MyType2(type(res)): + pass + + res2 = MyType2(42) + assert type(res2) is MyType2 + assert res2.x == 42 + + del res2 + import gc + gc.collect() + def test_prebuilt_type(): def mytype_new(space, w_subtype, x): return space.wrap(W_MyType(space, x)) Modified: pypy/dist/pypy/objspace/cpy/typedef.py ============================================================================== --- pypy/dist/pypy/objspace/cpy/typedef.py (original) +++ pypy/dist/pypy/objspace/cpy/typedef.py Thu Aug 17 18:11:14 2006 @@ -10,7 +10,7 @@ from pypy.interpreter.typedef import GetSetProperty from pypy.rpython.objectmodel import we_are_translated from pypy.rpython.rcpy import CPyTypeInterface, cpy_export, cpy_import -from pypy.rpython.rcpy import cpy_typeobject, rpython_object +from pypy.rpython.rcpy import cpy_typeobject, rpython_object, cpy_allocate from pypy.rpython.rcpy import init_rpython_data, get_rpython_data from pypy.rpython.lltypesystem import lltype @@ -76,6 +76,11 @@ cpython2rpython._annspecialcase_ = 'specialize:arg(1)' cpython2rpython.allow_someobjects = True +def cpython_allocate(cls, w_subtype): + return cpy_allocate(cls, w_subtype.value) +cpython_allocate._annspecialcase_ = 'specialize:arg(0)' +cpython_allocate.allow_someobjects = True + # ____________________________________________________________ class TypeDefCache(SpaceCache): @@ -97,7 +102,7 @@ # typedef.name, name)) w_value = space.wrap(value) objects[name] = lltype.pyobjectptr(w_value.value) - typeintf = CPyTypeInterface(typedef.name, objects) + typeintf = CPyTypeInterface(typedef.name, objects, typedef.acceptable_as_base_class) return typeintf def wraptypeintf(cache, cls, typeintf): Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 18:11:14 2006 @@ -8,7 +8,7 @@ class CPyTypeInterface(object): - def __init__(self, name, objects={}): + def __init__(self, name, objects={}, subclassable=False): # the exported name of the type self.name = name @@ -16,6 +16,7 @@ # a dict {name: pyobjectptr()} for general class attributes # (not for special methods!) self.objects = objects.copy() + self.subclassable = subclassable def _freeze_(self): return True @@ -41,6 +42,9 @@ def cpy_typeobject(cpytype, cls): raise NotImplementedError("only works in translated versions") +def cpy_allocate(cls, cpytype): + raise NotImplementedError("only works in translated versions") + # ____________________________________________________________ # Implementation @@ -115,6 +119,26 @@ cpytype = build_pytypeobject(r_inst) return hop.inputconst(PyObjPtr, cpytype) +class Entry(ExtRegistryEntry): + _about_ = cpy_allocate + + def compute_result_annotation(self, s_cls, s_cpytype): + from pypy.annotation.model import SomeObject, SomeInstance + assert s_cls.is_constant() + [classdesc] = s_cls.descriptions + classdef = classdesc.getuniqueclassdef() + return SomeInstance(classdef) + + def specialize_call(self, hop): + from pypy.rpython.rclass import getinstancerepr + s_cls = hop.args_s[0] + assert s_cls.is_constant() + [classdesc] = s_cls.descriptions + classdef = classdesc.getuniqueclassdef() + r_inst = getinstancerepr(hop.rtyper, classdef) + vinst = r_inst.new_instance(hop.llops, v_cpytype = hop.args_v[1]) + return vinst + def attach_cpy_flavor(classdef, cpytype): for parentdef in classdef.getmro(): @@ -218,6 +242,7 @@ from pypy.rpython.rtyper import LowLevelOpList typetype = lltype.pyobjectptr(type) + # XXX default tp_new should go away # make the graph of tp_new manually v1 = Variable('tp'); v1.concretetype = lltype.Ptr(PY_TYPE_OBJECT) v2 = Variable('args'); v2.concretetype = PyObjPtr @@ -242,8 +267,12 @@ p[len(name)] = '\x00' pytypeobj.c_tp_name = lltype.direct_arrayitems(p) pytypeobj.c_tp_basicsize = llmemory.sizeof(r_inst.lowleveltype.TO) - pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_CHECKTYPES)''') + if cpytype.subclassablei and False: # XXX deallocation of subclass object segfaults! + pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_BASETYPE)''') + else: + pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_CHECKTYPES)''') pytypeobj.c_tp_new = rtyper.type_system.getcallable(tp_new_graph) pytypeobj.c_tp_dealloc = rtyper.annotate_helper_fn(ll_tp_dealloc, [PyObjPtr]) From pedronis at codespeak.net Thu Aug 17 18:39:24 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 18:39:24 +0200 (CEST) Subject: [pypy-svn] r31377 - pypy/dist/pypy/jit/timeshifter Message-ID: <20060817163924.74B7C10080@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 18:39:22 2006 New Revision: 31377 Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py Log: (pedronis, arre) small refactoring: use varoftype and copyvar as necessary Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Thu Aug 17 18:39:22 2006 @@ -10,6 +10,7 @@ from pypy.jit.timeshifter import rtimeshift from pypy.jit.timeshifter.rtyper import HintRTyper, originalconcretetype from pypy.jit.timeshifter.rtyper import GreenRepr, RedRepr, HintLowLevelOpList +from pypy.translator.unsimplify import varoftype, copyvar from pypy.translator.backendopt import support # ___________________________________________________________ @@ -91,10 +92,8 @@ def getexitindex(self, link, inputargs, args_r, entering_links): self.latestexitindex += 1 - v_jitstate = flowmodel.Variable('jitstate') - v_jitstate.concretetype = self.r_JITState.lowleveltype - v_boxes = flowmodel.Variable('boxes') - v_boxes.concretetype = self.r_box_accum.lowleveltype + v_jitstate = varoftype(self.r_JITState.lowleveltype, 'jitstate') + v_boxes = varoftype(self.r_box_accum.lowleveltype, 'boxes') reentry_block = flowmodel.Block([v_jitstate, v_boxes]) @@ -212,8 +211,7 @@ def insert_start_setup(self): newstartblock = self.insert_before_block(self.graph.startblock, None, closeblock=True) - v_builder = flowmodel.Variable('builder') - v_builder.concretetype = self.r_ResidualGraphBuilder.lowleveltype + v_builder = varoftype(self.r_ResidualGraphBuilder.lowleveltype, 'builder') v_jitstate = newstartblock.inputargs[0] newstartblock.inputargs[0] = v_builder llops = HintLowLevelOpList(self, None) @@ -238,15 +236,8 @@ # assert False, "the return block should not be seen" def insert_before_block(self, block, entering_links, closeblock=True): - newinputargs = [] - for var in block.inputargs: - newvar = flowmodel.Variable(var) - newvar.concretetype = var.concretetype - try: - self.hannotator.bindings[newvar] = hs = self.hannotator.bindings[var] - except KeyError: - pass - newinputargs.append(newvar) + newinputargs = [copyvar(self.hannotator, var) for var in block.inputargs] + newblock = flowmodel.Block(newinputargs) if block.isstartblock: # xxx block.isstartblock = False @@ -397,18 +388,12 @@ # now read out the possibly modified red boxes out of v_boxes - v_newjitstate2 = flowmodel.Variable(v_newjitstate) - v_newjitstate2.concretetype = self.r_JITState.lowleveltype - v_boxes2 = flowmodel.Variable(v_boxes) - v_boxes2.concretetype = self.r_box_list.lowleveltype - - + v_newjitstate2 = varoftype(self.r_JITState.lowleveltype, v_newjitstate) + v_boxes2 = varoftype(self.r_box_list.lowleveltype, v_boxes) read_boxes_block_vars = [v_newjitstate2, v_boxes2] for greenvar in orig_key_v: - greenvar2 = flowmodel.Variable(greenvar) - greenvar2.concretetype = greenvar.concretetype - read_boxes_block_vars.append(greenvar2) + read_boxes_block_vars.append(copyvar(None, greenvar)) read_boxes_block = flowmodel.Block(read_boxes_block_vars) to_read_boxes_block = flowmodel.Link([v_newjitstate, v_boxes] + orig_key_v, read_boxes_block) @@ -462,12 +447,7 @@ def introduce(v): if isinstance(v, flowmodel.Variable): if v not in renamemap: - vprime = renamemap[v] = flowmodel.Variable(v) - try: - self.hannotator.bindings[vprime] = self.hannotator.bindings[v] - except KeyError: - pass - vprime.concretetype = v.concretetype + vprime = renamemap[v] = copyvar(self.hannotator, v) inargs.append(v) orig_v_jitstate = block.inputargs[0] @@ -475,9 +455,8 @@ newlinks = [] - v_newjitstate = flowmodel.Variable('jitstate') + v_newjitstate = varoftype(self.r_JITState.lowleveltype, 'jitstate') self.hannotator.bindings[v_newjitstate] = self.s_JITState - v_newjitstate.concretetype = self.r_JITState.lowleveltype def rename_on_link(v): if v is orig_v_jitstate: @@ -562,8 +541,7 @@ llops = HintLowLevelOpList(self, None) - v_returnbuilder = flowmodel.Variable('builder') - v_returnbuilder.concretetype = self.r_ResidualGraphBuilder.lowleveltype + v_returnbuilder = varoftype(self.r_ResidualGraphBuilder.lowleveltype, 'builder') returnblock = flowmodel.Block([v_returnbuilder]) returnblock.operations = () self.graph.returnblock = returnblock @@ -578,8 +556,7 @@ dispatchblock.operations = list(llops) dispatch_to = self.dispatch_to - v_jitstate2 = flowmodel.Variable('jitstate') - v_jitstate2.concretetype = self.r_JITState.lowleveltype + v_jitstate2 = varoftype(self.r_JITState.lowleveltype, 'jitstate') prepare_return_block = flowmodel.Block([v_jitstate2]) prepare_return_link = flowmodel.Link([v_jitstate], prepare_return_block) dispatch_to.append(('default', prepare_return_link)) @@ -616,8 +593,7 @@ def getjitstate(self, block): if block not in self.block2jitstate: - v_jitstate = flowmodel.Variable('jitstate') - v_jitstate.concretetype = self.r_JITState.lowleveltype + v_jitstate = varoftype(self.r_JITState.lowleveltype, 'jitstate') self.block2jitstate[block] = v_jitstate return self.block2jitstate[block] From rhymes at codespeak.net Thu Aug 17 18:48:16 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 18:48:16 +0200 (CEST) Subject: [pypy-svn] r31378 - pypy/dist/pypy/rpython Message-ID: <20060817164816.BA9C910080@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 18:48:14 2006 New Revision: 31378 Modified: pypy/dist/pypy/rpython/rcpy.py Log: fix typo Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 18:48:14 2006 @@ -267,7 +267,7 @@ p[len(name)] = '\x00' pytypeobj.c_tp_name = lltype.direct_arrayitems(p) pytypeobj.c_tp_basicsize = llmemory.sizeof(r_inst.lowleveltype.TO) - if cpytype.subclassablei and False: # XXX deallocation of subclass object segfaults! + if cpytype.subclassable and False: # XXX deallocation of subclass object segfaults! pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_BASETYPE)''') else: From rhymes at codespeak.net Thu Aug 17 18:54:31 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 18:54:31 +0200 (CEST) Subject: [pypy-svn] r31379 - pypy/dist/pypy/module/bz2 Message-ID: <20060817165431.D783D10080@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 18:54:27 2006 New Revision: 31379 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: more fixes towards compilation Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 18:54:27 2006 @@ -106,8 +106,11 @@ pythonapi.PyMem_Free.argtypes = [c_char_p] pythonapi.PyMem_Free.restype = c_void +# the least but one parameter should be c_void_p but it's not used +# so I trick the compiler to not complaint about constanst pointer passed +# to void* arg libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, - c_int, c_void_p, c_int] + c_int, POINTER(c_int), c_int] libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE) libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int, c_int, c_int] @@ -177,7 +180,8 @@ dst = buf if not obj.f_univ_newline: - nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + buf_p = cast(buf, POINTER(c_char)) + nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf_p, n) return nread, buf newlinetypes = obj.f_newlinetypes @@ -188,7 +192,8 @@ while n: src = dst - nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf, n) + buf_p = cast(buf, POINTER(c_char)) + nread = libbz2.BZ2_bzRead(byref(bzerror), stream, buf_p, n) n -= nread # assuming 1 byte out for each in; will adjust shortread = n != 0 # True iff EOF or error @@ -327,7 +332,8 @@ used_v_size = buf_pos if used_v_size != total_v_size: - return "".join(buf_lst[:used_v_size]) + assert used_v_size >= 0 + return "".join(buf_lst[0:used_v_size]) return "".join(buf_lst) def _new_buffer_size(current_size): @@ -410,8 +416,9 @@ bzerror = c_int() if mode_char == 'r': + x = c_int() # little trick for null former c_void_p argument self.fp = libbz2.BZ2_bzReadOpen(byref(bzerror), self._file, - 0, 0, c_void_p(), 0) + 0, 0, byref(x), 0) else: self.fp = libbz2.BZ2_bzWriteOpen(byref(bzerror), self._file, compresslevel, 0, 0) @@ -641,10 +648,8 @@ buf_lst = [c for c in buf.value] if bytesread != bufsize: - start = 0 assert bytesread >= 0 - assert start >= 0 - return self.space.wrap("".join(buf_lst[start:bytesread])) + return self.space.wrap("".join(buf_lst[0:bytesread])) return self.space.wrap("".join(buf_lst)) read.unwrap_spec = ['self', int] From rhymes at codespeak.net Thu Aug 17 19:27:21 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 19:27:21 +0200 (CEST) Subject: [pypy-svn] r31380 - pypy/dist/pypy/module/bz2 Message-ID: <20060817172721.AB24710080@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 19:27:17 2006 New Revision: 31380 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: more fixes Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 19:27:17 2006 @@ -664,16 +664,14 @@ # it seems size definitely ignored in CPython, so... lines = [] - while True: w_line = self.readline() line = self.space.str_w(w_line) if not line: break - lines.append(line) - return self.space.wrap(lines) + return self.space.newtuple(lines) readlines.unwrap_spec = ['self', int] def write(self, data): @@ -732,7 +730,7 @@ else: raise OperationError(space.w_SystemError, space.wrap( - "Unknown newlines value 0x%s\n" % hex(self.f_newlinetypes))) + "Unknown newlines value 0x%x" % self.f_newlinetypes)) def fget_closed(space, self): return space.wrap(self.mode == MODE_CLOSED) From rhymes at codespeak.net Thu Aug 17 19:36:23 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 19:36:23 +0200 (CEST) Subject: [pypy-svn] r31381 - pypy/dist/pypy/module/bz2 Message-ID: <20060817173623.27C7010080@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 19:36:20 2006 New Revision: 31381 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: fix readlines for real. now it fails badly at gcc compiling stage but is better then before... Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 19:36:20 2006 @@ -671,7 +671,7 @@ break lines.append(line) - return self.space.newtuple(lines) + return self.space.newlist([self.space.wrap(i) for i in lines]) readlines.unwrap_spec = ['self', int] def write(self, data): From rhymes at codespeak.net Thu Aug 17 21:03:05 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 21:03:05 +0200 (CEST) Subject: [pypy-svn] r31382 - pypy/dist/pypy/module/bz2 Message-ID: <20060817190305.81F351006E@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 21:03:01 2006 New Revision: 31382 Modified: pypy/dist/pypy/module/bz2/bzlib.py pypy/dist/pypy/module/bz2/fileobject.py pypy/dist/pypy/module/bz2/interp_bz2.py Log: more fixes to make it compile. Almost there Modified: pypy/dist/pypy/module/bz2/bzlib.py ============================================================================== --- pypy/dist/pypy/module/bz2/bzlib.py (original) +++ pypy/dist/pypy/module/bz2/bzlib.py Thu Aug 17 21:03:01 2006 @@ -65,6 +65,7 @@ ('_base', POINTER(c_ubyte)), ('_size', c_int), ] +__sbuf.__name__ = "__sbuf__" assert sizeof(__sbuf) == 8, sizeof(__sbuf) assert alignment(__sbuf) == 4, alignment(__sbuf) class __sFILEX(Structure): @@ -95,6 +96,7 @@ ('_blksize', c_int), ('_offset', fpos_t), ] +__sFILE.__name__ = "__sFILE__" assert sizeof(__sFILE) == 88, sizeof(__sFILE) assert alignment(__sFILE) == 4, alignment(__sFILE) FILE = __sFILE Modified: pypy/dist/pypy/module/bz2/fileobject.py ============================================================================== --- pypy/dist/pypy/module/bz2/fileobject.py (original) +++ pypy/dist/pypy/module/bz2/fileobject.py Thu Aug 17 21:03:01 2006 @@ -107,6 +107,7 @@ ('_base', POINTER(c_ubyte)), ('_size', c_int), ] +__sbuf.__name__ = "__sbuf_" assert sizeof(__sbuf) == 8, sizeof(__sbuf) assert alignment(__sbuf) == 4, alignment(__sbuf) class __sFILEX(Structure): @@ -137,6 +138,7 @@ ('_blksize', c_int), ('_offset', fpos_t), ] +__sFILE.__name__ = "__sFILE_" assert sizeof(__sFILE) == 88, sizeof(__sFILE) assert alignment(__sFILE) == 4, alignment(__sFILE) FILE = __sFILE @@ -684,6 +686,7 @@ pass class _object(Structure): pass +_object.__name__ = "_object_" PyObject = _object PyFileObject._fields_ = [ ('ob_refcnt', Py_ssize_t), @@ -848,12 +851,15 @@ class PyMethodDef(Structure): pass PyMethodDef._fields_ = [] +PyMethodDef.__name__ = "PyMethodDef_" class PyMemberDef(Structure): pass PyMemberDef._fields_ = [] +PyMemberDef.__name__ = "PyMemberDef_" class PyGetSetDef(Structure): pass PyGetSetDef._fields_ = [] +PyGetSetDef.__name__ = "PyGetSetDef_" _typeobject._fields_ = [ ('ob_refcnt', Py_ssize_t), ('ob_type', POINTER(_typeobject)), @@ -904,6 +910,7 @@ ('tp_weaklist', POINTER(PyObject)), ('tp_del', destructor), ] +_typeobject.__name__ = "_typeobject_" assert sizeof(_typeobject) == 192, sizeof(_typeobject) assert alignment(_typeobject) == 4, alignment(_typeobject) PyTypeObject = _typeobject Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Thu Aug 17 21:03:01 2006 @@ -24,6 +24,9 @@ #include #include """ + # XXX: with this it should compile fine but on my machine pypy doesn't + # inject this header so it's pretty useless. Kept as a remind. + # _includes_ = ["bzlib.h"] off_t = ctypes_platform.SimpleType("off_t", c_longlong) size_t = ctypes_platform.SimpleType("size_t", c_ulong) BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ") From rhymes at codespeak.net Thu Aug 17 21:26:07 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Thu, 17 Aug 2006 21:26:07 +0200 (CEST) Subject: [pypy-svn] r31383 - in pypy/dist/pypy/module/mmap: . test Message-ID: <20060817192607.9992510076@code0.codespeak.net> Author: rhymes Date: Thu Aug 17 21:26:03 2006 New Revision: 31383 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/module/mmap/test/test_mmap.py Log: make mmap more pythonic. Not it has special methods. Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Thu Aug 17 21:26:03 2006 @@ -511,15 +511,13 @@ # raise WinError(dwErrCode) resize.unwrap_spec = ['self', int] - ## XXX: The following is a series of "special" methods turned into regular - # because the ext compiler does not support __xx__ methods right now. - def get_len(self): + def __len__(self): self._check_valid() return self.space.wrap(self._size) - get_len.unwrap_spec = ['self'] + __len__.unwrap_spec = ['self'] - def get_item(self, index): + def __getitem__(self, index): self._check_valid() # XXX this does not support slice() instances @@ -529,9 +527,9 @@ except IndexError: raise OperationError(self.space.w_IndexError, self.space.wrap("mmap index out of range")) - get_item.unwrap_spec = ['self', int] + __getitem__.unwrap_spec = ['self', int] - def set_item(self, index, value): + def __setitem__(self, index, value): self._check_valid() self._check_writeable() @@ -553,30 +551,30 @@ p = c_char_p(str_data) libc.memcpy(self._data, p, len(str_data)) - set_item.unwrap_spec = ['self', int, str] + __setitem__.unwrap_spec = ['self', int, str] - def del_item(self, index): + def __delitem__(self, index): self._check_valid() # XXX this does not support slice() instances (does it matter?) raise OperationError(self.space.w_TypeError, self.space.wrap("mmap object doesn't support item deletion")) - del_item.unwrap_spec = ['self', int] + __delitem__.unwrap_spec = ['self', int] - def add(self, w_other): + def __add__(self, w_other): self._check_valid() raise OperationError(self.space.w_SystemError, self.space.wrap("mmaps don't support concatenation")) - add.unwrap_spec = ['self', W_Root] + __add__.unwrap_spec = ['self', W_Root] - def mul(self, w_other): + def __mul__(self, w_other): self._check_valid() raise OperationError(self.space.w_SystemError, self.space.wrap("mmaps don't support repeat operation")) - mul.unwrap_spec = ['self', W_Root] + __mul__.unwrap_spec = ['self', W_Root] _mmap.typedef = TypeDef("_mmap", @@ -604,12 +602,15 @@ move = interp2app(_mmap.move, unwrap_spec=_mmap.move.unwrap_spec), resize = interp2app(_mmap.resize, unwrap_spec=_mmap.resize.unwrap_spec), - get_len = interp2app(_mmap.get_len, unwrap_spec=_mmap.get_len.unwrap_spec), - get_item = interp2app(_mmap.get_item, unwrap_spec=_mmap.get_item.unwrap_spec), - set_item = interp2app(_mmap.set_item, unwrap_spec=_mmap.set_item.unwrap_spec), - del_item = interp2app(_mmap.del_item, unwrap_spec=_mmap.del_item.unwrap_spec), - add = interp2app(_mmap.add, unwrap_spec=_mmap.add.unwrap_spec), - mul = interp2app(_mmap.mul, unwrap_spec=_mmap.mul.unwrap_spec), + __len__ = interp2app(_mmap.__len__, unwrap_spec=_mmap.__len__.unwrap_spec), + __getitem__ = interp2app(_mmap.__getitem__, + unwrap_spec=_mmap.__getitem__.unwrap_spec), + __setitem__ = interp2app(_mmap.__setitem__, + unwrap_spec=_mmap.__setitem__.unwrap_spec), + __delitem__ = interp2app(_mmap.__delitem__, + unwrap_spec=_mmap.__delitem__.unwrap_spec), + __add__ = interp2app(_mmap.__add__, unwrap_spec=_mmap.__add__.unwrap_spec), + __mul__ = interp2app(_mmap.__mul__, unwrap_spec=_mmap.__mul__.unwrap_spec), ) Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Thu Aug 17 21:26:03 2006 @@ -306,7 +306,7 @@ f.flush() m = mmap(f.fileno(), 6) - assert m.get_len() == 6 + assert len(m) == 6 m.close() f.close() @@ -318,12 +318,12 @@ f.flush() m = mmap(f.fileno(), 6) - fn = lambda: m.get_item("foo") + fn = lambda: m["foo"] raises(TypeError, fn) - fn = lambda: m.get_item(-7) + fn = lambda: m[-7] raises(IndexError, fn) - assert m.get_item(0) == 'f' - assert m.get_item(-1) == 'r' + assert m[0] == 'f' + assert m[-1] == 'r' # sl = slice(1, 2) # assert m.get_item(sl) == 'o' m.close() @@ -337,14 +337,14 @@ f.flush() m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) - fn = lambda: m.set_item(1, 'a') + def fn(): m[1] = 'a' raises(TypeError, fn) m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) - fn = lambda: m.set_item("foo", 'a') + def fn(): m["foo"] = 'a' raises(TypeError, fn) - fn = lambda: m.set_item(-7, 'a') + def fn(): m[-7] = 'a' raises(IndexError, fn) - fn = lambda: m.set_item(0, 'ab') + def fn(): m[0] = 'ab' raises(IndexError, fn) # def f(m): m[1:3] = u'xx' # py.test.raises(IndexError, f, m) @@ -356,9 +356,9 @@ # m[1:3] = 'xx' # assert m.read(6) == "fxxbar" # m.seek(0) - m.set_item(0, 'x') - assert m.get_item(0) == 'x' - m.set_item(-6, 'y') + m[0] = 'x' + assert m[0] == 'x' + m[-6] = 'y' data = m.read(6) assert data == "yoobar" # yxxbar with slice's stuff m.close() @@ -372,11 +372,11 @@ f.flush() m = mmap(f.fileno(), 6) - fn = lambda: m.del_item("foo") + def fn(): del m["foo"] raises(TypeError, fn) # def f(m): del m[1:3] # py.test.raises(TypeError, f, m) - fn = lambda: m.del_item(1) + def fn(): del m[1] raises(TypeError, fn) m.close() f.close() @@ -389,12 +389,12 @@ f.flush() m = mmap(f.fileno(), 6) - fn = lambda: m.add(1) + def fn(): m + 1 raises(SystemError, fn) - # def f(m): m += 1 - # py.test.raises(SystemError, f, m) - # f = lambda: 1 + m - # py.test.raises(TypeError, f) + def fn(m): m += 1 + raises(SystemError, fn, m) + def fn(): 1 + m + raises(TypeError, fn) m.close() f.close() @@ -406,13 +406,12 @@ f.flush() m = mmap(f.fileno(), 6) - fn = lambda: m.mul(1) + def fn(): m * 1 raises(SystemError, fn) - # def f(m): - # m *= 1 - # py.test.raises(SystemError, f, m) - # f = lambda: 1 * m - # py.test.raises(TypeError, f) + def fn(m): m *= 1 + raises(SystemError, fn, m) + def fn(): 1 * m + raises(TypeError, fn) m.close() f.close() # @@ -442,16 +441,16 @@ # sanity checks assert m.find("foo") == PAGESIZE - assert m.get_len() == 2 * PAGESIZE - assert m.get_item(0) == '\0' + assert len(m) == 2 * PAGESIZE + assert m[0] == '\0' # assert m[0:3] == '\0\0\0' # modify the file's content - m.set_item(0, '3') + m[0] = '3' # m[PAGESIZE+3:PAGESIZE+3+3] = 'bar' # check that the modification worked - assert m.get_item(0) == '3' + assert m[0] == '3' # assert m[0:3] == '3\0\0' # assert m[PAGESIZE-1:PAGESIZE+7] == '\0foobar\0' @@ -463,17 +462,17 @@ m.seek(42, 1) assert m.tell() == 42 m.seek(0, 2) - assert m.tell() == m.get_len() + assert m.tell() == len(m) raises(ValueError, m.seek, -1) raises(ValueError, m.seek, 1, 2) - raises(ValueError, m.seek, -m.get_len() - 1, 2) + raises(ValueError, m.seek, -len(m) - 1, 2) # try resizing map if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)): m.resize(512) - assert m.get_len() == 512 + assert len(m) == 512 raises(ValueError, m.seek, 513, 0) # check that the underlying file is truncated too @@ -494,7 +493,7 @@ # assert m[:] == 'a' * mapsize # def f(m): m[:] = 'b' * mapsize # py.test.raises(TypeError, f, m) - fn = lambda: m.set_item(0, 'b') + def fn(): m[0] = 'b' raises(TypeError, fn) def fn(m): m.seek(0, 0); m.write("abc") raises(TypeError, fn, m) @@ -586,7 +585,7 @@ f.close() f = open(filename, "rb+") m = mmap.mmap(f.fileno(), 0) - assert m.get_len() == 2**16 + assert len(m) == 2**16 assert m.read(2**16) == 2**16 * "m" m.close() f.close() From ac at codespeak.net Thu Aug 17 22:47:37 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 17 Aug 2006 22:47:37 +0200 (CEST) Subject: [pypy-svn] r31384 - in pypy/dist/pypy/rpython/rctypes: . test Message-ID: <20060817204737.97DAE1006E@code0.codespeak.net> Author: ac Date: Thu Aug 17 22:47:35 2006 New Revision: 31384 Modified: pypy/dist/pypy/rpython/rctypes/rstringbuf.py pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py Log: Support assignment to stringbuf.raw Modified: pypy/dist/pypy/rpython/rctypes/rstringbuf.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rstringbuf.py (original) +++ pypy/dist/pypy/rpython/rctypes/rstringbuf.py Thu Aug 17 22:47:35 2006 @@ -32,7 +32,7 @@ def rtype_setattr(self, hop): s_attr = hop.args_s[1] assert s_attr.is_constant() - assert s_attr.const == 'value' + assert s_attr.const in ('value', 'raw') v_box, v_attr, v_value = hop.inputargs(self, lltype.Void, string_repr) hop.gendirectcall(ll_stringbuf_setvalue_from_string, v_box, v_value) Modified: pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rstringbuf.py Thu Aug 17 22:47:35 2006 @@ -142,6 +142,8 @@ assert buf.value == 'abcde' buf.value = 'x' assert buf.value == 'x' + buf.raw = 'y' + assert buf.value == 'y' return ord(buf[2]) res = interpret(func, [12]) From ac at codespeak.net Thu Aug 17 23:06:50 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 17 Aug 2006 23:06:50 +0200 (CEST) Subject: [pypy-svn] r31385 - in pypy/dist/pypy: rpython translator/c Message-ID: <20060817210650.56A321006C@code0.codespeak.net> Author: ac Date: Thu Aug 17 23:06:49 2006 New Revision: 31385 Modified: pypy/dist/pypy/rpython/rcpy.py pypy/dist/pypy/translator/c/node.py Log: Support typedef defined external structs/unions. Modified: pypy/dist/pypy/rpython/rcpy.py ============================================================================== --- pypy/dist/pypy/rpython/rcpy.py (original) +++ pypy/dist/pypy/rpython/rcpy.py Thu Aug 17 23:06:49 2006 @@ -152,14 +152,11 @@ PyObjPtr = lltype.Ptr(lltype.PyObject) PyNumberMethods = lltype.Struct('PyNumberMethods', - ('data', lltype.FixedSizeArray(lltype.Signed, 38)) -) + hints={'c_name': 'PyNumberMethods', 'external': True, 'typedef': True}) PyMappingMethods = lltype.Struct('PyMappingMethods', - ('data', lltype.FixedSizeArray(lltype.Signed, 3)) -) + hints={'c_name': 'PyMappingMethods', 'external': True, 'typedef': True}) PySequenceMethods = lltype.Struct('PySequenceMethods', - ('data', lltype.FixedSizeArray(lltype.Signed, 10)) -) + hints={'c_name': 'PySequenceMethods', 'external': True, 'typedef': True}) PY_TYPE_OBJECT = lltype.PyForwardReference() PY_TYPE_OBJECT.become(lltype.PyStruct( 'PyTypeObject', @@ -213,7 +210,7 @@ #lltype.Ptr(lltype.FuncType([llmemory.Address], # lltype.Void))), - hints={'c_name': '_typeobject', 'external': True, 'inline_head': True})) + hints={'c_name': 'PyTypeObject', 'external': True, 'typedef': True, 'inline_head': True})) # XXX 'c_name' should be 'PyTypeObject' but genc inserts 'struct' :-( def ll_tp_dealloc(p): Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Thu Aug 17 23:06:49 2006 @@ -52,6 +52,9 @@ if STRUCT._hints.get('union'): self.typetag = 'union' assert STRUCT._gckind == 'raw' # not supported: "GcUnion" + if STRUCT._hints.get('typedef'): + self.typetag = '' + assert STRUCT._hints.get('external') if STRUCT._hints.get('c_name'): self.barename = self.name = STRUCT._hints['c_name'] self.c_struct_field_name = self.verbatim_field_name From pedronis at codespeak.net Thu Aug 17 23:31:06 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Aug 2006 23:31:06 +0200 (CEST) Subject: [pypy-svn] r31386 - pypy/dist/pypy/module/_demo Message-ID: <20060817213106.81BA91006E@code0.codespeak.net> Author: pedronis Date: Thu Aug 17 23:31:05 2006 New Revision: 31386 Modified: pypy/dist/pypy/module/_demo/demo.py Log: fix Modified: pypy/dist/pypy/module/_demo/demo.py ============================================================================== --- pypy/dist/pypy/module/_demo/demo.py (original) +++ pypy/dist/pypy/module/_demo/demo.py Thu Aug 17 23:31:05 2006 @@ -36,7 +36,7 @@ while 1: first = lst[head] if first > math.sqrt(n) + 1: - lst_w = [space.newint(i) for i in range(n)] + lst_w = [space.newint(i) for i in lst] return space.newlist(lst_w) newlst = [] for element in lst: From arigo at codespeak.net Fri Aug 18 00:11:38 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Aug 2006 00:11:38 +0200 (CEST) Subject: [pypy-svn] r31387 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060817221138.CE0F110036@code0.codespeak.net> Author: arigo Date: Fri Aug 18 00:11:30 2006 New Revision: 31387 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: sharing accomodation with mwh and fijal Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Fri Aug 18 00:11:30 2006 @@ -11,7 +11,7 @@ Anders Chrigstroem 20-28/8 on-campus Samuele Pedroni 20-28/8 on-campus Michael Hudson 20-27 Brookfield Hall, Castletroy -Armin Rigo 21-28 ? +Armin Rigo 21-28 Brookfield Hall, Castletroy Holger Krekel 20-25/8 aiming for brookfield Maciej Fijalkowski 21-28?/8 Brookfield Hall, Castletroy Bea During 20-25/8 aiming for brookfield From ac at codespeak.net Fri Aug 18 12:23:19 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Fri, 18 Aug 2006 12:23:19 +0200 (CEST) Subject: [pypy-svn] r31388 - pypy/dist/pypy/translator/c Message-ID: <20060818102319.B647610060@code0.codespeak.net> Author: ac Date: Fri Aug 18 12:23:17 2006 New Revision: 31388 Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py Log: Do not emit a forward declaration for external structs/unions Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Fri Aug 18 12:23:17 2006 @@ -396,6 +396,8 @@ print >> fi, '/*** Structure definitions ***/' print >> fi for node in structdeflist: + if node.is_external(): + continue print >> fi, '%s %s;' % (node.typetag, node.name) print >> fi for node in structdeflist: @@ -512,7 +514,7 @@ print >> f, '/*** Structure definitions ***/' print >> f for node in structdeflist: - if node.name: + if node.name and not node.is_external(): print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Fri Aug 18 12:23:17 2006 @@ -145,6 +145,9 @@ fldname = self.c_struct_field_name(fldname) return '%s->%s' % (baseexpr, fldname) + def is_external(self): + return self.STRUCT._hints.get('external') # XXX hack + def definition(self): if self.fields is None: # external definition only return @@ -240,6 +243,9 @@ def ptr_access_expr(self, baseexpr, index): return '%s->items[%d]' % (baseexpr, index) + def is_external(self): + return False + def definition(self): gcpolicy = self.db.gcpolicy yield 'struct %s {' % self.name @@ -323,6 +329,9 @@ ptr_access_expr = access_expr + def is_external(self): + return False + def definition(self): return [] # no declaration is needed From pedronis at codespeak.net Fri Aug 18 14:31:50 2006 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 18 Aug 2006 14:31:50 +0200 (CEST) Subject: [pypy-svn] r31389 - pypy/dist/pypy/jit/timeshifter Message-ID: <20060818123150.0368510060@code0.codespeak.net> Author: pedronis Date: Fri Aug 18 14:31:48 2006 New Revision: 31389 Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py Log: (arre, pedronis) add comments explaining a bit more the contents of block inserted by the timeshifting process, (having the code itself doing this more readable would be better still :( ) Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Fri Aug 18 14:31:48 2006 @@ -90,6 +90,17 @@ r_instance = self.annhelper.getdelayedrepr(s_instance) return s_instance, r_instance + # creates and numbers reentry_block for block reached by link + # argument: + # + # the reentry_block takes a jitstate and a list of boxes + # and explodes the content of boxes into variables for each + # link argument; + # redboxes are preserved, green values are read out of constant + # boxes + # + # then passes the variables to the target of link + # def getexitindex(self, link, inputargs, args_r, entering_links): self.latestexitindex += 1 v_jitstate = varoftype(self.r_JITState.lowleveltype, 'jitstate') @@ -326,6 +337,14 @@ v_boxes = rlist.newlist(llops, self.r_box_list, type_erased_v) return v_boxes + # insert before non-join blocks a block with: + # + # newjitstate = rtimeshift.enter_block(jitstate, boxes) + # where boxes = [current-redboxes] + # + # then pass the newjitstate (XXX modified?) and boxes + # to the block original; boxes are modified in place + # def bookkeeping_enter_simple(self, args_r, newinputargs, before_block, llops, v_boxes, is_returnblock=False): v_newjitstate = llops.genmixlevelhelpercall(rtimeshift.enter_block, @@ -339,16 +358,36 @@ before_block.operations[:] = llops return None + # insert before join blocks a block with: - # key = () - # boxes = [] - # jitstate = ll_retrieve_jitstate_for_merge({}, # <- constant dict (key->...) - # jitstate, key, boxes) - # and which passes then to the original block the possibly new jitstate, - # and possible changed redbox read back again out of the 'boxes' list. - # ll_retrieve_jitstate_for_merge is supposed to use the "constant" dict as cache + # + # newjiststate = merge_point(jitstate, key, boxes) + # where + # key = (current-green-values) + # boxes = [current-redboxes] + # merge_point = (lambda jitstate, key, boxes: + # rtimeshift.retrieve_jitstate_for_merge( + # constant {}, jitstate, + # key, boxes)) + # if newjistate is None then go to dispatch_block(jitstate) + # else go to read_boxes_block(newjiststate, boxes) + # + # and the other block read_boxes_block which reads the redboxes back out boxes + # and pass them along to the original block together with the new jitstate + # + # for the return block case (which is always considered a join block) the + # read_boxes_block is special: + # + # rtimeshift.save_return(newjitstate, boxes) + # go to dispatch_block(newjitstate) + # + # retrieve_jitstate_for_merge is supposed to use the "constant" dict as cache # mapping green values combinations to frozen states for red boxes values # and generated blocks + # + # if the newjitstate is None, it means an old state/block could be reused + # and execution continues to the dispatch_block + # def bookkeeping_enter_for_join(self, args_r, newinputargs, before_block, llops, v_boxes, is_returnblock): getrepr = self.rtyper.getrepr @@ -436,7 +475,30 @@ read_boxes_block.closeblock(to_target) return cache - + + # insert after blocks the following logic: + # if the original block is the returnblock: + # + # go to dispatch_block(jitstate) + # + # if the original block has just one exit or the exitswitch is green: + # + # newjitstate = rtimeshift.leave_block(jitstate) + # + # if the original block has more than one exit (split case): + # + # res = rtimeshift.leave_block_split(jitstate, exitswitchredbox, exitindex, boxes) + # where + # boxes =[boxes-for-all-current-red-and-green-values] + # exitindex = number identifying the false branch of the switch + # if res then go to true_exit_block + # else go to false_exit_block + # + # exitindex is computed by getexitindex, see comment for that method + # + # leave_block_split if the exitswitchredbox is constant just returns its value as res + # otherwise returns True and schedule the false case + # def insert_bookkeeping_leave_block(self, block, entering_links): # XXX wrong with exceptions as much else @@ -535,6 +597,27 @@ newblock.exitswitch = v_res newblock.operations[:] = llops + # put the following logic in the dispatch block: + # + # boxes = [] + # next = rtimeshift.dispatch_next(jitstate, boxes) + # switch next: + # : + # go to reentry_block(jitstate, boxes) + # ... + # default: + # go to prepare_return_block(jitstate) + # + # the prepare_return_block does: + # + # builder = prepare_return(jitstate) + # where + # prepare_return = (lambda jitstate: + # rtimeshift.prepare_return(jitstate, return_cache, + # constTYPE(RETURN_TYPE))) + # where return_cache is a predefined cache + # return builder + # def insert_dispatch_logic(self): dispatchblock = self.dispatchblock [v_jitstate] = dispatchblock.inputargs @@ -600,6 +683,7 @@ def timeshift_block(self, timeshifted_blocks, entering_links, block): blocks = [block] i = 0 + # XXX in-progress, split block at direct_calls for call support while i < len(block.operations): op = block.operations[i] if op.opname == 'direct_call': From rhymes at codespeak.net Fri Aug 18 17:36:45 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Fri, 18 Aug 2006 17:36:45 +0200 (CEST) Subject: [pypy-svn] r31391 - pypy/dist/pypy/module/_ssl Message-ID: <20060818153645.3E0A410060@code0.codespeak.net> Author: rhymes Date: Fri Aug 18 17:36:44 2006 New Revision: 31391 Modified: pypy/dist/pypy/module/_ssl/interp_ssl.py Log: one less fix to go. Modified: pypy/dist/pypy/module/_ssl/interp_ssl.py ============================================================================== --- pypy/dist/pypy/module/_ssl/interp_ssl.py (original) +++ pypy/dist/pypy/module/_ssl/interp_ssl.py Fri Aug 18 17:36:44 2006 @@ -586,6 +586,6 @@ def ssl(space, w_socket, w_key_file=None, w_cert_file=None): """ssl(socket, [keyfile, certfile]) -> sslobject""" - return new_sslobject(space, w_socket, w_key_file, w_cert_file) + return space.wrap(new_sslobject(space, w_socket, w_key_file, w_cert_file)) ssl.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] From hpk at codespeak.net Fri Aug 18 19:37:31 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 18 Aug 2006 19:37:31 +0200 (CEST) Subject: [pypy-svn] r31393 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060818173731.8942610069@code0.codespeak.net> Author: hpk Date: Fri Aug 18 19:37:30 2006 New Revision: 31393 Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt Log: added accomodation details for bea and me. Modified: pypy/extradoc/sprintinfo/ireland-2006/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/people.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/people.txt Fri Aug 18 19:37:30 2006 @@ -12,9 +12,9 @@ Samuele Pedroni 20-28/8 on-campus Michael Hudson 20-27 Brookfield Hall, Castletroy Armin Rigo 21-28 Brookfield Hall, Castletroy -Holger Krekel 20-25/8 aiming for brookfield +Holger Krekel 20-25/8 Old Quarter Lodge (city) Maciej Fijalkowski 21-28?/8 Brookfield Hall, Castletroy -Bea During 20-25/8 aiming for brookfield +Bea During 20-25/8 Old quarter Lodge (city) ==================== ============== ======================== People on the following list were present at previous sprints: From rhymes at codespeak.net Sat Aug 19 14:34:40 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 14:34:40 +0200 (CEST) Subject: [pypy-svn] r31399 - pypy/dist/pypy/module/rctime Message-ID: <20060819123440.1667710060@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 14:34:38 2006 New Revision: 31399 Removed: pypy/dist/pypy/module/rctime/strptime.txt Log: not needed anymore From rhymes at codespeak.net Sat Aug 19 15:29:54 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 15:29:54 +0200 (CEST) Subject: [pypy-svn] r31400 - in pypy/dist/pypy/module/rctime: . test Message-ID: <20060819132954.869B410060@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 15:29:51 2006 New Revision: 31400 Modified: pypy/dist/pypy/module/rctime/interp_time.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: time() works in Windows! tzname, altzone, daylight and timezone are not supported by now. Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Aug 19 15:29:51 2006 @@ -9,38 +9,49 @@ import math _POSIX = os.name == "posix" +_WIN = os.name == "nt" +_include = "#include " +if _POSIX: + _include = """%s + #include """ % _include + class CConfig: - _header_ = """ - #include - #include - """ - timeval = ctypes_platform.Struct("struct timeval", [("tv_sec", c_int), - ("tv_usec", c_int)]) - tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), - ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), - ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), - ("tm_yday", c_int), ("tm_isdst", c_int), ("tm_gmtoff", c_long), - ("tm_zone", c_char_p)]) + _header_ = _include CLOCKS_PER_SEC = ctypes_platform.ConstantInteger("CLOCKS_PER_SEC") clock_t = ctypes_platform.SimpleType("clock_t", c_ulong) time_t = ctypes_platform.SimpleType("time_t", c_long) size_t = ctypes_platform.SimpleType("size_t", c_long) + +if _POSIX: + CConfig.timeval = ctypes_platform.Struct("struct timeval", [("tv_sec", c_int), + ("tv_usec", c_int)]) + CConfig.tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), + ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), + ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), + ("tm_yday", c_int), ("tm_isdst", c_int), ("tm_gmtoff", c_long), + ("tm_zone", c_char_p)]) +elif _WIN: + CConfig.tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), + ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), + ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), + ("tm_yday", c_int), ("tm_isdst", c_int)]) class cConfig: pass cConfig.__dict__.update(ctypes_platform.configure(CConfig)) -cConfig.timeval.__name__ = "_timeval" cConfig.tm.__name__ = "_tm" +if _POSIX: + cConfig.timeval.__name__ = "_timeval" + timeval = cConfig.timeval + CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC clock_t = cConfig.clock_t time_t = cConfig.time_t size_t = cConfig.size_t -timeval = cConfig.timeval tm = cConfig.tm - has_gettimeofday = False if hasattr(libc, "gettimeofday"): libc.gettimeofday.argtypes = [c_void_p, c_void_p] @@ -60,7 +71,8 @@ libc.mktime.restype = time_t libc.asctime.argtypes = [POINTER(tm)] libc.asctime.restype = c_char_p -libc.tzset.restype = None # tzset() returns void +if _POSIX: + libc.tzset.restype = None # tzset() returns void libc.strftime.argtypes = [c_char_p, size_t, c_char_p, POINTER(tm)] libc.strftime.restype = size_t @@ -68,9 +80,13 @@ return (1, 0)[bool(os.getenv("PYTHONY2K"))] def _init_timezone(): - timezone = daylight = tzname = altzone = None + timezone = daylight = altzone = 0 + tzname = ["", ""] + + # pypy cant' use in_dll to access global exported variables + # so we can't compute these attributes - # if _MS_WINDOWS: + # if _WIN: # cdll.msvcrt._tzset() # # timezone = c_long.in_dll(cdll.msvcrt, "_timezone").value @@ -123,9 +139,8 @@ time() has a resolution in seconds """ - # if _MS_WINDOWS: - # return libc.time(None) - # + if _WIN: + return float(libc.time(None)) if has_gettimeofday: t = timeval() if libc.gettimeofday(byref(t), c_void_p(None)) == 0: Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Sat Aug 19 15:29:51 2006 @@ -1,10 +1,6 @@ from pypy.conftest import gettestobjspace import os -if os.name == "nt": - from py.test import skip - skip("Windows is not supported") - class AppTestRCTime: def setup_class(cls): space = gettestobjspace(usemodules=('rctime',)) From rhymes at codespeak.net Sat Aug 19 15:41:03 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 15:41:03 +0200 (CEST) Subject: [pypy-svn] r31401 - in pypy/dist/pypy/module/rctime: . test Message-ID: <20060819134103.3AF5B10053@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 15:41:00 2006 New Revision: 31401 Modified: pypy/dist/pypy/module/rctime/__init__.py pypy/dist/pypy/module/rctime/app_time.py pypy/dist/pypy/module/rctime/interp_time.py pypy/dist/pypy/module/rctime/test/test_rctime.py Log: sleep() for Windows Modified: pypy/dist/pypy/module/rctime/__init__.py ============================================================================== --- pypy/dist/pypy/module/rctime/__init__.py (original) +++ pypy/dist/pypy/module/rctime/__init__.py Sat Aug 19 15:41:00 2006 @@ -21,6 +21,12 @@ def buildloaders(cls): from pypy.module.rctime import interp_time + import os + + if os.name == "posix": + Module.appleveldefs['sleep'] = 'app_time.sleep' + elif os.name == "nt": + Module.interpleveldefs['sleep'] = 'interp_time.sleep' # this machinery is needed to expose constants # that have to be initialized one time only @@ -38,7 +44,6 @@ buildloaders = classmethod(buildloaders) appleveldefs = { - 'sleep': 'app_time.sleep', '_check_float': 'app_time._check_float', 'struct_time': 'app_time.struct_time', '__doc__': 'app_time.__doc__', Modified: pypy/dist/pypy/module/rctime/app_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/app_time.py (original) +++ pypy/dist/pypy/module/rctime/app_time.py Sat Aug 19 15:41:00 2006 @@ -46,33 +46,18 @@ except ValueError: raise TypeError, "a float is required" -def _float_sleep(secs): - import select - - if _POSIX: +if _POSIX: + def _float_sleep(secs): + import select select.select([], [], [], secs) - return - # elif _MS_WINDOWS: - # msecs = secs * 1000.0 - # if msecs > float(sys.maxint * 2 - 1): # ULONG_MAX - # raise OverflowError, "sleep length is too large" - # lock.acquire() - # ul_millis = c_ulong(long(msecs)) - # windll.kernel32.Sleep(ul_millis) - # lock.release() - # else: - # lock.acquire() - # _libc.sleep.argtypes = [c_int] - # _libc.sleep(int(secs)) - # lock.release() - -def sleep(secs): - """sleep(seconds) - - Delay execution for a given number of seconds. The argument may be - a floating point number for subsecond precision.""" - _check_float(secs) - _float_sleep(secs) + + def sleep(secs): + """sleep(seconds) + + Delay execution for a given number of seconds. The argument may be + a floating point number for subsecond precision.""" + _check_float(secs) + _float_sleep(secs) def strptime(string, format="%a %b %d %H:%M:%S %Y"): Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Aug 19 15:41:00 2006 @@ -7,6 +7,7 @@ from ctypes import * import os import math +import sys _POSIX = os.name == "posix" _WIN = os.name == "nt" @@ -147,7 +148,6 @@ return float(t.tv_sec) + t.tv_usec * 0.000001 return 0.0 - # elif hasattr(_libc, "ftime"): # t = _timeb() # _libc.ftime.argtypes = [c_void_p] @@ -159,6 +159,25 @@ # _libc.time(byref(t)) # return t.value +if _WIN: + def _float_sleep(space, secs): + msecs = secs * 1000.0 + if msecs > float(sys.maxint * 2 - 1): # ULONG_MAX + raise OperationError(space.w_OverflowError, + space.wrap("sleep length is too large")) + ul_millis = c_ulong(long(msecs)) + windll.kernel32.Sleep(ul_millis) + + def sleep(space, w_secs): + """sleep(seconds) + + Delay execution for a given number of seconds. The argument may be + a floating point number for subsecond precision.""" + + secs = space.float_w(w_secs) + _check_float(space, secs) + space.wrap(_float_sleep(space, secs)) + def _check_float(space, seconds): # this call the app level _check_float to check the type of # the given seconds Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Sat Aug 19 15:41:00 2006 @@ -17,8 +17,10 @@ def test_sleep(self): import rctime + import sys raises(TypeError, rctime.sleep, "foo") rctime.sleep(1.2345) + raises(OverflowError, rctime.sleep, sys.maxint) def test_clock(self): import rctime From rhymes at codespeak.net Sat Aug 19 15:42:32 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 15:42:32 +0200 (CEST) Subject: [pypy-svn] r31402 - pypy/dist/pypy/module/rctime/test Message-ID: <20060819134232.C159110053@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 15:42:31 2006 New Revision: 31402 Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py Log: OverflowError is meant for win version of sleep Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Sat Aug 19 15:42:31 2006 @@ -20,7 +20,8 @@ import sys raises(TypeError, rctime.sleep, "foo") rctime.sleep(1.2345) - raises(OverflowError, rctime.sleep, sys.maxint) + if os.name == "nt": + raises(OverflowError, rctime.sleep, sys.maxint) def test_clock(self): import rctime From rhymes at codespeak.net Sat Aug 19 15:59:18 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 15:59:18 +0200 (CEST) Subject: [pypy-svn] r31404 - pypy/dist/pypy/module/rctime Message-ID: <20060819135918.C94E210060@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 15:59:16 2006 New Revision: 31404 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: clock() for Windows Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Aug 19 15:59:16 2006 @@ -33,6 +33,7 @@ ("tm_yday", c_int), ("tm_isdst", c_int), ("tm_gmtoff", c_long), ("tm_zone", c_char_p)]) elif _WIN: + from ctypes import wintypes CConfig.tm = ctypes_platform.Struct("struct tm", [("tm_sec", c_int), ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), @@ -277,22 +278,22 @@ if _POSIX: res = float(float(libc.clock()) / CLOCKS_PER_SEC) return space.wrap(res) - # elif _MS_WINDOWS: - # divisor = 0.0 - # ctrStart = _LARGE_INTEGER() - # now = _LARGE_INTEGER() - # - # if divisor == 0.0: - # freq = _LARGE_INTEGER() - # windll.kernel32.QueryPerformanceCounter(byref(ctrStart)) - # res = windll.kernel32.QueryPerformanceCounter(byref(freq)) - # if not res or freq.QuadPart == 0: - # return float(windll.msvcrt.clock()) - # divisor = float(freq.QuadPart) - # - # windll.kernel32.QueryPerformanceCounter(byref(now)) - # diff = float(now.QuadPart - ctrStart.QuadPart) - # return float(diff / divisor) + elif _WIN: + divisor = 0.0 + ctrStart = wintypes.LARGE_INTEGER() + now = wintypes.LARGE_INTEGER() + + if divisor == 0.0: + freq = wintypes.LARGE_INTEGER() + windll.kernel32.QueryPerformanceCounter(byref(ctrStart)) + res = windll.kernel32.QueryPerformanceCounter(byref(freq)) + if not res or not freq: + return space.wrap(float(windll.msvcrt.clock())) + divisor = float(freq.value) + + windll.kernel32.QueryPerformanceCounter(byref(now)) + diff = float(now.value - ctrStart.value) + return space.wrap(float(diff / divisor)) def ctime(space, w_seconds=None): """ctime([seconds]) -> string From rhymes at codespeak.net Sat Aug 19 16:12:37 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 16:12:37 +0200 (CEST) Subject: [pypy-svn] r31405 - pypy/dist/pypy/module/rctime/test Message-ID: <20060819141237.C167810069@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 16:12:35 2006 New Revision: 31405 Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py Log: fix test. Now rctime module works fully in Windows too! Except for timezone, tzname, altzone and daylight module attributes... Modified: pypy/dist/pypy/module/rctime/test/test_rctime.py ============================================================================== --- pypy/dist/pypy/module/rctime/test/test_rctime.py (original) +++ pypy/dist/pypy/module/rctime/test/test_rctime.py Sat Aug 19 16:12:35 2006 @@ -18,6 +18,7 @@ def test_sleep(self): import rctime import sys + import os raises(TypeError, rctime.sleep, "foo") rctime.sleep(1.2345) if os.name == "nt": @@ -134,7 +135,7 @@ import os if not os.name == "posix": - py.test.skip("available only under Unix") + skip("tzset available only under Unix") # epoch time of midnight Dec 25th 2002. Never DST in northern # hemisphere. From rhymes at codespeak.net Sat Aug 19 16:41:46 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 16:41:46 +0200 (CEST) Subject: [pypy-svn] r31406 - pypy/dist/pypy/module/rctime Message-ID: <20060819144146.CAF1210060@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 16:41:44 2006 New Revision: 31406 Modified: pypy/dist/pypy/module/rctime/interp_time.py Log: two unwrap_spec lost in the void Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Aug 19 16:41:44 2006 @@ -178,6 +178,7 @@ secs = space.float_w(w_secs) _check_float(space, secs) space.wrap(_float_sleep(space, secs)) + sleep.unwrap_spec = [ObjSpace, int] def _check_float(space, seconds): # this call the app level _check_float to check the type of @@ -267,6 +268,7 @@ secs = _floattime() return space.wrap(secs) +time.unwrap_spec = [ObjSpace] def clock(space): """clock() -> floating point number @@ -294,6 +296,7 @@ windll.kernel32.QueryPerformanceCounter(byref(now)) diff = float(now.value - ctrStart.value) return space.wrap(float(diff / divisor)) +clock.unwrap_spec = [ObjSpace] def ctime(space, w_seconds=None): """ctime([seconds]) -> string From rhymes at codespeak.net Sat Aug 19 16:45:43 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 16:45:43 +0200 (CEST) Subject: [pypy-svn] r31407 - pypy/dist/pypy/module/mmap Message-ID: <20060819144543.A610A10060@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 16:45:42 2006 New Revision: 31407 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: specify params as the right type Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Aug 19 16:45:42 2006 @@ -623,14 +623,10 @@ space.wrap("memory mapped size is too large (limited by C int)")) if _POSIX: - def mmap(space, w_fileno, w_length, flags=MAP_SHARED, + def mmap(space, fileno, length, flags=MAP_SHARED, prot=PROT_WRITE | PROT_READ, access=_ACCESS_DEFAULT): - # flags = MAP_SHARED - # prot = PROT_WRITE | PROT_READ - # access = _ACCESS_DEFAULT - fd = space.int_w(w_fileno) - length = space.int_w(w_length) + fd = fileno # check size boundaries _check_map_size(space, length) @@ -697,7 +693,7 @@ m._access = access return space.wrap(m) - mmap.unwrap_spec = [ObjSpace, W_Root, W_Root, int, int, int] + mmap.unwrap_spec = [ObjSpace, int, int, int, int, int] # elif _MS_WINDOWS: # def mmap(fileno, length, *args, **kwargs): # size_hi = wintypes.DWORD() # upper 32 bits of m._size From rhymes at codespeak.net Sat Aug 19 17:57:51 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 17:57:51 +0200 (CEST) Subject: [pypy-svn] r31408 - in pypy/dist/pypy/module/mmap: . test Message-ID: <20060819155751.11B7810068@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 17:57:49 2006 New Revision: 31408 Modified: pypy/dist/pypy/module/mmap/__init__.py pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/module/mmap/test/test_mmap.py Log: mmap start working on Windows. PAGESIZE, other attributes, the instantiation of mmap object, close() and read_byte() all work. Modified: pypy/dist/pypy/module/mmap/__init__.py ============================================================================== --- pypy/dist/pypy/module/mmap/__init__.py (original) +++ pypy/dist/pypy/module/mmap/__init__.py Sat Aug 19 17:57:49 2006 @@ -15,12 +15,14 @@ def buildloaders(cls): from pypy.module.mmap import interp_mmap + import os Module.interpleveldefs["PAGESIZE"] = 'space.wrap(%r)' %\ interp_mmap._get_page_size() - - for constant, value in interp_mmap.constants.iteritems(): - Module.interpleveldefs[constant] = "space.wrap(%r)" % value + + if os.name == "posix": + for constant, value in interp_mmap.constants.iteritems(): + Module.interpleveldefs[constant] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Aug 19 17:57:49 2006 @@ -18,23 +18,33 @@ _64BIT = "64bit" in platform.architecture()[0] class CConfig: - _header_ = """ - #include - #include - """ + _header_ = "#include " size_t = ctypes_platform.SimpleType("size_t", c_long) off_t = ctypes_platform.SimpleType("off_t", c_long) -# constants, look in sys/mman.h and platform docs for the meaning -# some constants are linux only so they will be correctly exposed outside -# depending on the OS constants = {} -constant_names = ['MAP_SHARED', 'MAP_PRIVATE', 'MAP_ANON', 'MAP_ANONYMOUS', - 'PROT_READ', 'PROT_WRITE', 'PROT_EXEC', 'MAP_DENYWRITE', 'MAP_EXECUTABLE', - 'MS_SYNC'] -for name in constant_names: - setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) - +if _POSIX: + CConfig._header_ = """ + %s + #include """ % CConfig._header_ + # constants, look in sys/mman.h and platform docs for the meaning + # some constants are linux only so they will be correctly exposed outside + # depending on the OS + constant_names = ['MAP_SHARED', 'MAP_PRIVATE', 'MAP_ANON', 'MAP_ANONYMOUS', + 'PROT_READ', 'PROT_WRITE', 'PROT_EXEC', 'MAP_DENYWRITE', 'MAP_EXECUTABLE', + 'MS_SYNC'] + for name in constant_names: + setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) +elif _MS_WINDOWS: + CConfig._header_ = """ + %s + #include """ % CConfig._header_ + constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', + 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', + 'DUPLICATE_SAME_ACCESS'] + for name in constant_names: + setattr(CConfig, name, ctypes_platform.ConstantInteger(name)) + class cConfig: pass @@ -46,12 +56,14 @@ if value is not None: constants[name] = value -# MAP_ANONYMOUS is not always present but it's always available at CPython level -if cConfig.MAP_ANONYMOUS is None: - cConfig.MAP_ANONYMOUS = cConfig.MAP_ANON - constants["MAP_ANONYMOUS"] = cConfig.MAP_ANON +if _POSIX: + # MAP_ANONYMOUS is not always present but it's always available at CPython level + if cConfig.MAP_ANONYMOUS is None: + cConfig.MAP_ANONYMOUS = cConfig.MAP_ANON + constants["MAP_ANONYMOUS"] = cConfig.MAP_ANON locals().update(constants) + _ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4) @@ -61,123 +73,135 @@ libc.strerror.argtypes = [c_int] libc.memcpy.argtypes = [POINTER(c_char), c_char_p, c_int] libc.memcpy.restype = c_void_p -libc.mmap.argtypes = [c_void_p, size_t, c_int, c_int, c_int, off_t] -libc.mmap.restype = c_void_p -libc.close.argtypes = [c_int] -libc.close.restype = c_int -libc.munmap.argtypes = [c_void_p, size_t] -libc.munmap.restype = c_int -libc.msync.argtypes = [c_char_p, size_t, c_int] -libc.msync.restype = c_int - -## LINUX msync syscall helper stuff -# you don't have addressof() in rctypes so I looked up the implementation -# of addressof in ctypes source and come up with this. -pythonapi.PyLong_FromVoidPtr.argtypes = [c_char_p] -pythonapi.PyLong_FromVoidPtr.restype = c_int -# we also need to alias msync to take a c_void_p instead of c_char_p -linux_msync = libc["msync"] -linux_msync.argtypes = [c_void_p, size_t, c_int] -linux_msync.restype = c_int - -libc.memmove.argtypes = [c_char_p, c_char_p, size_t] -libc.memmove.restype = c_void_p -has_mremap = False -if hasattr(libc, "mremap"): - libc.mremap.argtypes = [POINTER(c_char), size_t, size_t, c_ulong] - libc.mremap.restype = c_void_p - has_mremap = True -libc.ftruncate.argtypes = [c_int, off_t] -libc.ftruncate.restype = c_int if _POSIX: + libc.mmap.argtypes = [c_void_p, size_t, c_int, c_int, c_int, off_t] + libc.mmap.restype = c_void_p + libc.close.argtypes = [c_int] + libc.close.restype = c_int + libc.munmap.argtypes = [c_void_p, size_t] + libc.munmap.restype = c_int + libc.msync.argtypes = [c_char_p, size_t, c_int] + libc.msync.restype = c_int + + ## LINUX msync syscall helper stuff + # you don't have addressof() in rctypes so I looked up the implementation + # of addressof in ctypes source and come up with this. + pythonapi.PyLong_FromVoidPtr.argtypes = [c_char_p] + pythonapi.PyLong_FromVoidPtr.restype = c_int + # we also need to alias msync to take a c_void_p instead of c_char_p + linux_msync = libc["msync"] + linux_msync.argtypes = [c_void_p, size_t, c_int] + linux_msync.restype = c_int + + libc.memmove.argtypes = [c_char_p, c_char_p, size_t] + libc.memmove.restype = c_void_p + has_mremap = False + if hasattr(libc, "mremap"): + libc.mremap.argtypes = [POINTER(c_char), size_t, size_t, c_ulong] + libc.mremap.restype = c_void_p + has_mremap = True + libc.ftruncate.argtypes = [c_int, off_t] + libc.ftruncate.restype = c_int + def _get_page_size(): return libc.getpagesize() def _get_error_msg(): errno = geterrno() return libc.strerror(errno) -# elif _MS_WINDOWS: -# from ctypes import wintypes -# _LPVOID = c_void_p -# _LPCVOID = _LPVOID -# _DWORD_PTR = wintypes.DWORD -# _INVALID_HANDLE_VALUE = wintypes.HANDLE(-1).value -# -# _PAGE_READONLY = 0x02 -# _PAGE_READWRITE = 0x04 -# _PAGE_WRITECOPY = 0x08 -# -# _FILE_MAP_READ = 0x0004 -# _FILE_MAP_WRITE = 0x0002 -# _FILE_MAP_COPY = 0x0001 -# -# -# class _STRUCT(Structure): -# _fields_ = [("wProcessorArchitecture", wintypes.WORD), -# ("wReserved", wintypes.WORD)] -# -# class _UNION(Union): -# _fields_ = [("dwOemId", wintypes.DWORD), -# ("struct", _STRUCT)] -# -# class _SYSTEM_INFO(Structure): -# _fields_ = [("union", _UNION), -# ("dwPageSize", wintypes.DWORD), -# ("lpMinimumApplicationAddress", _LPVOID), -# ("lpMaximumApplicationAddress", _LPVOID), -# ("dwActiveProcessorMask", _DWORD_PTR), -# ("dwNumberOfProcessors", wintypes.DWORD), -# ("dwProcessorType", wintypes.DWORD), -# ("dwAllocationGranularity", wintypes.DWORD), -# ("wProcessorLevel", wintypes.WORD), -# ("wProcessorRevision", wintypes.WORD)] -# -# def _get_page_size(): -# si = _SYSTEM_INFO() -# windll.kernel32.GetSystemInfo.argtypes = [POINTER(_SYSTEM_INFO)] -# windll.kernel32.GetSystemInfo(byref(si)) -# return int(si.dwPageSize) -# -# def _get_file_size(handle): -# windll.kernel32.GetFileSize.restype = wintypes.DWORD -# low = wintypes.DWORD() -# high = wintypes.DWORD() -# low = wintypes.DWORD(windll.kernel32.GetFileSize(handle, byref(high))) -# # low might just happen to have the value INVALID_FILE_SIZE -# # so we need to check the last error also -# INVALID_FILE_SIZE = wintypes.DWORD(0xFFFFFFFF).value -# NO_ERROR = 0 -# dwErr = GetLastError() -# if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: -# raise WinError(dwErr) -# return low, high -# -# def _get_error_msg(): -# cdll.msvcrt.strerror.restype = c_char_p -# msg = cdll.msvcrt.strerror(GetLastError()) -# return msg -# -# windll.kernel32.CloseHandle.argtypes = [wintypes.HANDLE] -# windll.kernel32.CloseHandle.restype = wintypes.BOOL +elif _MS_WINDOWS: + from ctypes import wintypes + + WORD = wintypes.WORD + DWORD = wintypes.DWORD + BOOL = wintypes.BOOL + LPVOID = c_void_p + LPCVOID = LPVOID + DWORD_PTR = DWORD + HANDLE = wintypes.HANDLE + INVALID_HANDLE_VALUE = HANDLE(-1).value + + class SYSINFO_STRUCT(Structure): + _fields_ = [("wProcessorArchitecture", WORD), + ("wReserved", WORD)] + + class SYSINFO_UNION(Union): + _fields_ = [("dwOemId", DWORD), + ("struct", SYSINFO_STRUCT)] + + class SYSTEM_INFO(Structure): + _fields_ = [("union", SYSINFO_UNION), + ("dwPageSize", DWORD), + ("lpMinimumApplicationAddress", LPVOID), + ("lpMaximumApplicationAddress", LPVOID), + ("dwActiveProcessorMask", DWORD_PTR), + ("dwNumberOfProcessors", DWORD), + ("dwProcessorType", DWORD), + ("dwAllocationGranularity", DWORD), + ("wProcessorLevel", WORD), + ("wProcessorRevision", WORD)] + + windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] + windll.kernel32.GetFileSize.restype = DWORD + GetCurrentProcess = windll.kernel32.GetCurrentProcess + GetCurrentProcess.restype = HANDLE + DuplicateHandle = windll.kernel32.DuplicateHandle + DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, POINTER(HANDLE), DWORD, + BOOL, DWORD] + DuplicateHandle.restype = BOOL + CreateFileMapping = windll.kernel32.CreateFileMappingA + CreateFileMapping.argtypes = [HANDLE, c_void_p, DWORD, DWORD, DWORD, + c_char_p] + CreateFileMapping.restype = HANDLE + MapViewOfFile = windll.kernel32.MapViewOfFile + MapViewOfFile.argtypes = [HANDLE, DWORD, DWORD, DWORD, DWORD] + MapViewOfFile.restype = c_void_p + CloseHandle = windll.kernel32.CloseHandle + CloseHandle.argtypes = [HANDLE] + CloseHandle.restype = BOOL + UnmapViewOfFile = windll.kernel32.UnmapViewOfFile + UnmapViewOfFile.argtypes = [LPCVOID] + UnmapViewOfFile.restype = BOOL + + def _get_page_size(): + si = SYSTEM_INFO() + windll.kernel32.GetSystemInfo(byref(si)) + return int(si.dwPageSize) + + def _get_file_size(space, handle): + low = DWORD() + high = DWORD() + low = DWORD(windll.kernel32.GetFileSize(handle, byref(high))) + # low might just happen to have the value INVALID_FILE_SIZE + # so we need to check the last error also + INVALID_FILE_SIZE = DWORD(0xFFFFFFFF).value + NO_ERROR = 0 + dwErr = GetLastError() + if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: + raise OperationError(space.wrap(WinError), space.wrap(dwErr)) + return low, high + + def _get_error_msg(): + errno = GetLastError() + return libc.strerror(GetLastError()) PAGESIZE = _get_page_size() class _mmap(Wrappable): def __init__(self, space): self.space = space - #self._data = self._size = 0 self._pos = 0 self._access = _ACCESS_DEFAULT - self._closed = False - # if _MS_WINDOWS: - # self._map_handle = wintypes.HANDLE() - # self._file_handle = wintypes.HANDLE() - # self._tagname = None - if _POSIX: + if _MS_WINDOWS: + self._map_handle = wintypes.HANDLE() + self._file_handle = wintypes.HANDLE() + self._tagname = "" + elif _POSIX: self._fd = 0 + self._closed = False def _to_str(self): str = "".join([self._data[i] for i in range(self._size)]) @@ -185,12 +209,13 @@ _to_str.unwrap_spec = ['self'] def _check_valid(self): - # if _MS_WINDOWS: - # if self._map_handle.value == _INVALID_HANDLE_VALUE: - # raise ValueError, "map closed or invalid" - if _POSIX: - if self._closed: - raise OperationError(self.space.w_ValueError, + if _MS_WINDOWS: + to_close = self._map_handle.value == INVALID_HANDLE_VALUE + elif _POSIX: + to_close = self._closed + + if to_close: + raise OperationError(self.space.w_ValueError, self.space.wrap("map closed or invalid")) _check_valid.unwrap_spec = ['self'] @@ -208,17 +233,17 @@ _check_resizeable.unwrap_spec = ['self'] def close(self): - # if _MS_WINDOWS: - # if self._data: - # self._unmapview() - # self._data = None - # if self._map_handle.value != _INVALID_HANDLE_VALUE: - # windll.kernel32.CloseHandle(self._map_handle) - # self._map_handle.value = _INVALID_HANDLE_VALUE - # if self._file_handle.value != _INVALID_HANDLE_VALUE: - # windll.kernel32.CloseHandle(self._file_handle) - # self._file_handle.value = _INVALID_HANDLE_VALUE - if _POSIX: + if _MS_WINDOWS: + if self._data: + self._unmapview() + self._data = None + if self._map_handle.value != INVALID_HANDLE_VALUE: + CloseHandle(self._map_handle) + self._map_handle.value = INVALID_HANDLE_VALUE + if self._file_handle.value != INVALID_HANDLE_VALUE: + CloseHandle(self._file_handle) + self._file_handle.value = INVALID_HANDLE_VALUE + elif _POSIX: self._closed = True libc.close(self._fd) self._fd = -1 @@ -226,6 +251,10 @@ libc.munmap(self._data, self._size) close.unwrap_spec = ['self'] + def _unmapview(self): + self._data = cast(self._data, c_void_p) + UnmapViewOfFile(self._data) + def read_byte(self): self._check_valid() @@ -610,8 +639,7 @@ __delitem__ = interp2app(_mmap.__delitem__, unwrap_spec=_mmap.__delitem__.unwrap_spec), __add__ = interp2app(_mmap.__add__, unwrap_spec=_mmap.__add__.unwrap_spec), - __mul__ = interp2app(_mmap.__mul__, unwrap_spec=_mmap.__mul__.unwrap_spec), - + __mul__ = interp2app(_mmap.__mul__, unwrap_spec=_mmap.__mul__.unwrap_spec), ) def _check_map_size(space, size): @@ -694,147 +722,102 @@ return space.wrap(m) mmap.unwrap_spec = [ObjSpace, int, int, int, int, int] -# elif _MS_WINDOWS: -# def mmap(fileno, length, *args, **kwargs): -# size_hi = wintypes.DWORD() # upper 32 bits of m._size -# size_lo = wintypes.DWORD() # lower 32 bits of m._size -# tagname = "" -# dwErr = wintypes.DWORD(0) -# fh = wintypes.HANDLE(0) -# access = _ACCESS_DEFAULT -# flProtect = wintypes.DWORD() -# dwDesiredAccess = wintypes.DWORD() -# keywords = ["tagname", "access"] -# _check_args(fileno, length, 4, keywords, args, kwargs) -# -# try: -# tagname = args[0] -# except IndexError: -# try: -# tagname = kwargs[keywords[0]] -# except KeyError: -# pass -# if not _is_str(tagname) or tagname: -# raise TypeError, "tagname must be string or None" -# -# try: -# access = args[1] -# except IndexError: -# try: -# access = kwargs[keywords[1]] -# except KeyError: -# pass -# if not _is_int(access): -# raise TypeError, "an integer is required" -# -# if access == ACCESS_READ: -# flProtect = wintypes.DWORD(_PAGE_READONLY) -# dwDesiredAccess = wintypes.DWORD(_FILE_MAP_READ) -# elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: -# flProtect = wintypes.DWORD(_PAGE_READWRITE) -# dwDesiredAccess = wintypes.DWORD(_FILE_MAP_WRITE) -# elif access == ACCESS_COPY: -# flProtect = wintypes.DWORD(_PAGE_WRITECOPY) -# dwDesiredAccess = wintypes.DWORD(_FILE_MAP_COPY) -# else: -# raise ValueError, "mmap invalid access parameter." -# -# # check size boundaries -# _check_map_size(length) -# map_size = length -# -# # assume -1 and 0 both mean invalid filedescriptor -# # to 'anonymously' map memory. -# if fileno != -1 and fileno != 0: -# fh = cdll.msvcr71._get_osfhandle(fileno) -# if fh == -1: -# raise error, _get_error_msg() -# # Win9x appears to need us seeked to zero -# SEEK_SET = 0 -# cdll.msvcrt._lseek(fileno, 0, SEEK_SET) -# -# m = _mmap() -# m._file_handle = wintypes.HANDLE(_INVALID_HANDLE_VALUE) -# m._map_handle = wintypes.HANDLE(_INVALID_HANDLE_VALUE) -# -# if fh: -# # it is necessary to duplicate the handle, so the -# # Python code can close it on us -# DUPLICATE_SAME_ACCESS = 0x00000002 -# GetCurrentProcess = windll.kernel32.GetCurrentProcess -# GetCurrentProcess.restype = wintypes.HANDLE -# DuplicateHandle = windll.kernel32.DuplicateHandle -# DuplicateHandle.argtypes = [wintypes.HANDLE, -# wintypes.HANDLE, -# wintypes.HANDLE, -# POINTER(wintypes.HANDLE), -# wintypes.DWORD, -# wintypes.BOOL, -# wintypes.DWORD] -# DuplicateHandle.restype = wintypes.BOOL -# -# res = DuplicateHandle(GetCurrentProcess(), # source process handle -# fh, # handle to be duplicated -# GetCurrentProcess(), # target process handle -# byref(m._file_handle), # result -# 0, # access - ignored due to options value -# wintypes.BOOL(False), # inherited by child procs? -# DUPLICATE_SAME_ACCESS) # options -# if not res: -# raise WinError() -# -# if not map_size: -# low, high = _get_file_size(fh) -# if _64BIT: -# m._size = (c_long(low.value).value << 32) + 1 -# else: -# if high: -# # file is too large to map completely -# m._size = -1L -# else: -# m._size = low.value -# else: -# m._size = map_size -# else: -# m._size = map_size -# -# # set the tag name -# if tagname: -# m._tagname = tagname -# -# m._access = access -# -# # DWORD is a 4-byte int. If int > 4-byte it must be divided -# if _64BIT: -# size_hi = wintypes.DWORD(m._size >> 32) -# size_lo = wintypes.DWORD(m._size & 0xFFFFFFFF) -# else: -# size_hi = wintypes.DWORD(0) -# size_lo = wintypes.DWORD(m._size) -# -# CreateFileMapping = windll.kernel32.CreateFileMappingA -# CreateFileMapping.argtypes = [wintypes.HANDLE, c_void_p, wintypes.DWORD, -# wintypes.DWORD, wintypes.DWORD, c_char_p] -# CreateFileMapping.restype = wintypes.HANDLE -# -# m._map_handle = wintypes.HANDLE(CreateFileMapping(m._file_handle, None, flProtect, -# size_hi, size_lo, m._tagname)) -# -# if m._map_handle: -# MapViewOfFile = windll.kernel32.MapViewOfFile -# MapViewOfFile.argtypes = [wintypes.HANDLE, wintypes.DWORD, -# wintypes.DWORD, wintypes.DWORD, -# wintypes.DWORD] -# MapViewOfFile.restype = c_void_p -# m._data = MapViewOfFile(m._map_handle, dwDesiredAccess, -# 0, 0, 0) -# if m._data: -# m._data = cast(m._data, POINTER(c_char)) -# return m -# else: -# dwErr = GetLastError() -# else: -# dwErr = GetLastError() -# -# raise WinError(dwErr) +elif _MS_WINDOWS: + def mmap(space, fileno, length, tagname="", access=_ACCESS_DEFAULT): + # check size boundaries + _check_map_size(space, length) + map_size = length + + WORD = wintypes.WORD + DWORD = wintypes.DWORD + HANDLE = wintypes.HANDLE + + flProtect = WORD() + dwDesiredAccess = WORD() + fh = HANDLE(0) + + if access == ACCESS_READ: + flProtect = DWORD(PAGE_READONLY) + dwDesiredAccess = DWORD(FILE_MAP_READ) + elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: + flProtect = DWORD(PAGE_READWRITE) + dwDesiredAccess = DWORD(FILE_MAP_WRITE) + elif access == ACCESS_COPY: + flProtect = DWORD(PAGE_WRITECOPY) + dwDesiredAccess = DWORD(FILE_MAP_COPY) + else: + raise OperationError(space.w_ValueError, + space.wrap("mmap invalid access parameter.")) + + # assume -1 and 0 both mean invalid file descriptor + # to 'anonymously' map memory. + if fileno != -1 and fileno != 0: + fh = cdll.msvcr71._get_osfhandle(fileno) + if fh == -1: + raise OperationError(space.w_EnvironmentError, + space.wrap(_get_error_msg())) + # Win9x appears to need us seeked to zero + SEEK_SET = 0 + libc._lseek(fileno, 0, SEEK_SET) + + m = _mmap(space) + m._file_handle = HANDLE(INVALID_HANDLE_VALUE) + m._map_handle = HANDLE(INVALID_HANDLE_VALUE) + + if fh: + res = BOOL() + # it is necessary to duplicate the handle, so the + # Python code can close it on us + res = DuplicateHandle(GetCurrentProcess(), # source process handle + fh, # handle to be duplicated + GetCurrentProcess(), # target process handle + byref(m._file_handle), # result + 0, # access - ignored due to options value + wintypes.BOOL(False), # inherited by child procs? + DUPLICATE_SAME_ACCESS) # options + if not res: + raise OperationError(space.wrap(WinError), space.wrap("")) + + if not map_size: + low, high = _get_file_size(fh) + if _64BIT: + m._size = (c_long(low.value).value << 32) + 1 + else: + if high: + # file is too large to map completely + m._size = -1L + else: + m._size = low.value + else: + m._size = map_size + else: + m._size = map_size + + if tagname: + m._tagname = tagname + m._access = access + + # DWORD is a 4-byte int. If int > 4-byte it must be divided + if _64BIT: + size_hi = DWORD(m._size >> 32) + size_lo = DWORD(m._size & 0xFFFFFFFF) + else: + size_hi = DWORD(0) + size_lo = DWORD(m._size) + + m._map_handle = HANDLE(CreateFileMapping(m._file_handle, None, flProtect, + size_hi, size_lo, m._tagname)) + + if m._map_handle: + m._data = MapViewOfFile(m._map_handle, dwDesiredAccess, + 0, 0, 0) + if m._data: + m._data = cast(m._data, POINTER(c_char)) + return m + else: + dwErr = GetLastError() + else: + dwErr = GetLastError() + raise OperationError(space.wrap(WinError), space.wrap(dwErr)) + mmap.unwrap_spec = [ObjSpace, int, int, str, int] Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Sat Aug 19 17:57:49 2006 @@ -1,13 +1,13 @@ from pypy.conftest import gettestobjspace import os -if os.name == "nt": - from py.test import skip - skip("Windows is not supported") - def teardown_module(mod): - if os.path.exists("foo"): - os.unlink("foo") + for p in "abcde": + if os.path.exists(p): + try: + os.unlink(p) + except OSError: + print "Cannot delete '%s'" % p class AppTestMMap: def setup_class(cls): @@ -21,16 +21,18 @@ def test_attributes(self): import mmap + import os assert isinstance(mmap.ACCESS_READ, int) assert isinstance(mmap.ACCESS_WRITE, int) assert isinstance(mmap.ACCESS_COPY, int) - assert isinstance(mmap.MAP_ANON, int) - assert isinstance(mmap.MAP_ANONYMOUS, int) - assert isinstance(mmap.MAP_PRIVATE, int) - assert isinstance(mmap.MAP_SHARED, int) - assert isinstance(mmap.PROT_EXEC, int) - assert isinstance(mmap.PROT_READ, int) - assert isinstance(mmap.PROT_WRITE, int) + if os.name == "posix": + assert isinstance(mmap.MAP_ANON, int) + assert isinstance(mmap.MAP_ANONYMOUS, int) + assert isinstance(mmap.MAP_PRIVATE, int) + assert isinstance(mmap.MAP_SHARED, int) + assert isinstance(mmap.PROT_EXEC, int) + assert isinstance(mmap.PROT_READ, int) + assert isinstance(mmap.PROT_WRITE, int) assert mmap.error is EnvironmentError @@ -50,17 +52,19 @@ raises(OverflowError, mmap, 0, sys.maxint) raises(ValueError, mmap, 0, 1, flags=2, access=3) raises(ValueError, mmap, 0, 1, access=123) - # elif _MS_WINDOWS: - # py.test.raises(TypeError, mmap, 0, 1, 2, 3, 4) - # py.test.raises(TypeError, mmap, 0, 1, tagname=123) - # py.test.raises(TypeError, mmap, 0, 1, access="foo") - # py.test.raises(ValueError, mmap, 0, 1, access=-1) + elif os.name == "nt": + raises(TypeError, mmap, 0, 1, 2, 3, 4) + raises(TypeError, mmap, 0, 1, tagname=123) + raises(TypeError, mmap, 0, 1, access="foo") + raises(ValueError, mmap, 0, 1, access=-1) def test_file_size(self): - # if _MS_WINDOWS: - # py.test.skip("Only Unix checks file size") + import os + if os.name == "nt": + skip("Only Unix checks file size") + from mmap import mmap - f = open("foo", "w+") + f = open("a", "w+") f.write("c") f.flush() @@ -69,7 +73,7 @@ def test_mmap_creation(self): from mmap import mmap - f = open("foo", "w+") + f = open("b", "w+") f.write("c") f.flush() @@ -80,7 +84,7 @@ def test_close(self): from mmap import mmap - f = open("foo", "w+") + f = open("c", "w+") f.write("c") f.flush() @@ -90,7 +94,7 @@ def test_read_byte(self): from mmap import mmap - f = open("foo", "w+") + f = open("d", "w+") f.write("c") f.flush() @@ -103,7 +107,7 @@ def test_readline(self): from mmap import mmap import os - f = open("foo", "w+") + f = open("e", "w+") f.write("foo\n") f.flush() From rhymes at codespeak.net Sat Aug 19 18:23:03 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 18:23:03 +0200 (CEST) Subject: [pypy-svn] r31409 - in pypy/dist/pypy/module/mmap: . test Message-ID: <20060819162303.3DFBD1006F@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 18:23:01 2006 New Revision: 31409 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/module/mmap/test/test_mmap.py Log: various fixes. Now the porting of the mmap module in Windows is completed. Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Aug 19 18:23:01 2006 @@ -116,6 +116,7 @@ WORD = wintypes.WORD DWORD = wintypes.DWORD BOOL = wintypes.BOOL + LONG = wintypes.LONG LPVOID = c_void_p LPCVOID = LPVOID DWORD_PTR = DWORD @@ -163,6 +164,12 @@ UnmapViewOfFile = windll.kernel32.UnmapViewOfFile UnmapViewOfFile.argtypes = [LPCVOID] UnmapViewOfFile.restype = BOOL + FlushViewOfFile = windll.kernel32.FlushViewOfFile + FlushViewOfFile.argtypes = [c_void_p, c_int] + SetFilePointer = windll.kernel32.SetFilePointer + SetFilePointer.argtypes = [HANDLE, LONG, POINTER(LONG), DWORD] + SetEndOfFile = windll.kernel32.SetEndOfFile + SetEndOfFile.argtypes = [HANDLE] def _get_page_size(): si = SYSTEM_INFO() @@ -358,16 +365,16 @@ def size(self): self._check_valid() - # if _MS_WINDOWS: - # if self._file_handle.value != _INVALID_HANDLE_VALUE: - # low, high = _get_file_size(self._file_handle) - # if not high and low.value < sys.maxint: - # return int(low.value) - # size = (long(high.value) << 32) + low.value - # return long(size) - # else: - # return self._size - if _POSIX: + if _MS_WINDOWS: + if self._file_handle.value != INVALID_HANDLE_VALUE: + low, high = _get_file_size(self.space, self._file_handle) + if not high and low.value < sys.maxint: + return self.space.wrap(int(low.value)) + size = (long(high.value) << 32) + low.value + return self.space.wrap(long(size)) + else: + return self.space.wrap(self._size) + elif _POSIX: st = os.fstat(self._fd) SIZE_BIT = 6 return self.space.wrap(st[SIZE_BIT]) @@ -417,12 +424,9 @@ else: data = c_char_p("".join([self._data[i] for i in range(offset, size)])) - # if _MS_WINDOWS: - # FlushViewOfFile = windll.kernel32.FlushViewOfFile - # FlushViewOfFile.argtypes = [c_void_p, c_int] - # res = FlushViewOfFile(data, size) - # return res - if _POSIX: + if _MS_WINDOWS: + return self.space.wrap(FlushViewOfFile(data, size)) + elif _POSIX: if _LINUX: # alignment of the address value = pythonapi.PyLong_FromVoidPtr(data) @@ -483,61 +487,44 @@ MREMAP_MAYMOVE = 1 libc.mremap(self._data, self._size, newsize, MREMAP_MAYMOVE) self._size = newsize - # elif _MS_WINDOWS: - # # disconnect the mapping - # self._unmapview() - # windll.kernel32.CloseHandle(self._map_handle) - # - # # move to the desired EOF position - # if _64BIT: - # newsize_high = wintypes.DWORD(newsize >> 32) - # newsize_low = wintypes.DWORD(newsize & 0xFFFFFFFF) - # else: - # newsize_high = wintypes.DWORD(0) - # newsize_low = wintypes.DWORD(newsize) - # - # FILE_BEGIN = wintypes.DWORD(0) - # SetFilePointer = windll.kernel32.SetFilePointer - # SetFilePointer.argtypes = [wintypes.HANDLE, wintypes.LONG, - # POINTER(wintypes.LONG), wintypes.DWORD] - # SetFilePointer(self._file_handle, - # wintypes.LONG(newsize_low.value), - # byref(wintypes.LONG(newsize_high.value)), - # FILE_BEGIN) - # # resize the file - # SetEndOfFile = windll.kernel32.SetEndOfFile - # SetEndOfFile.argtypes = [wintypes.HANDLE] - # SetEndOfFile(self._file_handle) - # # create another mapping object and remap the file view - # CreateFileMapping = windll.kernel32.CreateFileMappingA - # CreateFileMapping.argtypes = [wintypes.HANDLE, c_void_p, wintypes.DWORD, - # wintypes.DWORD, wintypes.DWORD, c_char_p] - # CreateFileMapping.restype = wintypes.HANDLE - # - # self._map_handle = wintypes.HANDLE(CreateFileMapping(self._file_handle, None, - # _PAGE_READWRITE, - # newsize_high, newsize_low, - # self._tagname)) - # - # dwErrCode = wintypes.DWORD(0) - # if self._map_handle: - # MapViewOfFile = windll.kernel32.MapViewOfFile - # MapViewOfFile.argtypes = [wintypes.HANDLE, wintypes.DWORD, - # wintypes.DWORD, wintypes.DWORD, - # wintypes.DWORD] - # MapViewOfFile.restype = c_void_p - # self._data = MapViewOfFile(self._map_handle, _FILE_MAP_WRITE, - # 0, 0, 0) - # if self._data: - # self._data = cast(self._data, POINTER(c_char)) - # self._size = newsize - # return - # else: - # dwErrCode = GetLastError() - # else: - # dwErrCode = GetLastError() - # - # raise WinError(dwErrCode) + elif _MS_WINDOWS: + # disconnect the mapping + self._unmapview() + CloseHandle(self._map_handle) + + # move to the desired EOF position + if _64BIT: + newsize_high = DWORD(newsize >> 32) + newsize_low = DWORD(newsize & 0xFFFFFFFF) + else: + newsize_high = DWORD(0) + newsize_low = DWORD(newsize) + + FILE_BEGIN = DWORD(0) + SetFilePointer(self._file_handle, LONG(newsize_low.value), + LONG(newsize_high.value), FILE_BEGIN) + # resize the file + SetEndOfFile(self._file_handle) + # create another mapping object and remap the file view + res = CreateFileMapping(self._file_handle, None, PAGE_READWRITE, + newsize_high, newsize_low, self._tagname) + self._map_handle = HANDLE(res) + + dwErrCode = DWORD(0) + if self._map_handle: + self._data = MapViewOfFile(self._map_handle, FILE_MAP_WRITE, + 0, 0, 0) + if self._data: + self._data = cast(self._data, POINTER(c_char)) + self._size = newsize + return + else: + dwErrCode = GetLastError() + else: + dwErrCode = GetLastError() + + raise OperationError(self.space.wrap(WinError), + self.space.wrap(dwErrCode)) resize.unwrap_spec = ['self', int] def __len__(self): @@ -779,7 +766,7 @@ raise OperationError(space.wrap(WinError), space.wrap("")) if not map_size: - low, high = _get_file_size(fh) + low, high = _get_file_size(space, fh) if _64BIT: m._size = (c_long(low.value).value << 32) + 1 else: Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Sat Aug 19 18:23:01 2006 @@ -2,7 +2,8 @@ import os def teardown_module(mod): - for p in "abcde": + import string + for p in string.ascii_lowercase: if os.path.exists(p): try: os.unlink(p) @@ -112,10 +113,10 @@ f.write("foo\n") f.flush() m = mmap(f.fileno(), 4) - # if _MS_WINDOWS: - # # windows replaces \n with \r. it's time to change to \n only MS! - # assert m.readline() == "foo\r" - if os.name == "posix": + if os.name == "nt": + # windows replaces \n with \r. it's time to change to \n only MS! + assert m.readline() == "foo\r" + elif os.name == "posix": assert m.readline() == "foo\n" assert m.readline() == "" m.close() @@ -123,7 +124,7 @@ def test_read(self): from mmap import mmap - f = open("foo", "w+") + f = open("f", "w+") f.write("foobar") f.flush() @@ -137,7 +138,7 @@ def test_find(self): from mmap import mmap - f = open("foo", "w+") + f = open("g", "w+") f.write("foobar\0") f.flush() @@ -154,7 +155,7 @@ def test_is_modifiable(self): import mmap - f = open("foo", "w+") + f = open("h", "w+") f.write("foobar") f.flush() @@ -166,7 +167,7 @@ def test_seek(self): from mmap import mmap - f = open("foo", "w+") + f = open("i", "w+") f.write("foobar") f.flush() @@ -191,7 +192,7 @@ def test_write(self): import mmap - f = open("foo", "w+") + f = open("j", "w+") f.write("foobar") f.flush() @@ -207,7 +208,7 @@ def test_write_byte(self): import mmap - f = open("foo", "w+") + f = open("k", "w+") f.write("foobar") f.flush() @@ -223,7 +224,7 @@ def test_size(self): from mmap import mmap - f = open("foo", "w+") + f = open("l", "w+") f.write("foobar") f.flush() @@ -234,7 +235,7 @@ def test_tell(self): from mmap import mmap - f = open("foo", "w+") + f = open("m", "w+") f.write("c") f.flush() @@ -245,7 +246,7 @@ def test_flush(self): from mmap import mmap - f = open("foo", "w+") + f = open("n", "w+") f.write("foobar") f.flush() @@ -259,7 +260,7 @@ def test_move(self): import mmap - f = open("foo", "w+") + f = open("o", "w+") f.write("foobar") f.flush() @@ -286,7 +287,7 @@ import mmap import os - f = open("foo", "w+") + f = open("p", "w+") f.write("foobar") f.flush() m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) @@ -305,7 +306,7 @@ def test_len(self): from mmap import mmap - f = open("foo", "w+") + f = open("q", "w+") f.write("foobar") f.flush() @@ -317,7 +318,7 @@ def test_get_item(self): from mmap import mmap - f = open("foo", "w+") + f = open("r", "w+") f.write("foobar") f.flush() @@ -336,7 +337,7 @@ def test_set_item(self): import mmap - f = open("foo", "w+") + f = open("s", "w+") f.write("foobar") f.flush() @@ -371,7 +372,7 @@ def test_del_item(self): from mmap import mmap - f = open("foo", "w+") + f = open("t", "w+") f.write("foobar") f.flush() @@ -388,7 +389,7 @@ def test_concatenation(self): from mmap import mmap - f = open("foo", "w+") + f = open("u", "w+") f.write("foobar") f.flush() @@ -405,7 +406,7 @@ def test_repeatition(self): from mmap import mmap - f = open("foo", "w+") + f = open("v", "w+") f.write("foobar") f.flush() @@ -431,7 +432,7 @@ import sys import os - filename = "foo" + filename = "w" f = open(filename, "w+") From rhymes at codespeak.net Sat Aug 19 18:33:24 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 18:33:24 +0200 (CEST) Subject: [pypy-svn] r31410 - pypy/dist/pypy/module/mmap Message-ID: <20060819163324.1B2CC10050@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 18:33:22 2006 New Revision: 31410 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: small fixes. thanks pylint Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Aug 19 18:33:22 2006 @@ -191,7 +191,7 @@ def _get_error_msg(): errno = GetLastError() - return libc.strerror(GetLastError()) + return libc.strerror(errno) PAGESIZE = _get_page_size() @@ -211,8 +211,8 @@ self._closed = False def _to_str(self): - str = "".join([self._data[i] for i in range(self._size)]) - return self.space.wrap(str) + data = "".join([self._data[i] for i in range(self._size)]) + return self.space.wrap(data) _to_str.unwrap_spec = ['self'] def _check_valid(self): @@ -313,7 +313,7 @@ return self.space.wrap(res) read.unwrap_spec = ['self', int] - def find(self, str, start=0): + def find(self, tofind, start=0): self._check_valid() # since we don't have to update positions we @@ -321,7 +321,7 @@ w_str_data = self._to_str() str_data = self.space.str_w(w_str_data) assert start >= 0 - return self.space.wrap(str_data.find(str, start)) + return self.space.wrap(str_data.find(tofind, start)) find.unwrap_spec = ['self', str, int] def seek(self, pos, whence=0): @@ -380,17 +380,18 @@ return self.space.wrap(st[SIZE_BIT]) size.unwrap_spec = ['self'] - def write(self, str): + def write(self, data): self._check_valid() self._check_writeable() - if self._pos + len(str) > self._size: + data_len = len(data) + if self._pos + data_len > self._size: raise OperationError(self.space.w_ValueError, self.space.wrap("data out of range")) p = c_char_p(str) - libc.memcpy(self._data, p, len(str)) - self._pos += len(str) + libc.memcpy(self._data, p, data_len) + self._pos += data_len write.unwrap_spec = ['self', str] def write_byte(self, byte): @@ -488,42 +489,42 @@ libc.mremap(self._data, self._size, newsize, MREMAP_MAYMOVE) self._size = newsize elif _MS_WINDOWS: - # disconnect the mapping - self._unmapview() - CloseHandle(self._map_handle) - - # move to the desired EOF position - if _64BIT: - newsize_high = DWORD(newsize >> 32) - newsize_low = DWORD(newsize & 0xFFFFFFFF) - else: - newsize_high = DWORD(0) - newsize_low = DWORD(newsize) - - FILE_BEGIN = DWORD(0) - SetFilePointer(self._file_handle, LONG(newsize_low.value), + # disconnect the mapping + self._unmapview() + CloseHandle(self._map_handle) + + # move to the desired EOF position + if _64BIT: + newsize_high = DWORD(newsize >> 32) + newsize_low = DWORD(newsize & 0xFFFFFFFF) + else: + newsize_high = DWORD(0) + newsize_low = DWORD(newsize) + + FILE_BEGIN = DWORD(0) + SetFilePointer(self._file_handle, LONG(newsize_low.value), LONG(newsize_high.value), FILE_BEGIN) - # resize the file - SetEndOfFile(self._file_handle) - # create another mapping object and remap the file view - res = CreateFileMapping(self._file_handle, None, PAGE_READWRITE, + # resize the file + SetEndOfFile(self._file_handle) + # create another mapping object and remap the file view + res = CreateFileMapping(self._file_handle, None, PAGE_READWRITE, newsize_high, newsize_low, self._tagname) - self._map_handle = HANDLE(res) - - dwErrCode = DWORD(0) - if self._map_handle: - self._data = MapViewOfFile(self._map_handle, FILE_MAP_WRITE, - 0, 0, 0) - if self._data: - self._data = cast(self._data, POINTER(c_char)) - self._size = newsize - return - else: - dwErrCode = GetLastError() - else: - dwErrCode = GetLastError() - - raise OperationError(self.space.wrap(WinError), + self._map_handle = HANDLE(res) + + dwErrCode = DWORD(0) + if self._map_handle: + self._data = MapViewOfFile(self._map_handle, FILE_MAP_WRITE, + 0, 0, 0) + if self._data: + self._data = cast(self._data, POINTER(c_char)) + self._size = newsize + return + else: + dwErrCode = GetLastError() + else: + dwErrCode = GetLastError() + + raise OperationError(self.space.wrap(WinError), self.space.wrap(dwErrCode)) resize.unwrap_spec = ['self', int] @@ -715,10 +716,6 @@ _check_map_size(space, length) map_size = length - WORD = wintypes.WORD - DWORD = wintypes.DWORD - HANDLE = wintypes.HANDLE - flProtect = WORD() dwDesiredAccess = WORD() fh = HANDLE(0) From rhymes at codespeak.net Sat Aug 19 18:34:48 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 18:34:48 +0200 (CEST) Subject: [pypy-svn] r31411 - pypy/dist/pypy/module/mmap Message-ID: <20060819163448.B2AB81006F@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 18:34:47 2006 New Revision: 31411 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py Log: s/str/data in write() Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sat Aug 19 18:34:47 2006 @@ -389,7 +389,7 @@ raise OperationError(self.space.w_ValueError, self.space.wrap("data out of range")) - p = c_char_p(str) + p = c_char_p(data) libc.memcpy(self._data, p, data_len) self._pos += data_len write.unwrap_spec = ['self', str] From rhymes at codespeak.net Sat Aug 19 19:03:16 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sat, 19 Aug 2006 19:03:16 +0200 (CEST) Subject: [pypy-svn] r31412 - pypy/dist/pypy/module/rctime Message-ID: <20060819170316.47FCC1005A@code0.codespeak.net> Author: rhymes Date: Sat Aug 19 19:03:14 2006 New Revision: 31412 Modified: pypy/dist/pypy/module/rctime/__init__.py pypy/dist/pypy/module/rctime/interp_time.py Log: Small fixes. Now the rctime module compiles even on Windows. great :-) Modified: pypy/dist/pypy/module/rctime/__init__.py ============================================================================== --- pypy/dist/pypy/module/rctime/__init__.py (original) +++ pypy/dist/pypy/module/rctime/__init__.py Sat Aug 19 19:03:14 2006 @@ -15,7 +15,6 @@ 'gmtime': 'interp_time.gmtime', 'localtime': 'interp_time.localtime', 'mktime': 'interp_time.mktime', - 'tzset': 'interp_time.tzset', 'strftime': 'interp_time.strftime', } @@ -25,6 +24,7 @@ if os.name == "posix": Module.appleveldefs['sleep'] = 'app_time.sleep' + Module.interpleveldefs['tzset'] = 'interp_time.tzset' elif os.name == "nt": Module.interpleveldefs['sleep'] = 'interp_time.sleep' Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Aug 19 19:03:14 2006 @@ -38,6 +38,10 @@ ("tm_min", c_int), ("tm_hour", c_int), ("tm_mday", c_int), ("tm_mon", c_int), ("tm_year", c_int), ("tm_wday", c_int), ("tm_yday", c_int), ("tm_isdst", c_int)]) + + LARGE_INTEGER = wintypes.LARGE_INTEGER + BOOL = wintypes.BOOL + DWORD = wintypes.DWORD class cConfig: pass @@ -75,6 +79,13 @@ libc.asctime.restype = c_char_p if _POSIX: libc.tzset.restype = None # tzset() returns void +elif _WIN: + QueryPerformanceCounter = windll.kernel32.QueryPerformanceCounter + QueryPerformanceCounter.argtypes = [POINTER(c_int)] + QueryPerformanceCounter.restype = BOOL + Sleep = windll.kernel32.Sleep + Sleep.argtypes = [DWORD] + Sleep.restype = None libc.strftime.argtypes = [c_char_p, size_t, c_char_p, POINTER(tm)] libc.strftime.restype = size_t @@ -166,8 +177,8 @@ if msecs > float(sys.maxint * 2 - 1): # ULONG_MAX raise OperationError(space.w_OverflowError, space.wrap("sleep length is too large")) - ul_millis = c_ulong(long(msecs)) - windll.kernel32.Sleep(ul_millis) + ul_millis = c_ulong(int(msecs)) + Sleep(ul_millis) def sleep(space, w_secs): """sleep(seconds) @@ -177,8 +188,7 @@ secs = space.float_w(w_secs) _check_float(space, secs) - space.wrap(_float_sleep(space, secs)) - sleep.unwrap_spec = [ObjSpace, int] + _float_sleep(space, secs) def _check_float(space, seconds): # this call the app level _check_float to check the type of @@ -268,7 +278,6 @@ secs = _floattime() return space.wrap(secs) -time.unwrap_spec = [ObjSpace] def clock(space): """clock() -> floating point number @@ -282,21 +291,20 @@ return space.wrap(res) elif _WIN: divisor = 0.0 - ctrStart = wintypes.LARGE_INTEGER() - now = wintypes.LARGE_INTEGER() + ctrStart = c_int() + now = c_int() if divisor == 0.0: - freq = wintypes.LARGE_INTEGER() - windll.kernel32.QueryPerformanceCounter(byref(ctrStart)) - res = windll.kernel32.QueryPerformanceCounter(byref(freq)) + freq = c_int() + QueryPerformanceCounter(byref(ctrStart)) + res = QueryPerformanceCounter(byref(freq)) if not res or not freq: - return space.wrap(float(windll.msvcrt.clock())) + return space.wrap(float(libc.clock())) divisor = float(freq.value) - windll.kernel32.QueryPerformanceCounter(byref(now)) + QueryPerformanceCounter(byref(now)) diff = float(now.value - ctrStart.value) return space.wrap(float(diff / divisor)) -clock.unwrap_spec = [ObjSpace] def ctime(space, w_seconds=None): """ctime([seconds]) -> string @@ -544,4 +552,3 @@ i += i strftime.unwrap_spec = [ObjSpace, W_Root, W_Root] - From rhymes at codespeak.net Sun Aug 20 02:19:26 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 02:19:26 +0200 (CEST) Subject: [pypy-svn] r31413 - in pypy/dist/pypy/module/mmap: . test Message-ID: <20060820001926.D1ED810050@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 02:19:20 2006 New Revision: 31413 Modified: pypy/dist/pypy/module/mmap/interp_mmap.py pypy/dist/pypy/module/mmap/test/test_mmap.py Log: Made mmap compile on Windows also. Modified: pypy/dist/pypy/module/mmap/interp_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/interp_mmap.py (original) +++ pypy/dist/pypy/module/mmap/interp_mmap.py Sun Aug 20 02:19:20 2006 @@ -17,6 +17,7 @@ _LINUX = "linux" in sys.platform _64BIT = "64bit" in platform.architecture()[0] + class CConfig: _header_ = "#include " size_t = ctypes_platform.SimpleType("size_t", c_long) @@ -73,6 +74,8 @@ libc.strerror.argtypes = [c_int] libc.memcpy.argtypes = [POINTER(c_char), c_char_p, c_int] libc.memcpy.restype = c_void_p +libc.memmove.argtypes = [c_char_p, c_char_p, size_t] +libc.memmove.restype = c_void_p if _POSIX: libc.mmap.argtypes = [c_void_p, size_t, c_int, c_int, c_int, off_t] @@ -94,8 +97,6 @@ linux_msync.argtypes = [c_void_p, size_t, c_int] linux_msync.restype = c_int - libc.memmove.argtypes = [c_char_p, c_char_p, size_t] - libc.memmove.restype = c_void_p has_mremap = False if hasattr(libc, "mremap"): libc.mremap.argtypes = [POINTER(c_char), size_t, size_t, c_ulong] @@ -120,8 +121,8 @@ LPVOID = c_void_p LPCVOID = LPVOID DWORD_PTR = DWORD - HANDLE = wintypes.HANDLE - INVALID_HANDLE_VALUE = HANDLE(-1).value + c_int = wintypes.c_int + INVALID_c_int_VALUE = c_int(-1).value class SYSINFO_STRUCT(Structure): _fields_ = [("wProcessorArchitecture", WORD), @@ -144,32 +145,41 @@ ("wProcessorRevision", WORD)] windll.kernel32.GetSystemInfo.argtypes = [POINTER(SYSTEM_INFO)] - windll.kernel32.GetFileSize.restype = DWORD + GetFileSize = windll.kernel32.GetFileSize + GetFileSize.argtypes = [c_int, POINTER(c_int)] + GetFileSize.restype = c_int GetCurrentProcess = windll.kernel32.GetCurrentProcess - GetCurrentProcess.restype = HANDLE + GetCurrentProcess.restype = c_int DuplicateHandle = windll.kernel32.DuplicateHandle - DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, POINTER(HANDLE), DWORD, + DuplicateHandle.argtypes = [c_int, c_int, c_int, POINTER(c_int), DWORD, BOOL, DWORD] DuplicateHandle.restype = BOOL CreateFileMapping = windll.kernel32.CreateFileMappingA - CreateFileMapping.argtypes = [HANDLE, c_void_p, DWORD, DWORD, DWORD, + CreateFileMapping.argtypes = [c_int, c_void_p, c_int, c_int, c_int, c_char_p] - CreateFileMapping.restype = HANDLE + CreateFileMapping.restype = c_int MapViewOfFile = windll.kernel32.MapViewOfFile - MapViewOfFile.argtypes = [HANDLE, DWORD, DWORD, DWORD, DWORD] + MapViewOfFile.argtypes = [c_int, DWORD, DWORD, DWORD, DWORD] MapViewOfFile.restype = c_void_p CloseHandle = windll.kernel32.CloseHandle - CloseHandle.argtypes = [HANDLE] + CloseHandle.argtypes = [c_int] CloseHandle.restype = BOOL UnmapViewOfFile = windll.kernel32.UnmapViewOfFile UnmapViewOfFile.argtypes = [LPCVOID] UnmapViewOfFile.restype = BOOL FlushViewOfFile = windll.kernel32.FlushViewOfFile - FlushViewOfFile.argtypes = [c_void_p, c_int] + FlushViewOfFile.argtypes = [c_char_p, c_int] + FlushViewOfFile.restype = BOOL SetFilePointer = windll.kernel32.SetFilePointer - SetFilePointer.argtypes = [HANDLE, LONG, POINTER(LONG), DWORD] + SetFilePointer.argtypes = [c_int, c_int, POINTER(c_int), c_int] SetEndOfFile = windll.kernel32.SetEndOfFile - SetEndOfFile.argtypes = [HANDLE] + SetEndOfFile.argtypes = [c_int] + msvcr71 = cdll.LoadLibrary("msvcr71.dll") + msvcr71._get_osfhandle.argtypes = [c_int] + msvcr71._get_osfhandle.restype = c_int + # libc._lseek.argtypes = [c_int, c_int, c_int] + # libc._lseek.restype = c_int + def _get_page_size(): si = SYSTEM_INFO() @@ -177,20 +187,21 @@ return int(si.dwPageSize) def _get_file_size(space, handle): - low = DWORD() - high = DWORD() - low = DWORD(windll.kernel32.GetFileSize(handle, byref(high))) + high = c_int(0) + low = c_int(windll.kernel32.GetFileSize(c_int(handle.value), byref(high))) # low might just happen to have the value INVALID_FILE_SIZE # so we need to check the last error also - INVALID_FILE_SIZE = DWORD(0xFFFFFFFF).value + INVALID_FILE_SIZE = -1 NO_ERROR = 0 dwErr = GetLastError() if low.value == INVALID_FILE_SIZE and dwErr != NO_ERROR: - raise OperationError(space.wrap(WinError), space.wrap(dwErr)) - return low, high + raise OperationError(space.w_EnvironmentError, + space.wrap(_get_error_msg(dwErr))) + return low.value, high.value - def _get_error_msg(): - errno = GetLastError() + def _get_error_msg(errno=0): + if not errno: + errno = GetLastError() return libc.strerror(errno) PAGESIZE = _get_page_size() @@ -203,8 +214,8 @@ self._access = _ACCESS_DEFAULT if _MS_WINDOWS: - self._map_handle = wintypes.HANDLE() - self._file_handle = wintypes.HANDLE() + self._map_handle = wintypes.c_int() + self._file_handle = wintypes.c_int() self._tagname = "" elif _POSIX: self._fd = 0 @@ -217,7 +228,7 @@ def _check_valid(self): if _MS_WINDOWS: - to_close = self._map_handle.value == INVALID_HANDLE_VALUE + to_close = self._map_handle.value == INVALID_c_int_VALUE elif _POSIX: to_close = self._closed @@ -243,13 +254,13 @@ if _MS_WINDOWS: if self._data: self._unmapview() - self._data = None - if self._map_handle.value != INVALID_HANDLE_VALUE: + self._data = POINTER(c_char)() + if self._map_handle.value != INVALID_c_int_VALUE: CloseHandle(self._map_handle) - self._map_handle.value = INVALID_HANDLE_VALUE - if self._file_handle.value != INVALID_HANDLE_VALUE: + self._map_handle.value = INVALID_c_int_VALUE + if self._file_handle.value != INVALID_c_int_VALUE: CloseHandle(self._file_handle) - self._file_handle.value = INVALID_HANDLE_VALUE + self._file_handle.value = INVALID_c_int_VALUE elif _POSIX: self._closed = True libc.close(self._fd) @@ -259,8 +270,8 @@ close.unwrap_spec = ['self'] def _unmapview(self): - self._data = cast(self._data, c_void_p) - UnmapViewOfFile(self._data) + data = cast(self._data, c_void_p) + UnmapViewOfFile(data) def read_byte(self): self._check_valid() @@ -301,6 +312,7 @@ num_bytes = num # silently adjust out of range requests + assert self._pos >= 0; assert num_bytes >= 0; assert self._size >= 0 if self._pos + num_bytes > self._size: num_bytes -= (self._pos + num_bytes) - self._size @@ -349,6 +361,7 @@ raise OperationError(self.space.w_ValueError, self.space.wrap("unknown seek type")) + assert where >= 0 if where > self._size: raise OperationError(self.space.w_ValueError, self.space.wrap("seek out of range")) @@ -366,12 +379,12 @@ self._check_valid() if _MS_WINDOWS: - if self._file_handle.value != INVALID_HANDLE_VALUE: + if self._file_handle.value != INVALID_c_int_VALUE: low, high = _get_file_size(self.space, self._file_handle) - if not high and low.value < sys.maxint: - return self.space.wrap(int(low.value)) - size = (long(high.value) << 32) + low.value - return self.space.wrap(long(size)) + if not high and low < sys.maxint: + return self.space.wrap(low) + size = c_int((high << 32) + low).value + return self.space.wrap(size) else: return self.space.wrap(self._size) elif _POSIX: @@ -419,6 +432,8 @@ if size == 0: size = self._size + assert offset >= 0 + assert size >= 0 if offset + size > self._size: raise OperationError(self.space.w_ValueError, self.space.wrap("flush values out of range")) @@ -426,7 +441,8 @@ data = c_char_p("".join([self._data[i] for i in range(offset, size)])) if _MS_WINDOWS: - return self.space.wrap(FlushViewOfFile(data, size)) + res = FlushViewOfFile(data, size) + return self.space.wrap(res) elif _POSIX: if _LINUX: # alignment of the address @@ -451,13 +467,14 @@ self._check_writeable() # check boundings + assert src >= 0; assert dest >= 0; assert count >= 0; assert self._size >= 0 if (src + count > self._size) or (dest + count > self._size): raise OperationError(self.space.w_ValueError, self.space.wrap("source or destination out of range")) data_dest = c_char_p("".join([self._data[i] for i in range(dest, self._size)])) data_src = c_char_p("".join([self._data[i] for i in range(src, src+count)])) - libc.memmove(data_dest, data_src, count) + libc.memmove(data_dest, data_src, size_t(count)) assert dest >= 0 str_left = self.space.str_w(self._to_str())[0:dest] @@ -498,25 +515,25 @@ newsize_high = DWORD(newsize >> 32) newsize_low = DWORD(newsize & 0xFFFFFFFF) else: - newsize_high = DWORD(0) - newsize_low = DWORD(newsize) + newsize_high = c_int(0) + newsize_low = c_int(newsize) - FILE_BEGIN = DWORD(0) - SetFilePointer(self._file_handle, LONG(newsize_low.value), - LONG(newsize_high.value), FILE_BEGIN) + FILE_BEGIN = c_int(0) + SetFilePointer(self._file_handle, newsize_low, byref(newsize_high), + FILE_BEGIN) # resize the file SetEndOfFile(self._file_handle) # create another mapping object and remap the file view - res = CreateFileMapping(self._file_handle, None, PAGE_READWRITE, + res = CreateFileMapping(self._file_handle, c_void_p(0), PAGE_READWRITE, newsize_high, newsize_low, self._tagname) - self._map_handle = HANDLE(res) + self._map_handle = c_int(res) dwErrCode = DWORD(0) if self._map_handle: - self._data = MapViewOfFile(self._map_handle, FILE_MAP_WRITE, + data = MapViewOfFile(self._map_handle, FILE_MAP_WRITE, 0, 0, 0) - if self._data: - self._data = cast(self._data, POINTER(c_char)) + if data: + self._data = cast(data, POINTER(c_char)) self._size = newsize return else: @@ -524,8 +541,8 @@ else: dwErrCode = GetLastError() - raise OperationError(self.space.wrap(WinError), - self.space.wrap(dwErrCode)) + raise OperationError(self.space.w_EnvironmentError, + self.space.wrap(_get_error_msg(dwErrCode))) resize.unwrap_spec = ['self', int] def __len__(self): @@ -716,18 +733,18 @@ _check_map_size(space, length) map_size = length - flProtect = WORD() + flProtect = c_int() dwDesiredAccess = WORD() - fh = HANDLE(0) + fh = 0 if access == ACCESS_READ: - flProtect = DWORD(PAGE_READONLY) + flProtect = c_int(PAGE_READONLY) dwDesiredAccess = DWORD(FILE_MAP_READ) elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE: - flProtect = DWORD(PAGE_READWRITE) + flProtect = c_int(PAGE_READWRITE) dwDesiredAccess = DWORD(FILE_MAP_WRITE) elif access == ACCESS_COPY: - flProtect = DWORD(PAGE_WRITECOPY) + flProtect = c_int(PAGE_WRITECOPY) dwDesiredAccess = DWORD(FILE_MAP_COPY) else: raise OperationError(space.w_ValueError, @@ -736,17 +753,17 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = cdll.msvcr71._get_osfhandle(fileno) + fh = msvcr71._get_osfhandle(fileno) if fh == -1: raise OperationError(space.w_EnvironmentError, space.wrap(_get_error_msg())) # Win9x appears to need us seeked to zero - SEEK_SET = 0 - libc._lseek(fileno, 0, SEEK_SET) + # SEEK_SET = 0 + # libc._lseek(fileno, 0, SEEK_SET) m = _mmap(space) - m._file_handle = HANDLE(INVALID_HANDLE_VALUE) - m._map_handle = HANDLE(INVALID_HANDLE_VALUE) + m._file_handle = c_int(INVALID_c_int_VALUE) + m._map_handle = c_int(INVALID_c_int_VALUE) if fh: res = BOOL() @@ -760,18 +777,19 @@ wintypes.BOOL(False), # inherited by child procs? DUPLICATE_SAME_ACCESS) # options if not res: - raise OperationError(space.wrap(WinError), space.wrap("")) + raise OperationError(space.w_EnvironmentError, + space.wrap(_get_error_msg())) if not map_size: - low, high = _get_file_size(space, fh) + low, high = _get_file_size(space, c_int(fh)) if _64BIT: - m._size = (c_long(low.value).value << 32) + 1 + m._size = c_int((low << 32) + 1).value else: if high: # file is too large to map completely - m._size = -1L + m._size = -1 else: - m._size = low.value + m._size = low else: m._size = map_size else: @@ -786,22 +804,23 @@ size_hi = DWORD(m._size >> 32) size_lo = DWORD(m._size & 0xFFFFFFFF) else: - size_hi = DWORD(0) - size_lo = DWORD(m._size) + size_hi = c_int(0) + size_lo = c_int(m._size) - m._map_handle = HANDLE(CreateFileMapping(m._file_handle, None, flProtect, + m._map_handle = c_int(CreateFileMapping(m._file_handle, c_void_p(0), flProtect, size_hi, size_lo, m._tagname)) if m._map_handle: - m._data = MapViewOfFile(m._map_handle, dwDesiredAccess, - 0, 0, 0) - if m._data: - m._data = cast(m._data, POINTER(c_char)) - return m + res = MapViewOfFile(m._map_handle, dwDesiredAccess, + 0, 0, 0) + if res: + m._data = cast(res, POINTER(c_char)) + return space.wrap(m) else: dwErr = GetLastError() else: dwErr = GetLastError() - raise OperationError(space.wrap(WinError), space.wrap(dwErr)) + raise OperationError(space.w_EnvironmentError, + space.wrap(_get_error_msg(dwErr))) mmap.unwrap_spec = [ObjSpace, int, int, str, int] Modified: pypy/dist/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/dist/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/dist/pypy/module/mmap/test/test_mmap.py Sun Aug 20 02:19:20 2006 @@ -72,7 +72,7 @@ raises(ValueError, mmap, f.fileno(), 123) f.close() - def test_mmap_creation(self): + def test_create(self): from mmap import mmap f = open("b", "w+") From rhymes at codespeak.net Sun Aug 20 18:09:14 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 18:09:14 +0200 (CEST) Subject: [pypy-svn] r31415 - pypy/dist/pypy/module/posix/test Message-ID: <20060820160914.11D0810074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 18:09:12 2006 New Revision: 31415 Modified: pypy/dist/pypy/module/posix/test/test_posix2.py Log: more tests and fix test_fork Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 18:09:12 2006 @@ -95,8 +95,9 @@ assert isinstance(self.posix.strerror(0), str) assert isinstance(self.posix.strerror(1), str) - if hasattr(__import__(os.name), "fork"): - def test_fork(self): + def test_fork(self): + import os + if hasattr(__import__(os.name), "fork"): os = self.posix pid = os.fork() if pid == 0: # child @@ -104,6 +105,23 @@ pid1, status1 = os.waitpid(pid, 0) assert pid1 == pid # XXX check status1 + else: + skip("fork not supported") + + def test_read_write(self): + path = self.path + posix = self.posix + fd = posix.open(path, posix.O_WRONLY) + posix.write(fd, "\nfoo") + posix.close(fd) + fd = posix.open(path, posix.O_RDONLY) + raises(OSError, posix.write, fd, "foo") + buf = [] + buf.append(posix.read(fd, 4)) + assert len(buf[0]) == 4 + buf.append(posix.read(fd, 255)) + assert "".join(buf) == "\nfoo is a test" + posix.close(fd) class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 19:07:09 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 19:07:09 +0200 (CEST) Subject: [pypy-svn] r31416 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820170709.CF73510074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 19:07:05 2006 New Revision: 31416 Modified: pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: more tests and proper handling of IOError in ftruncate() Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 19:07:05 2006 @@ -17,6 +17,18 @@ space.wrap(errno), space.wrap(msg)) return OperationError(space.w_OSError, w_error) + +def wrap_ioerror(space, e): + assert isinstance(e, IOError) + errno = e.errno + try: + msg = os.strerror(errno) + except ValueError: + msg = 'error %d' % errno + w_error = space.call_function(space.w_IOError, + space.wrap(errno), + space.wrap(msg)) + return OperationError(space.w_IOError, w_error) def open(space, fname, flag, mode=0777): """Open a file (for low level IO). @@ -85,7 +97,9 @@ try: os.ftruncate(fd, length) except OSError, e: - raise wrap_oserror(space, e) + raise wrap_oserror(space, e) + except IOError, e: + raise wrap_ioerror(space, e) ftruncate.unwrap_spec = [ObjSpace, int, int] def build_stat_result(space, st): Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 19:07:05 2006 @@ -29,13 +29,16 @@ fd = posix.open(path, posix.O_RDONLY, 0777) fd2 = posix.dup(fd) assert not posix.isatty(fd2) + fd3 = 1 + posix.dup2(fd2, fd3) + assert not posix.isatty(fd3) s = posix.read(fd, 1) assert s == 't' posix.lseek(fd, 5, 0) s = posix.read(fd, 1) assert s == 'i' stat = posix.fstat(fd) - assert stat # XXX + assert stat # XXX posix.close(fd2) posix.close(fd) @@ -68,8 +71,16 @@ #UMPF cpython raises IOError ex(self.posix.ftruncate, UNUSEDFD, 123) ex(self.posix.fstat, UNUSEDFD) ex(self.posix.stat, "qweqwehello") + ex(self.posix.lstat, "qweqwehello") # how can getcwd() raise? ex(self.posix.dup, UNUSEDFD) + ex(self.posix.dup2, UNUSEDFD, UNUSEDFD) + ex(self.posix.unlink, str(UNUSEDFD)) + ex(self.posix.remove, str(UNUSEDFD)) + ex(self.posix.chdir, str(UNUSEDFD)) + ex(self.posix.rmdir, str(UNUSEDFD)) + ex(self.posix.listdir, str(UNUSEDFD)) + ex(self.posix.chmod, str(UNUSEDFD), 0777) def test_fdopen(self): path = self.path @@ -122,16 +133,89 @@ buf.append(posix.read(fd, 255)) assert "".join(buf) == "\nfoo is a test" posix.close(fd) + + def test_unlink(self): + import os + posix = self.posix + path = "foo" + fd = posix.open(path, posix.O_WRONLY | posix.O_CREAT) + assert os.path.exists(path) + try: + posix.unlink(path) + except OSError: + print "can't delete '%s'" % path + else: + assert not os.path.exists(path) + posix.close(fd) + test_remove = test_unlink + + def test_getcwd_chdir(self): + import os + posix = self.posix + path = os.path.split(posix.getcwd())[1] + posix.chdir('..') + posix.chdir(path) + posix.getcwd() + + def test_mkdir_rmdir(self): + import os + posix = self.posix + path = 'foo' + try: + posix.mkdir(path) + except OSError: + print "cannot create '%s' directory" % path + else: + assert os.path.exists(path) + try: + posix.rmdir(path) + except OSError: + print "cannot remove '%s' directory" % path + else: + assert not os.path.exists(path) + + def test_pipe(self): + posix = self.posix + r, w = posix.pipe() + data = 'foobar' + amount = posix.write(w, data) + posix.close(w) + read_data = posix.read(r, amount) + posix.close(r) + assert read_data == data + + def test_rename(self): + path = self.path + posix = self.posix + new_path = "foo" + posix.rename(path, new_path) + posix.rename(new_path, path) + + def test_ftruncate(self): + import os + pdir = self.pdir + posix = self.posix + path = os.path.join(pdir, 'file1') + fd = posix.open(path, posix.O_WRONLY) + posix.ftruncate(fd, 2) + assert posix.stat(path)[6] == 2 + posix.close(fd) + raises(IOError, posix.ftruncate, 123123, 1) + class AppTestEnvironment(object): def setup_class(cls): cls.space = space cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) cls.w_os = space.appexec([], "(): import os; return os") cls.w_path = space.wrap(str(path)) + def test_environ(self): posix = self.posix - os = self.os + assert posix.environ['PATH'] + del posix.environ['PATH'] + def fn(): posix.environ['PATH'] + raises(KeyError, fn) if hasattr(__import__(os.name), "unsetenv"): def test_unsetenv_nonexisting(self): From rhymes at codespeak.net Sun Aug 20 19:26:18 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 19:26:18 +0200 (CEST) Subject: [pypy-svn] r31417 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820172618.9931810074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 19:26:14 2006 New Revision: 31417 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: implemented abort and access Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 19:26:14 2006 @@ -44,6 +44,8 @@ 'chmod' : 'interp_posix.chmod', 'rename' : 'interp_posix.rename', '_exit' : 'interp_posix._exit', + 'abort' : 'interp_posix.abort', + 'access' : 'interp_posix.access', } if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 19:26:14 2006 @@ -384,12 +384,30 @@ def fork(space): pid = os.fork() return space.wrap(pid) +fork.unwrap_spec = [ObjSpace] +fork.__doc__ = os.fork.__doc__ def waitpid(space, pid, options): pid, status = os.waitpid(pid, options) return space.newtuple([space.wrap(pid), space.wrap(status)]) waitpid.unwrap_spec = [ObjSpace, int, int] +waitpid.__doc__ = os.waitpid.__doc__ def _exit(space, status): os._exit(status) _exit.unwrap_spec = [ObjSpace, int] +_exit.__doc__ = os._exit.__doc__ + +def abort(space): + os.abort() +abort.unwrap_spec = [ObjSpace] +abort.__doc__ = os.abort.__doc__ + +def access(space, path, mode): + try: + res = os.access(path, mode) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(res) +access.unwrap_spec = [ObjSpace, str, int] +access.__doc__ = os.access.__doc__ Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 19:26:14 2006 @@ -201,7 +201,16 @@ assert posix.stat(path)[6] == 2 posix.close(fd) raises(IOError, posix.ftruncate, 123123, 1) + + def test_abort(self): + posix = self.posix + pid = posix.fork() + if pid == 0: # child + posix.abort() + def test_access(self): + posix = self.posix + assert posix.access('.', posix.W_OK) class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 19:43:50 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 19:43:50 +0200 (CEST) Subject: [pypy-svn] r31418 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820174350.981B410074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 19:43:46 2006 New Revision: 31418 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: chown and chroot Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 19:43:46 2006 @@ -65,6 +65,10 @@ interpleveldefs['fork'] = 'interp_posix.fork' if hasattr(os, 'waitpid'): interpleveldefs['waitpid'] = 'interp_posix.waitpid' + if hasattr(os, 'chown'): + interpleveldefs['chown'] = 'interp_posix.chown' + if hasattr(os, 'chroot'): + interpleveldefs['chroot'] = 'interp_posix.chroot' for constant in dir(os): Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 19:43:46 2006 @@ -411,3 +411,19 @@ return space.wrap(res) access.unwrap_spec = [ObjSpace, str, int] access.__doc__ = os.access.__doc__ + +def chown(space, path, uid, gid): + try: + os.chown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e) +chown.unwrap_spec = [ObjSpace, str, int, int] +chown.__doc__ = os.chown.__doc__ + +def chroot(space, path): + try: + os.chroot(path) + except OSError, e: + raise wrap_oserror(space, e) +chroot.unwrap_spec = [ObjSpace, str] +chroot.__doc__ = os.chroot.__doc__ Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 19:43:46 2006 @@ -80,7 +80,9 @@ ex(self.posix.chdir, str(UNUSEDFD)) ex(self.posix.rmdir, str(UNUSEDFD)) ex(self.posix.listdir, str(UNUSEDFD)) - ex(self.posix.chmod, str(UNUSEDFD), 0777) + ex(self.posix.chmod, str(UNUSEDFD), 0777) + ex(self.posix.chown, str(UNUSEDFD), -1, -1) + ex(self.posix.chroot, str(UNUSEDFD)) def test_fdopen(self): path = self.path @@ -211,6 +213,18 @@ def test_access(self): posix = self.posix assert posix.access('.', posix.W_OK) + + def test_chown(self): + posix = self.posix + path = self.path + stat_info = posix.stat(path) + uid, gid = stat_info.st_uid, stat_info.st_gid + posix.chown(path, -1, -1) + stat_info = posix.stat(path) + assert uid == stat_info.st_uid + assert gid == stat_info.st_gid + raises(OSError, posix.chown, path, 1000, 1000) + class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 22:38:02 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 22:38:02 +0200 (CEST) Subject: [pypy-svn] r31420 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820203802.1FDB610074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 22:37:57 2006 New Revision: 31420 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: confstr() and confstr_names Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 22:37:57 2006 @@ -69,9 +69,12 @@ interpleveldefs['chown'] = 'interp_posix.chown' if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' - + if hasattr(os, 'confstr'): + interpleveldefs['confstr'] = 'interp_posix.confstr' for constant in dir(os): value = getattr(os, constant) if constant.isupper() and type(value) is int: Module.interpleveldefs[constant] = "space.wrap(%s)" % value +if hasattr(os, 'confstr_names'): + Module.interpleveldefs['confstr_names'] = "space.wrap(%s)" % os.confstr_names Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 22:37:57 2006 @@ -1,4 +1,4 @@ -from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rpython.rarithmetic import intmask from pypy.rpython import ros from pypy.interpreter.error import OperationError @@ -427,3 +427,27 @@ raise wrap_oserror(space, e) chroot.unwrap_spec = [ObjSpace, str] chroot.__doc__ = os.chroot.__doc__ + +def confstr(space, w_name): + w_name_type = space.type(w_name) + is_str = space.is_w(w_name_type, space.w_str) + is_int = space.is_w(w_name_type, space.w_int) + + res = '' + try: + if is_str: + res = os.confstr(space.str_w(w_name)) + elif is_int: + res = os.confstr(space.int_w(w_name)) + else: + raise OperationError(space.w_TypeError, + space.wrap("configuration names must be strings or integers")) + except OSError, e: + raise wrap_oserror(space, e) + except ValueError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.args[0])) + return space.wrap(res) +confstr.unwrap_spec = [ObjSpace, W_Root] +confstr.__doc__ = os.confstr.__doc__ + Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 22:37:57 2006 @@ -224,7 +224,18 @@ assert uid == stat_info.st_uid assert gid == stat_info.st_gid raises(OSError, posix.chown, path, 1000, 1000) - + + def test_confstr(self): + posix = self.posix + assert isinstance(posix.confstr_names, dict) + name = posix.confstr_names.keys()[0] + assert isinstance(posix.confstr(name), str) + val = posix.confstr_names.values()[0] + assert isinstance(posix.confstr(val), str) + raises(ValueError, posix.confstr, 'xYz') + raises(TypeError, posix.confstr, None) + raises(TypeError, posix.confstr, dict()) + assert isinstance(posix.confstr(12345), str) class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 23:14:32 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 23:14:32 +0200 (CEST) Subject: [pypy-svn] r31421 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820211432.4DF0B10074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 23:14:29 2006 New Revision: 31421 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: getcwdu(), ctermid(), fchdir() fpathconf() and pathconf_names Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 23:14:29 2006 @@ -34,6 +34,7 @@ 'unlink' : 'interp_posix.unlink', 'remove' : 'interp_posix.remove', 'getcwd' : 'interp_posix.getcwd', + 'getcwdu' : 'interp_posix.getcwdu', 'chdir' : 'interp_posix.chdir', 'mkdir' : 'interp_posix.mkdir', 'rmdir' : 'interp_posix.rmdir', @@ -71,6 +72,12 @@ interpleveldefs['chroot'] = 'interp_posix.chroot' if hasattr(os, 'confstr'): interpleveldefs['confstr'] = 'interp_posix.confstr' + if hasattr(os, 'ctermid'): + interpleveldefs['ctermid'] = 'interp_posix.ctermid' + if hasattr(os, 'fchdir'): + interpleveldefs['fchdir'] = 'interp_posix.fchdir' + if hasattr(os, 'fpathconf'): + interpleveldefs['fpathconf'] = 'interp_posix.fpathconf' for constant in dir(os): value = getattr(os, constant) @@ -78,3 +85,5 @@ Module.interpleveldefs[constant] = "space.wrap(%s)" % value if hasattr(os, 'confstr_names'): Module.interpleveldefs['confstr_names'] = "space.wrap(%s)" % os.confstr_names +if hasattr(os, 'pathconf_names'): + Module.interpleveldefs['pathconf_names'] = "space.wrap(%s)" % os.pathconf_names Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 23:14:29 2006 @@ -201,7 +201,6 @@ remove.unwrap_spec = [ObjSpace, str] def getcwd(space): - """Return the current working directory.""" try: cur = os.getcwd() except OSError, e: @@ -209,6 +208,17 @@ else: return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] +getcwd.__doc__ = os.getcwd.__doc__ + +def getcwdu(space): + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +getcwdu.unwrap_spec = [ObjSpace] +getcwdu.__doc__ = os.getcwdu.__doc__ def chdir(space, path): """Change the current working directory to the specified path.""" @@ -447,7 +457,46 @@ except ValueError, e: raise OperationError(space.w_ValueError, space.wrap(e.args[0])) - return space.wrap(res) + else: + return space.wrap(res) confstr.unwrap_spec = [ObjSpace, W_Root] confstr.__doc__ = os.confstr.__doc__ +def ctermid(space): + return space.wrap(os.ctermid()) +ctermid.unwrap_spec = [ObjSpace] +ctermid.__doc__ = os.ctermid.__doc__ + +def fchdir(space, fd): + try: + os.fchdir(fd) + except OSError, e: + raise wrap_oserror(space, e) +fchdir.unwrap_spec = [ObjSpace, int] +fchdir.__doc__ = os.fchdir.__doc__ + +def fpathconf(space, fd, w_name): + w_name_type = space.type(w_name) + is_str = space.is_w(w_name_type, space.w_str) + is_int = space.is_w(w_name_type, space.w_int) + + res = '' + try: + if is_str: + res = os.fpathconf(fd, space.str_w(w_name)) + elif is_int: + res = os.fpathconf(fd, space.int_w(w_name)) + else: + raise OperationError(space.w_TypeError, + space.wrap("configuration names must be strings or integers")) + except OSError, e: + raise wrap_oserror(space, e) + except ValueError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.args[0])) + else: + return space.wrap(res) +fpathconf.unwrap_spec = [ObjSpace, int, W_Root] +fpathconf.__doc__ = os.fpathconf.__doc__ + + Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 23:14:29 2006 @@ -83,6 +83,7 @@ ex(self.posix.chmod, str(UNUSEDFD), 0777) ex(self.posix.chown, str(UNUSEDFD), -1, -1) ex(self.posix.chroot, str(UNUSEDFD)) + ex(self.posix.fchdir, UNUSEDFD) def test_fdopen(self): path = self.path @@ -195,47 +196,96 @@ def test_ftruncate(self): import os - pdir = self.pdir - posix = self.posix - path = os.path.join(pdir, 'file1') - fd = posix.open(path, posix.O_WRONLY) - posix.ftruncate(fd, 2) - assert posix.stat(path)[6] == 2 - posix.close(fd) - raises(IOError, posix.ftruncate, 123123, 1) + if hasattr(__import__(os.name), "ftruncate"): + pdir = self.pdir + posix = self.posix + path = os.path.join(pdir, 'file1') + fd = posix.open(path, posix.O_WRONLY) + posix.ftruncate(fd, 2) + assert posix.stat(path)[6] == 2 + posix.close(fd) + raises(IOError, posix.ftruncate, 123123, 1) + else: + skip("ftruncate not supported") def test_abort(self): - posix = self.posix - pid = posix.fork() - if pid == 0: # child - posix.abort() + import os + if hasattr(__import__(os.name), "fork"): + posix = self.posix + pid = posix.fork() + if pid == 0: # child + posix.abort() + else: + skip("can't test abort, because fork is missing") def test_access(self): posix = self.posix assert posix.access('.', posix.W_OK) def test_chown(self): - posix = self.posix - path = self.path - stat_info = posix.stat(path) - uid, gid = stat_info.st_uid, stat_info.st_gid - posix.chown(path, -1, -1) - stat_info = posix.stat(path) - assert uid == stat_info.st_uid - assert gid == stat_info.st_gid - raises(OSError, posix.chown, path, 1000, 1000) + import os + if hasattr(__import__(os.name), "chown"): + posix = self.posix + path = self.path + stat_info = posix.stat(path) + uid, gid = stat_info.st_uid, stat_info.st_gid + posix.chown(path, -1, -1) + stat_info = posix.stat(path) + assert uid == stat_info.st_uid + assert gid == stat_info.st_gid + raises(OSError, posix.chown, path, 1000, 1000) + else: + skip("chown not supported") def test_confstr(self): - posix = self.posix - assert isinstance(posix.confstr_names, dict) - name = posix.confstr_names.keys()[0] - assert isinstance(posix.confstr(name), str) - val = posix.confstr_names.values()[0] - assert isinstance(posix.confstr(val), str) - raises(ValueError, posix.confstr, 'xYz') - raises(TypeError, posix.confstr, None) - raises(TypeError, posix.confstr, dict()) - assert isinstance(posix.confstr(12345), str) + import os + if hasattr(__import__(os.name), "confstr"): + posix = self.posix + assert isinstance(posix.confstr_names, dict) + name = posix.confstr_names.keys()[0] + assert isinstance(posix.confstr(name), str) + val = posix.confstr_names.values()[0] + assert isinstance(posix.confstr(val), str) + raises(ValueError, posix.confstr, 'xYz') + raises(TypeError, posix.confstr, None) + raises(TypeError, posix.confstr, dict()) + assert isinstance(posix.confstr(12345), str) + else: + skip("confstr and confstr_names not supported") + + def test_ctermid(self): + import os + if hasattr(__import__(os.name), "ctermid"): + assert isinstance(self.posix.ctermid(), str) + + def test_fchdir(self): + import os + if hasattr(__import__(os.name), "fchdir"): + pdir = self.pdir + posix = self.posix + whereami = posix.getcwd() + fd = posix.open(pdir, posix.O_RDONLY) + posix.fchdir(fd) + posix.chdir(whereami) + + def test_fpathconf(self): + import os + if hasattr(__import__(os.name), "fork"): + posix = self.posix + fd = posix.open(self.path, posix.O_RDONLY) + assert isinstance(posix.pathconf_names, dict) + name = posix.pathconf_names.keys()[-1] + assert isinstance(posix.fpathconf(fd, name), int) + val = posix.pathconf_names.values()[-1] + assert isinstance(posix.fpathconf(fd, val), int) + raises(ValueError, posix.fpathconf, fd, 'xYz') + raises(TypeError, posix.fpathconf, fd, None) + raises(TypeError, posix.fpathconf, fd, dict()) + else: + skip("fpathconf nad pathconf_names not supported") + + def test_getcwdu(self): + assert isinstance(self.posix.getcwdu(), unicode) class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 23:47:53 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 23:47:53 +0200 (CEST) Subject: [pypy-svn] r31422 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820214753.D088C10074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 23:47:50 2006 New Revision: 31422 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: getXYZid functions, getpgrp(), getgroups(), getlogin() and getloadavg() Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 23:47:50 2006 @@ -78,6 +78,31 @@ interpleveldefs['fchdir'] = 'interp_posix.fchdir' if hasattr(os, 'fpathconf'): interpleveldefs['fpathconf'] = 'interp_posix.fpathconf' + if hasattr(os, 'getegid'): + interpleveldefs['getegid'] = 'interp_posix.getegid' + if hasattr(os, 'geteuid'): + interpleveldefs['geteuid'] = 'interp_posix.geteuid' + if hasattr(os, 'getgid'): + interpleveldefs['getgid'] = 'interp_posix.getgid' + if hasattr(os, 'getuid'): + interpleveldefs['getuid'] = 'interp_posix.getuid' + if hasattr(os, 'getpgid'): + interpleveldefs['getpgid'] = 'interp_posix.getpgid' + if hasattr(os, 'getpid'): + interpleveldefs['getpid'] = 'interp_posix.getpid' + if hasattr(os, 'getppid'): + interpleveldefs['getppid'] = 'interp_posix.getppid' + if hasattr(os, 'getpgrp'): + interpleveldefs['getpgrp'] = 'interp_posix.getpgrp' + if hasattr(os, 'getsid'): + interpleveldefs['getsid'] = 'interp_posix.getsid' + if hasattr(os, 'getlogin'): + interpleveldefs['getlogin'] = 'interp_posix.getlogin' + if hasattr(os, 'getgroups'): + interpleveldefs['getgroups'] = 'interp_posix.getgroups' + if hasattr(os, 'getloadavg'): + interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' + for constant in dir(os): value = getattr(os, constant) Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Aug 20 23:47:50 2006 @@ -499,4 +499,79 @@ fpathconf.unwrap_spec = [ObjSpace, int, W_Root] fpathconf.__doc__ = os.fpathconf.__doc__ +def getegid(space): + return space.wrap(os.getegid()) +getegid.unwrap_spec = [ObjSpace] +getegid.__doc__ = os.getegid.__doc__ + +def geteuid(space): + return space.wrap(os.geteuid()) +geteuid.unwrap_spec = [ObjSpace] +geteuid.__doc__ = os.geteuid.__doc__ + +def getgid(space): + return space.wrap(os.getgid()) +getgid.unwrap_spec = [ObjSpace] +getgid.__doc__ = os.getgid.__doc__ + +def getuid(space): + return space.wrap(os.getuid()) +getuid.unwrap_spec = [ObjSpace] +getuid.__doc__ = os.getuid.__doc__ + +def getpgid(space, pid): + try: + res = os.getpgid(pid) + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(res) +getpgid.unwrap_spec = [ObjSpace, int] +getpgid.__doc__ = os.getpgid.__doc__ + +def getpid(space): + return space.wrap(os.getpid()) +getpid.unwrap_spec = [ObjSpace] +getpid.__doc__ = os.getpid.__doc__ + +def getppid(space): + return space.wrap(os.getppid()) +getppid.unwrap_spec = [ObjSpace] +getppid.__doc__ = os.getppid.__doc__ + +def getpgrp(space): + return space.wrap(os.getpgrp()) +getpgrp.unwrap_spec = [ObjSpace] +getpgrp.__doc__ = os.getpgrp.__doc__ + +def getsid(space, pid): + try: + res = os.getsid(pid) + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(res) +getsid.unwrap_spec = [ObjSpace, int] +getsid.__doc__ = os.getsid.__doc__ + +def getlogin(space): + return space.wrap(os.getlogin()) +getlogin.unwrap_spec = [ObjSpace] +getlogin.__doc__ = os.getlogin.__doc__ + +def getgroups(space): + return space.newlist(os.getgroups()) +getgroups.unwrap_spec = [ObjSpace] +getgroups.__doc__ = os.getgroups.__doc__ + +def getloadavg(space): + try: + res = os.getloadavg() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(res) +getloadavg.unwrap_spec = [ObjSpace] +getloadavg.__doc__ = os.getloadavg.__doc__ + Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sun Aug 20 23:47:50 2006 @@ -84,6 +84,8 @@ ex(self.posix.chown, str(UNUSEDFD), -1, -1) ex(self.posix.chroot, str(UNUSEDFD)) ex(self.posix.fchdir, UNUSEDFD) + ex(self.posix.getpgid, UNUSEDFD) + ex(self.posix.getsid, UNUSEDFD) def test_fdopen(self): path = self.path @@ -270,7 +272,7 @@ def test_fpathconf(self): import os - if hasattr(__import__(os.name), "fork"): + if hasattr(__import__(os.name), "fpathconf"): posix = self.posix fd = posix.open(self.path, posix.O_RDONLY) assert isinstance(posix.pathconf_names, dict) @@ -286,6 +288,39 @@ def test_getcwdu(self): assert isinstance(self.posix.getcwdu(), unicode) + + def test_get_ids(self): + import os + if hasattr(__import__(os.name), "getegid"): + posix = self.posix + assert isinstance(posix.getegid(), int) + assert isinstance(posix.geteuid(), int) + assert isinstance(posix.getgid(), int) + assert isinstance(posix.getuid(), int) + assert posix.getpgid(0) == posix.getpgrp() + assert isinstance(posix.getpid(), int) + assert isinstance(posix.getppid(), int) + assert isinstance(posix.getsid(0), int) + + def test_getlogin(self): + import os + if hasattr(__import__(os.name), "getlogin"): + posix = self.posix + assert isinstance(posix.getlogin(), str) + # assert posix.getlogin() == pwd.getpwuid(os.getuid())[0] + + def test_getgroups(self): + import os + if hasattr(__import__(os.name), "getgroups"): + assert isinstance(self.posix.getgroups(), list) + + def test_getloadavg(self): + import os + if hasattr(__import__(os.name), "getloadavg"): + posix = self.posix + load = posix.getloadavg() + assert isinstance(load, tuple) + assert len(load) == 3 class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Sun Aug 20 23:53:29 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Sun, 20 Aug 2006 23:53:29 +0200 (CEST) Subject: [pypy-svn] r31423 - pypy/dist/pypy/module/posix Message-ID: <20060820215329.00ADC10074@code0.codespeak.net> Author: rhymes Date: Sun Aug 20 23:53:27 2006 New Revision: 31423 Modified: pypy/dist/pypy/module/posix/__init__.py Log: refactor initialization of the module Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Aug 20 23:53:27 2006 @@ -48,67 +48,19 @@ 'abort' : 'interp_posix.abort', 'access' : 'interp_posix.access', } - if hasattr(os, 'ftruncate'): - interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' - if hasattr(os, 'putenv'): - interpleveldefs['putenv'] = 'interp_posix.putenv' - if hasattr(posix, 'unsetenv'): # note: emulated in os - interpleveldefs['unsetenv'] = 'interp_posix.unsetenv' - if hasattr(os, 'getpid'): - interpleveldefs['getpid'] = 'interp_posix.getpid' - if hasattr(os, 'link'): - interpleveldefs['link'] = 'interp_posix.link' - if hasattr(os, 'symlink'): - interpleveldefs['symlink'] = 'interp_posix.symlink' - if hasattr(os, 'readlink'): - interpleveldefs['readlink'] = 'interp_posix.readlink' - if hasattr(os, 'fork'): - interpleveldefs['fork'] = 'interp_posix.fork' - if hasattr(os, 'waitpid'): - interpleveldefs['waitpid'] = 'interp_posix.waitpid' - if hasattr(os, 'chown'): - interpleveldefs['chown'] = 'interp_posix.chown' - if hasattr(os, 'chroot'): - interpleveldefs['chroot'] = 'interp_posix.chroot' - if hasattr(os, 'confstr'): - interpleveldefs['confstr'] = 'interp_posix.confstr' - if hasattr(os, 'ctermid'): - interpleveldefs['ctermid'] = 'interp_posix.ctermid' - if hasattr(os, 'fchdir'): - interpleveldefs['fchdir'] = 'interp_posix.fchdir' - if hasattr(os, 'fpathconf'): - interpleveldefs['fpathconf'] = 'interp_posix.fpathconf' - if hasattr(os, 'getegid'): - interpleveldefs['getegid'] = 'interp_posix.getegid' - if hasattr(os, 'geteuid'): - interpleveldefs['geteuid'] = 'interp_posix.geteuid' - if hasattr(os, 'getgid'): - interpleveldefs['getgid'] = 'interp_posix.getgid' - if hasattr(os, 'getuid'): - interpleveldefs['getuid'] = 'interp_posix.getuid' - if hasattr(os, 'getpgid'): - interpleveldefs['getpgid'] = 'interp_posix.getpgid' - if hasattr(os, 'getpid'): - interpleveldefs['getpid'] = 'interp_posix.getpid' - if hasattr(os, 'getppid'): - interpleveldefs['getppid'] = 'interp_posix.getppid' - if hasattr(os, 'getpgrp'): - interpleveldefs['getpgrp'] = 'interp_posix.getpgrp' - if hasattr(os, 'getsid'): - interpleveldefs['getsid'] = 'interp_posix.getsid' - if hasattr(os, 'getlogin'): - interpleveldefs['getlogin'] = 'interp_posix.getlogin' - if hasattr(os, 'getgroups'): - interpleveldefs['getgroups'] = 'interp_posix.getgroups' - if hasattr(os, 'getloadavg'): - interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' - + for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link' + 'symlink', 'readlink', 'fork', 'fork', 'waitpid', 'chown', 'chroot', + 'confstr', 'ctermid', 'fchdir', 'fpathconf', 'getegid', 'geteuid', + 'getgid', 'getuid', 'getpgid', 'getpid', 'getppid', 'getpgrp', + 'getsid', 'getlogin', 'getgroups', 'getloadavg']: + if hasattr(os, func_name): + interpleveldefs[func_name] = 'interp_posix.%s' % func_name + for constant in dir(os): value = getattr(os, constant) if constant.isupper() and type(value) is int: Module.interpleveldefs[constant] = "space.wrap(%s)" % value -if hasattr(os, 'confstr_names'): - Module.interpleveldefs['confstr_names'] = "space.wrap(%s)" % os.confstr_names -if hasattr(os, 'pathconf_names'): - Module.interpleveldefs['pathconf_names'] = "space.wrap(%s)" % os.pathconf_names +for const in ['confstr_names', 'pathconf_names']: + if hasattr(os, const): + Module.interpleveldefs[const] = "space.wrap(%s)" % getattr(os, const) From rhymes at codespeak.net Mon Aug 21 00:03:54 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 00:03:54 +0200 (CEST) Subject: [pypy-svn] r31425 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820220354.C4C9210075@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 00:03:50 2006 New Revision: 31425 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: test linking and a small fix Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Aug 21 00:03:50 2006 @@ -49,8 +49,8 @@ 'access' : 'interp_posix.access', } - for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link' - 'symlink', 'readlink', 'fork', 'fork', 'waitpid', 'chown', 'chroot', + for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link', + 'symlink', 'readlink', 'fork', 'waitpid', 'chown', 'chroot', 'confstr', 'ctermid', 'fchdir', 'fpathconf', 'getegid', 'geteuid', 'getgid', 'getuid', 'getpgid', 'getpid', 'getppid', 'getpgrp', 'getsid', 'getlogin', 'getgroups', 'getloadavg']: Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon Aug 21 00:03:50 2006 @@ -86,6 +86,8 @@ ex(self.posix.fchdir, UNUSEDFD) ex(self.posix.getpgid, UNUSEDFD) ex(self.posix.getsid, UNUSEDFD) + ex(self.posix.link, "foo", "foo") + ex(self.posix.readlink, "foo") def test_fdopen(self): path = self.path @@ -321,6 +323,18 @@ load = posix.getloadavg() assert isinstance(load, tuple) assert len(load) == 3 + + def test_linking(self): + import os + if hasattr(__import__(os.name), "symlink"): + posix = self.posix + pdir = self.pdir + path = self.path + link = os.path.join(pdir, 'link') + posix.symlink(path, link) + hard_link = os.path.join(pdir, 'hard_link') + posix.link(path, hard_link) + assert posix.readlink(link) == path class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Mon Aug 21 00:18:38 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 00:18:38 +0200 (CEST) Subject: [pypy-svn] r31426 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820221838.1F97410078@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 00:18:34 2006 New Revision: 31426 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: more functions: major(), minor(), pathconf() and lchown() Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Aug 21 00:18:34 2006 @@ -47,13 +47,15 @@ '_exit' : 'interp_posix._exit', 'abort' : 'interp_posix.abort', 'access' : 'interp_posix.access', + 'major' : 'interp_posix.major', + 'minor' : 'interp_posix.minor', } for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link', 'symlink', 'readlink', 'fork', 'waitpid', 'chown', 'chroot', 'confstr', 'ctermid', 'fchdir', 'fpathconf', 'getegid', 'geteuid', 'getgid', 'getuid', 'getpgid', 'getpid', 'getppid', 'getpgrp', - 'getsid', 'getlogin', 'getgroups', 'getloadavg']: + 'getsid', 'getlogin', 'getgroups', 'getloadavg', 'lchown', 'pathconf']: if hasattr(os, func_name): interpleveldefs[func_name] = 'interp_posix.%s' % func_name Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Aug 21 00:18:34 2006 @@ -430,6 +430,14 @@ chown.unwrap_spec = [ObjSpace, str, int, int] chown.__doc__ = os.chown.__doc__ +def lchown(space, path, uid, gid): + try: + os.lchown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e) +lchown.unwrap_spec = [ObjSpace, str, int, int] +lchown.__doc__ = os.lchown.__doc__ + def chroot(space, path): try: os.chroot(path) @@ -498,6 +506,30 @@ return space.wrap(res) fpathconf.unwrap_spec = [ObjSpace, int, W_Root] fpathconf.__doc__ = os.fpathconf.__doc__ + +def pathconf(space, path, w_name): + w_name_type = space.type(w_name) + is_str = space.is_w(w_name_type, space.w_str) + is_int = space.is_w(w_name_type, space.w_int) + + res = '' + try: + if is_str: + res = os.pathconf(path, space.str_w(w_name)) + elif is_int: + res = os.pathconf(path, space.int_w(w_name)) + else: + raise OperationError(space.w_TypeError, + space.wrap("configuration names must be strings or integers")) + except OSError, e: + raise wrap_oserror(space, e) + except ValueError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.args[0])) + else: + return space.wrap(res) +pathconf.unwrap_spec = [ObjSpace, str, W_Root] +pathconf.__doc__ = os.pathconf.__doc__ def getegid(space): return space.wrap(os.getegid()) @@ -574,4 +606,13 @@ getloadavg.unwrap_spec = [ObjSpace] getloadavg.__doc__ = os.getloadavg.__doc__ +def major(space, device): + return space.wrap(os.major(device)) +major.unwrap_spec = [ObjSpace, int] +major.__doc__ = os.major.__doc__ + +def minor(space, device): + return space.wrap(os.minor(device)) +minor.unwrap_spec = [ObjSpace, int] +minor.__doc__ = os.minor.__doc__ Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon Aug 21 00:18:34 2006 @@ -234,6 +234,7 @@ stat_info = posix.stat(path) uid, gid = stat_info.st_uid, stat_info.st_gid posix.chown(path, -1, -1) + posix.lchown(path, -1, -1) stat_info = posix.stat(path) assert uid == stat_info.st_uid assert gid == stat_info.st_gid @@ -286,7 +287,23 @@ raises(TypeError, posix.fpathconf, fd, None) raises(TypeError, posix.fpathconf, fd, dict()) else: - skip("fpathconf nad pathconf_names not supported") + skip("fpathconf and pathconf_names not supported") + + def test_pathconf(self): + import os + if hasattr(__import__(os.name), "pathconf"): + posix = self.posix + path = self.path + assert isinstance(posix.pathconf_names, dict) + name = posix.pathconf_names.keys()[-1] + assert isinstance(posix.pathconf(path, name), int) + val = posix.pathconf_names.values()[-1] + assert isinstance(posix.pathconf(path, val), int) + raises(ValueError, posix.pathconf, path, 'xYz') + raises(TypeError, posix.pathconf, path, None) + raises(TypeError, posix.pathconf, path, dict()) + else: + skip("pathconf nad pathconf_names not supported") def test_getcwdu(self): assert isinstance(self.posix.getcwdu(), unicode) @@ -335,6 +352,12 @@ hard_link = os.path.join(pdir, 'hard_link') posix.link(path, hard_link) assert posix.readlink(link) == path + + def test_major_minor(self): + posix = self.posix + fd = posix.open("/dev/urandom", posix.O_RDONLY) + assert isinstance(posix.major(fd), int) + assert isinstance(posix.minor(fd), int) class AppTestEnvironment(object): def setup_class(cls): From rhymes at codespeak.net Mon Aug 21 00:27:48 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 00:27:48 +0200 (CEST) Subject: [pypy-svn] r31427 - pypy/dist/pypy/module/posix Message-ID: <20060820222748.B10A910075@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 00:27:46 2006 New Revision: 31427 Modified: pypy/dist/pypy/module/posix/__init__.py Log: more refactoring of the initialization Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Aug 21 00:27:46 2006 @@ -10,52 +10,25 @@ disguised Unix interface). Refer to the library manual and corresponding Unix manual entries for more information on calls.""" - applevel_name = os.name - appleveldefs = { - 'error' : 'app_posix.error', - 'stat_result': 'app_posix.stat_result', - 'fdopen' : 'app_posix.fdopen', + 'error': 'app_posix.error', + 'stat_result': 'app_posix.stat_result', + 'fdopen': 'app_posix.fdopen', } interpleveldefs = { - 'open' : 'interp_posix.open', - 'lseek' : 'interp_posix.lseek', - 'write' : 'interp_posix.write', - 'isatty' : 'interp_posix.isatty', - 'read' : 'interp_posix.read', - 'close' : 'interp_posix.close', - 'fstat' : 'interp_posix.fstat', - 'stat' : 'interp_posix.stat', - 'lstat' : 'interp_posix.lstat', - 'dup' : 'interp_posix.dup', - 'dup2' : 'interp_posix.dup2', - 'system' : 'interp_posix.system', - 'unlink' : 'interp_posix.unlink', - 'remove' : 'interp_posix.remove', - 'getcwd' : 'interp_posix.getcwd', - 'getcwdu' : 'interp_posix.getcwdu', - 'chdir' : 'interp_posix.chdir', - 'mkdir' : 'interp_posix.mkdir', - 'rmdir' : 'interp_posix.rmdir', - 'environ' : 'interp_posix.get(space).w_environ', - 'listdir' : 'interp_posix.listdir', - 'strerror' : 'interp_posix.strerror', - 'pipe' : 'interp_posix.pipe', - 'chmod' : 'interp_posix.chmod', - 'rename' : 'interp_posix.rename', - '_exit' : 'interp_posix._exit', - 'abort' : 'interp_posix.abort', - 'access' : 'interp_posix.access', - 'major' : 'interp_posix.major', - 'minor' : 'interp_posix.minor', + 'environ': 'interp_posix.get(space).w_environ' } for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link', 'symlink', 'readlink', 'fork', 'waitpid', 'chown', 'chroot', 'confstr', 'ctermid', 'fchdir', 'fpathconf', 'getegid', 'geteuid', 'getgid', 'getuid', 'getpgid', 'getpid', 'getppid', 'getpgrp', - 'getsid', 'getlogin', 'getgroups', 'getloadavg', 'lchown', 'pathconf']: + 'getsid', 'getlogin', 'getgroups', 'getloadavg', 'lchown', 'pathconf', + 'minor', 'major', 'access', 'abort', '_exit', 'rename', 'chmod', + 'pipe', 'strerror', 'listdir', 'rmdir', 'mkdir', 'chdir', 'getcwdu', + 'getcwd', 'remove', 'unlink', 'system', 'dup2', 'dup', 'lstat', + 'stat', 'fstat', 'close', 'read', 'write', 'isatty', 'lseek', 'open']: if hasattr(os, func_name): interpleveldefs[func_name] = 'interp_posix.%s' % func_name From rhymes at codespeak.net Mon Aug 21 00:49:36 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 00:49:36 +0200 (CEST) Subject: [pypy-svn] r31428 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060820224936.86E451007C@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 00:49:33 2006 New Revision: 31428 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: more functions again: sysconf(), wait(), uname(), umask(), ttyname() Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Aug 21 00:49:33 2006 @@ -28,7 +28,8 @@ 'minor', 'major', 'access', 'abort', '_exit', 'rename', 'chmod', 'pipe', 'strerror', 'listdir', 'rmdir', 'mkdir', 'chdir', 'getcwdu', 'getcwd', 'remove', 'unlink', 'system', 'dup2', 'dup', 'lstat', - 'stat', 'fstat', 'close', 'read', 'write', 'isatty', 'lseek', 'open']: + 'stat', 'fstat', 'close', 'read', 'write', 'isatty', 'lseek', 'open', + 'sysconf', 'wait', 'uname', 'umask', 'ttyname']: if hasattr(os, func_name): interpleveldefs[func_name] = 'interp_posix.%s' % func_name @@ -36,6 +37,6 @@ value = getattr(os, constant) if constant.isupper() and type(value) is int: Module.interpleveldefs[constant] = "space.wrap(%s)" % value -for const in ['confstr_names', 'pathconf_names']: +for const in ['confstr_names', 'pathconf_names', 'sysconf_names']: if hasattr(os, const): Module.interpleveldefs[const] = "space.wrap(%s)" % getattr(os, const) Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Aug 21 00:49:33 2006 @@ -302,7 +302,6 @@ del get(space).posix_putenv_garbage[name] unsetenv.unwrap_spec = [ObjSpace, str] - def enumeratedir(space, dir): result = [] while True: @@ -403,6 +402,12 @@ waitpid.unwrap_spec = [ObjSpace, int, int] waitpid.__doc__ = os.waitpid.__doc__ +def wait(space): + pid, status = os.wait() + return space.newtuple([space.wrap(pid), space.wrap(status)]) +wait.unwrap_spec = [ObjSpace] +wait.__doc__ = os.wait.__doc__ + def _exit(space, status): os._exit(status) _exit.unwrap_spec = [ObjSpace, int] @@ -616,3 +621,48 @@ minor.unwrap_spec = [ObjSpace, int] minor.__doc__ = os.minor.__doc__ +def sysconf(space, w_name): + w_name_type = space.type(w_name) + is_str = space.is_w(w_name_type, space.w_str) + is_int = space.is_w(w_name_type, space.w_int) + + res = '' + try: + if is_str: + res = os.sysconf(space.str_w(w_name)) + elif is_int: + res = os.sysconf(space.int_w(w_name)) + else: + raise OperationError(space.w_TypeError, + space.wrap("configuration names must be strings or integers")) + except OSError, e: + raise wrap_oserror(space, e) + except ValueError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.args[0])) + else: + return space.wrap(res) +sysconf.unwrap_spec = [ObjSpace, W_Root] +sysconf.__doc__ = os.sysconf.__doc__ + +def uname(space): + name = os.uname() + name_w = [space.wrap(i) for i in name] + return space.newtuple(name_w) +uname.unwrap_spec = [ObjSpace] +uname.__doc__ == os.uname.__doc__ + +def umask(space, mask): + return space.wrap(os.umask(mask)) +umask.unwrap_spec = [ObjSpace, int] +umask.__doc__ == os.umask.__doc__ + +def ttyname(space, fd): + try: + res = os.ttyname(fd) + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(res) +ttyname.unwrap_spec = [ObjSpace, int] +ttyname.__doc__ = os.ttyname.__doc__ Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon Aug 21 00:49:33 2006 @@ -88,6 +88,8 @@ ex(self.posix.getsid, UNUSEDFD) ex(self.posix.link, "foo", "foo") ex(self.posix.readlink, "foo") + ex(self.posix.sysconf, UNUSEDFD) + ex(self.posix.ttyname, UNUSEDFD) def test_fdopen(self): path = self.path @@ -358,6 +360,50 @@ fd = posix.open("/dev/urandom", posix.O_RDONLY) assert isinstance(posix.major(fd), int) assert isinstance(posix.minor(fd), int) + + def test_sysconf(self): + import os + if hasattr(__import__(os.name), "sysconf"): + posix = self.posix + assert isinstance(posix.sysconf_names, dict) + name = posix.sysconf_names.keys()[0] + assert isinstance(posix.sysconf(name), int) + val = posix.sysconf_names.values()[0] + assert isinstance(posix.sysconf(val), int) + raises(ValueError, posix.sysconf, 'xYz') + raises(TypeError, posix.sysconf, None) + raises(TypeError, posix.sysconf, dict()) + else: + skip("confstr and confstr_names not supported") + + def test_wait(self): + import os + if hasattr(__import__(os.name), "wait"): + posix = self.posix + pid = posix.fork() + if pid == 0: # child + posix._exit(4) + pid1, status1 = os.wait() + assert pid1 == pid + else: + skip("wait not supported") + + def test_uname(self): + import os + if hasattr(__import__(os.name), "uname"): + uname = self.posix.uname() + assert isinstance(uname, tuple) + assert len(uname) == 5 + + def test_umask(self): + import os + if hasattr(__import__(os.name), "umask"): + assert isinstance(self.posix.umask(022), int) + + def test_ttyname(self): + import os + if hasattr(__import__(os.name), "umask"): + assert isinstance(self.posix.ttyname(0), str) class AppTestEnvironment(object): def setup_class(cls): From hpk at codespeak.net Mon Aug 21 11:41:40 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 21 Aug 2006 11:41:40 +0200 (CEST) Subject: [pypy-svn] r31431 - pypy/extradoc/planning/secprototype Message-ID: <20060821094140.2AF3710036@code0.codespeak.net> Author: hpk Date: Mon Aug 21 11:41:38 2006 New Revision: 31431 Added: pypy/extradoc/planning/secprototype/ pypy/extradoc/planning/secprototype/links.txt (contents, props changed) Log: adding a directory for security prototype + IBM work shop planning adding a link to the Myers paper on flow control Added: pypy/extradoc/planning/secprototype/links.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/secprototype/links.txt Mon Aug 21 11:41:38 2006 @@ -0,0 +1,6 @@ + +JFlow: Practical Mostly-Static Information Flow Control +http://pag.csail.mit.edu/reading-group/myers99jflow.pdf + +major paper on data flow analysis based security models. + From rhymes at codespeak.net Mon Aug 21 13:08:54 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 13:08:54 +0200 (CEST) Subject: [pypy-svn] r31435 - pypy/dist/pypy/module/posix/test Message-ID: <20060821110854.51D0410074@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 13:08:51 2006 New Revision: 31435 Modified: pypy/dist/pypy/module/posix/test/test_posix2.py Log: remove not raising tests in Linux Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon Aug 21 13:08:51 2006 @@ -88,7 +88,6 @@ ex(self.posix.getsid, UNUSEDFD) ex(self.posix.link, "foo", "foo") ex(self.posix.readlink, "foo") - ex(self.posix.sysconf, UNUSEDFD) ex(self.posix.ttyname, UNUSEDFD) def test_fdopen(self): @@ -240,7 +239,6 @@ stat_info = posix.stat(path) assert uid == stat_info.st_uid assert gid == stat_info.st_gid - raises(OSError, posix.chown, path, 1000, 1000) else: skip("chown not supported") From ac at codespeak.net Mon Aug 21 13:26:40 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Mon, 21 Aug 2006 13:26:40 +0200 (CEST) Subject: [pypy-svn] r31436 - pypy/dist/pypy/translator/c Message-ID: <20060821112640.8FB4010076@code0.codespeak.net> Author: ac Date: Mon Aug 21 13:26:40 2006 New Revision: 31436 Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/node.py Log: Fix translation. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Mon Aug 21 13:26:40 2006 @@ -396,7 +396,7 @@ print >> fi, '/*** Structure definitions ***/' print >> fi for node in structdeflist: - if node.is_external(): + if getattr(node, 'is_external', False): continue print >> fi, '%s %s;' % (node.typetag, node.name) print >> fi @@ -514,7 +514,7 @@ print >> f, '/*** Structure definitions ***/' print >> f for node in structdeflist: - if node.name and not node.is_external(): + if node.name and not getattr(node, 'is_external', False): print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Mon Aug 21 13:26:40 2006 @@ -36,7 +36,7 @@ class StructDefNode: typetag = 'struct' - + is_external = False def __init__(self, db, STRUCT, varlength=1): self.db = db self.STRUCT = STRUCT @@ -55,6 +55,8 @@ if STRUCT._hints.get('typedef'): self.typetag = '' assert STRUCT._hints.get('external') + if self.STRUCT._hints.get('external'): # XXX hack + self.is_external = True if STRUCT._hints.get('c_name'): self.barename = self.name = STRUCT._hints['c_name'] self.c_struct_field_name = self.verbatim_field_name @@ -145,9 +147,6 @@ fldname = self.c_struct_field_name(fldname) return '%s->%s' % (baseexpr, fldname) - def is_external(self): - return self.STRUCT._hints.get('external') # XXX hack - def definition(self): if self.fields is None: # external definition only return @@ -243,9 +242,6 @@ def ptr_access_expr(self, baseexpr, index): return '%s->items[%d]' % (baseexpr, index) - def is_external(self): - return False - def definition(self): gcpolicy = self.db.gcpolicy yield 'struct %s {' % self.name @@ -329,9 +325,6 @@ ptr_access_expr = access_expr - def is_external(self): - return False - def definition(self): return [] # no declaration is needed From mwh at codespeak.net Mon Aug 21 13:55:15 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 21 Aug 2006 13:55:15 +0200 (CEST) Subject: [pypy-svn] r31438 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20060821115515.F0F4E10078@code0.codespeak.net> Author: mwh Date: Mon Aug 21 13:55:15 2006 New Revision: 31438 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: fixes to StrSmallDictImplementation and some ascii art Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Mon Aug 21 13:55:15 2006 @@ -6,6 +6,21 @@ def _is_str(space, w_key): return space.is_w(space.type(w_key), space.w_str) +# DictImplementation lattice + +# a dictionary starts with an EmptyDictImplementation, and moves down +# in this list: +# +# EmptyDictImplementation +# / \ +# SmallStrDictImplementation SmallDictImplementation +# | | +# StrDictImplementation | +# \ / +# RDictImplementation +# +# (in addition, any dictionary can go back to EmptyDictImplementation) + class DictImplementation(object): ## def get(self, w_lookup): @@ -42,13 +57,13 @@ return None def setitem(self, w_key, w_value): if _is_str(self.space, w_key): - return StrDictImplementation(self.space, w_key, w_value) + return StrDictImplementation(self.space).setitem_str(w_key, w_value) #return SmallStrDictImplementation(self.space, w_key, w_value) else: return RDictImplementation(self.space).setitem(w_key, w_value) #return SmallDictImplementation(self.space, w_key, w_value) def setitem_str(self, w_key, w_value): - return StrDictImplementation(self.space, w_key, w_value) + return StrDictImplementation(self.space).setitem_str(w_key, w_value) #return SmallStrDictImplementation(self.space, w_key, w_value) def delitem(self, w_key): raise KeyError @@ -171,12 +186,23 @@ self.space = space self.entries = [StrEntry(), StrEntry(), StrEntry(), StrEntry(), StrEntry()] key = space.str_w(w_key) - self.entries[0].key = key self.entries[0].hash = hash(key) + self.entries[0].key = key self.entries[0].w_value = w_value self.valid = 1 + def _is_sane_hash(self, w_lookup_type): + """ Handles the case of a non string key lookup. + Types that have a sane hash/eq function should allow us to return True + directly to signal that the key is not in the dict in any case. + XXX The types should provide such a flag. """ + + space = self.space + # XXX there are many more such types + return space.is_w(w_lookup_type, space.w_NoneType) or space.is_w(w_lookup_type, space.w_int) + def _lookup(self, key): + assert isinstance(key, str) _hash = hash(key) i = 0 last = self.entries[self.valid] @@ -199,12 +225,15 @@ i += 1 return newimpl - def _convert_to_sdict(self, w_key, w_value): - newimpl = StrDictImplementation(self.space, w_key, w_value) + def _convert_to_sdict(self, w_value): + # this relies on the fact that the new key is in the entries + # list already. + newimpl = StrDictImplementation(self.space) i = 0 while 1: entry = self.entries[i] if entry.w_value is None: + newimpl.content[entry.key] = w_value break newimpl.content[entry.key] = entry.w_value i += 1 @@ -219,7 +248,7 @@ entry = self._lookup(self.space.str_w(w_key)) if entry.w_value is None: if self.valid == 4: - return self._convert_to_sdict(w_key, w_value) + return self._convert_to_sdict(w_value) self.valid += 1 entry.w_value = w_value return self @@ -252,7 +281,7 @@ space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - return self._lookup(w_lookup).w_value + return self._lookup(space.str_w(w_lookup)).w_value elif self._is_sane_hash(w_lookup_type): return None else: @@ -274,9 +303,9 @@ for e in [self.entries[i] for i in range(self.valid)]] class StrDictImplementation(DictImplementation): - def __init__(self, space, w_key, w_value): + def __init__(self, space): self.space = space - self.content = {space.str_w(w_key): w_value} + self.content = {} def setitem(self, w_key, w_value): space = self.space Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Mon Aug 21 13:55:15 2006 @@ -133,9 +133,6 @@ class TestStrDictImplementation(TestRDictImplementation): ImplementionClass = StrDictImplementation - def get_impl(self): - return self.ImplementionClass(self.space, self.string, self.string2) - class TestSmallDictImplementation(TestRDictImplementation): ImplementionClass = SmallDictImplementation From rhymes at codespeak.net Mon Aug 21 15:27:34 2006 From: rhymes at codespeak.net (rhymes at codespeak.net) Date: Mon, 21 Aug 2006 15:27:34 +0200 (CEST) Subject: [pypy-svn] r31440 - in pypy/dist/pypy/module/posix: . test Message-ID: <20060821132734.1619010050@code0.codespeak.net> Author: rhymes Date: Mon Aug 21 15:27:31 2006 New Revision: 31440 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: rollback to rev #31414 because the compilation breaks Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Aug 21 15:27:31 2006 @@ -10,33 +10,62 @@ disguised Unix interface). Refer to the library manual and corresponding Unix manual entries for more information on calls.""" + applevel_name = os.name + appleveldefs = { - 'error': 'app_posix.error', - 'stat_result': 'app_posix.stat_result', - 'fdopen': 'app_posix.fdopen', + 'error' : 'app_posix.error', + 'stat_result': 'app_posix.stat_result', + 'fdopen' : 'app_posix.fdopen', } interpleveldefs = { - 'environ': 'interp_posix.get(space).w_environ' + 'open' : 'interp_posix.open', + 'lseek' : 'interp_posix.lseek', + 'write' : 'interp_posix.write', + 'isatty' : 'interp_posix.isatty', + 'read' : 'interp_posix.read', + 'close' : 'interp_posix.close', + 'fstat' : 'interp_posix.fstat', + 'stat' : 'interp_posix.stat', + 'lstat' : 'interp_posix.lstat', + 'dup' : 'interp_posix.dup', + 'dup2' : 'interp_posix.dup2', + 'system' : 'interp_posix.system', + 'unlink' : 'interp_posix.unlink', + 'remove' : 'interp_posix.remove', + 'getcwd' : 'interp_posix.getcwd', + 'chdir' : 'interp_posix.chdir', + 'mkdir' : 'interp_posix.mkdir', + 'rmdir' : 'interp_posix.rmdir', + 'environ' : 'interp_posix.get(space).w_environ', + 'listdir' : 'interp_posix.listdir', + 'strerror' : 'interp_posix.strerror', + 'pipe' : 'interp_posix.pipe', + 'chmod' : 'interp_posix.chmod', + 'rename' : 'interp_posix.rename', + '_exit' : 'interp_posix._exit', } - - for func_name in ['ftruncate', 'putenv', 'unsetenv', 'getpid', 'link', - 'symlink', 'readlink', 'fork', 'waitpid', 'chown', 'chroot', - 'confstr', 'ctermid', 'fchdir', 'fpathconf', 'getegid', 'geteuid', - 'getgid', 'getuid', 'getpgid', 'getpid', 'getppid', 'getpgrp', - 'getsid', 'getlogin', 'getgroups', 'getloadavg', 'lchown', 'pathconf', - 'minor', 'major', 'access', 'abort', '_exit', 'rename', 'chmod', - 'pipe', 'strerror', 'listdir', 'rmdir', 'mkdir', 'chdir', 'getcwdu', - 'getcwd', 'remove', 'unlink', 'system', 'dup2', 'dup', 'lstat', - 'stat', 'fstat', 'close', 'read', 'write', 'isatty', 'lseek', 'open', - 'sysconf', 'wait', 'uname', 'umask', 'ttyname']: - if hasattr(os, func_name): - interpleveldefs[func_name] = 'interp_posix.%s' % func_name - + if hasattr(os, 'ftruncate'): + interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' + if hasattr(os, 'putenv'): + interpleveldefs['putenv'] = 'interp_posix.putenv' + if hasattr(posix, 'unsetenv'): # note: emulated in os + interpleveldefs['unsetenv'] = 'interp_posix.unsetenv' + if hasattr(os, 'getpid'): + interpleveldefs['getpid'] = 'interp_posix.getpid' + if hasattr(os, 'link'): + interpleveldefs['link'] = 'interp_posix.link' + if hasattr(os, 'symlink'): + interpleveldefs['symlink'] = 'interp_posix.symlink' + if hasattr(os, 'readlink'): + interpleveldefs['readlink'] = 'interp_posix.readlink' + if hasattr(os, 'fork'): + interpleveldefs['fork'] = 'interp_posix.fork' + if hasattr(os, 'waitpid'): + interpleveldefs['waitpid'] = 'interp_posix.waitpid' + + for constant in dir(os): value = getattr(os, constant) if constant.isupper() and type(value) is int: Module.interpleveldefs[constant] = "space.wrap(%s)" % value -for const in ['confstr_names', 'pathconf_names', 'sysconf_names']: - if hasattr(os, const): - Module.interpleveldefs[const] = "space.wrap(%s)" % getattr(os, const) Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Aug 21 15:27:31 2006 @@ -1,4 +1,4 @@ -from pypy.interpreter.baseobjspace import ObjSpace, W_Root +from pypy.interpreter.baseobjspace import ObjSpace from pypy.rpython.rarithmetic import intmask from pypy.rpython import ros from pypy.interpreter.error import OperationError @@ -17,18 +17,6 @@ space.wrap(errno), space.wrap(msg)) return OperationError(space.w_OSError, w_error) - -def wrap_ioerror(space, e): - assert isinstance(e, IOError) - errno = e.errno - try: - msg = os.strerror(errno) - except ValueError: - msg = 'error %d' % errno - w_error = space.call_function(space.w_IOError, - space.wrap(errno), - space.wrap(msg)) - return OperationError(space.w_IOError, w_error) def open(space, fname, flag, mode=0777): """Open a file (for low level IO). @@ -97,9 +85,7 @@ try: os.ftruncate(fd, length) except OSError, e: - raise wrap_oserror(space, e) - except IOError, e: - raise wrap_ioerror(space, e) + raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, int, int] def build_stat_result(space, st): @@ -201,6 +187,7 @@ remove.unwrap_spec = [ObjSpace, str] def getcwd(space): + """Return the current working directory.""" try: cur = os.getcwd() except OSError, e: @@ -208,17 +195,6 @@ else: return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -getcwd.__doc__ = os.getcwd.__doc__ - -def getcwdu(space): - try: - cur = os.getcwdu() - except OSError, e: - raise wrap_oserror(space, e) - else: - return space.wrap(cur) -getcwdu.unwrap_spec = [ObjSpace] -getcwdu.__doc__ = os.getcwdu.__doc__ def chdir(space, path): """Change the current working directory to the specified path.""" @@ -302,6 +278,7 @@ del get(space).posix_putenv_garbage[name] unsetenv.unwrap_spec = [ObjSpace, str] + def enumeratedir(space, dir): result = [] while True: @@ -393,276 +370,12 @@ def fork(space): pid = os.fork() return space.wrap(pid) -fork.unwrap_spec = [ObjSpace] -fork.__doc__ = os.fork.__doc__ def waitpid(space, pid, options): pid, status = os.waitpid(pid, options) return space.newtuple([space.wrap(pid), space.wrap(status)]) waitpid.unwrap_spec = [ObjSpace, int, int] -waitpid.__doc__ = os.waitpid.__doc__ - -def wait(space): - pid, status = os.wait() - return space.newtuple([space.wrap(pid), space.wrap(status)]) -wait.unwrap_spec = [ObjSpace] -wait.__doc__ = os.wait.__doc__ def _exit(space, status): os._exit(status) _exit.unwrap_spec = [ObjSpace, int] -_exit.__doc__ = os._exit.__doc__ - -def abort(space): - os.abort() -abort.unwrap_spec = [ObjSpace] -abort.__doc__ = os.abort.__doc__ - -def access(space, path, mode): - try: - res = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e) - return space.wrap(res) -access.unwrap_spec = [ObjSpace, str, int] -access.__doc__ = os.access.__doc__ - -def chown(space, path, uid, gid): - try: - os.chown(path, uid, gid) - except OSError, e: - raise wrap_oserror(space, e) -chown.unwrap_spec = [ObjSpace, str, int, int] -chown.__doc__ = os.chown.__doc__ - -def lchown(space, path, uid, gid): - try: - os.lchown(path, uid, gid) - except OSError, e: - raise wrap_oserror(space, e) -lchown.unwrap_spec = [ObjSpace, str, int, int] -lchown.__doc__ = os.lchown.__doc__ - -def chroot(space, path): - try: - os.chroot(path) - except OSError, e: - raise wrap_oserror(space, e) -chroot.unwrap_spec = [ObjSpace, str] -chroot.__doc__ = os.chroot.__doc__ - -def confstr(space, w_name): - w_name_type = space.type(w_name) - is_str = space.is_w(w_name_type, space.w_str) - is_int = space.is_w(w_name_type, space.w_int) - - res = '' - try: - if is_str: - res = os.confstr(space.str_w(w_name)) - elif is_int: - res = os.confstr(space.int_w(w_name)) - else: - raise OperationError(space.w_TypeError, - space.wrap("configuration names must be strings or integers")) - except OSError, e: - raise wrap_oserror(space, e) - except ValueError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.args[0])) - else: - return space.wrap(res) -confstr.unwrap_spec = [ObjSpace, W_Root] -confstr.__doc__ = os.confstr.__doc__ - -def ctermid(space): - return space.wrap(os.ctermid()) -ctermid.unwrap_spec = [ObjSpace] -ctermid.__doc__ = os.ctermid.__doc__ - -def fchdir(space, fd): - try: - os.fchdir(fd) - except OSError, e: - raise wrap_oserror(space, e) -fchdir.unwrap_spec = [ObjSpace, int] -fchdir.__doc__ = os.fchdir.__doc__ - -def fpathconf(space, fd, w_name): - w_name_type = space.type(w_name) - is_str = space.is_w(w_name_type, space.w_str) - is_int = space.is_w(w_name_type, space.w_int) - - res = '' - try: - if is_str: - res = os.fpathconf(fd, space.str_w(w_name)) - elif is_int: - res = os.fpathconf(fd, space.int_w(w_name)) - else: - raise OperationError(space.w_TypeError, - space.wrap("configuration names must be strings or integers")) - except OSError, e: - raise wrap_oserror(space, e) - except ValueError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.args[0])) - else: - return space.wrap(res) -fpathconf.unwrap_spec = [ObjSpace, int, W_Root] -fpathconf.__doc__ = os.fpathconf.__doc__ - -def pathconf(space, path, w_name): - w_name_type = space.type(w_name) - is_str = space.is_w(w_name_type, space.w_str) - is_int = space.is_w(w_name_type, space.w_int) - - res = '' - try: - if is_str: - res = os.pathconf(path, space.str_w(w_name)) - elif is_int: - res = os.pathconf(path, space.int_w(w_name)) - else: - raise OperationError(space.w_TypeError, - space.wrap("configuration names must be strings or integers")) - except OSError, e: - raise wrap_oserror(space, e) - except ValueError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.args[0])) - else: - return space.wrap(res) -pathconf.unwrap_spec = [ObjSpace, str, W_Root] -pathconf.__doc__ = os.pathconf.__doc__ - -def getegid(space): - return space.wrap(os.getegid()) -getegid.unwrap_spec = [ObjSpace] -getegid.__doc__ = os.getegid.__doc__ - -def geteuid(space): - return space.wrap(os.geteuid()) -geteuid.unwrap_spec = [ObjSpace] -geteuid.__doc__ = os.geteuid.__doc__ - -def getgid(space): - return space.wrap(os.getgid()) -getgid.unwrap_spec = [ObjSpace] -getgid.__doc__ = os.getgid.__doc__ - -def getuid(space): - return space.wrap(os.getuid()) -getuid.unwrap_spec = [ObjSpace] -getuid.__doc__ = os.getuid.__doc__ - -def getpgid(space, pid): - try: - res = os.getpgid(pid) - except OSError, e: - raise wrap_oserror(space, e) - else: - return space.wrap(res) -getpgid.unwrap_spec = [ObjSpace, int] -getpgid.__doc__ = os.getpgid.__doc__ - -def getpid(space): - return space.wrap(os.getpid()) -getpid.unwrap_spec = [ObjSpace] -getpid.__doc__ = os.getpid.__doc__ - -def getppid(space): - return space.wrap(os.getppid()) -getppid.unwrap_spec = [ObjSpace] -getppid.__doc__ = os.getppid.__doc__ - -def getpgrp(space): - return space.wrap(os.getpgrp()) -getpgrp.unwrap_spec = [ObjSpace] -getpgrp.__doc__ = os.getpgrp.__doc__ - -def getsid(space, pid): - try: - res = os.getsid(pid) - except OSError, e: - raise wrap_oserror(space, e) - else: - return space.wrap(res) -getsid.unwrap_spec = [ObjSpace, int] -getsid.__doc__ = os.getsid.__doc__ - -def getlogin(space): - return space.wrap(os.getlogin()) -getlogin.unwrap_spec = [ObjSpace] -getlogin.__doc__ = os.getlogin.__doc__ - -def getgroups(space): - return space.newlist(os.getgroups()) -getgroups.unwrap_spec = [ObjSpace] -getgroups.__doc__ = os.getgroups.__doc__ - -def getloadavg(space): - try: - res = os.getloadavg() - except OSError, e: - raise wrap_oserror(space, e) - else: - return space.wrap(res) -getloadavg.unwrap_spec = [ObjSpace] -getloadavg.__doc__ = os.getloadavg.__doc__ - -def major(space, device): - return space.wrap(os.major(device)) -major.unwrap_spec = [ObjSpace, int] -major.__doc__ = os.major.__doc__ - -def minor(space, device): - return space.wrap(os.minor(device)) -minor.unwrap_spec = [ObjSpace, int] -minor.__doc__ = os.minor.__doc__ - -def sysconf(space, w_name): - w_name_type = space.type(w_name) - is_str = space.is_w(w_name_type, space.w_str) - is_int = space.is_w(w_name_type, space.w_int) - - res = '' - try: - if is_str: - res = os.sysconf(space.str_w(w_name)) - elif is_int: - res = os.sysconf(space.int_w(w_name)) - else: - raise OperationError(space.w_TypeError, - space.wrap("configuration names must be strings or integers")) - except OSError, e: - raise wrap_oserror(space, e) - except ValueError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.args[0])) - else: - return space.wrap(res) -sysconf.unwrap_spec = [ObjSpace, W_Root] -sysconf.__doc__ = os.sysconf.__doc__ - -def uname(space): - name = os.uname() - name_w = [space.wrap(i) for i in name] - return space.newtuple(name_w) -uname.unwrap_spec = [ObjSpace] -uname.__doc__ == os.uname.__doc__ - -def umask(space, mask): - return space.wrap(os.umask(mask)) -umask.unwrap_spec = [ObjSpace, int] -umask.__doc__ == os.umask.__doc__ - -def ttyname(space, fd): - try: - res = os.ttyname(fd) - except OSError, e: - raise wrap_oserror(space, e) - else: - return space.wrap(res) -ttyname.unwrap_spec = [ObjSpace, int] -ttyname.__doc__ = os.ttyname.__doc__ Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon Aug 21 15:27:31 2006 @@ -29,16 +29,13 @@ fd = posix.open(path, posix.O_RDONLY, 0777) fd2 = posix.dup(fd) assert not posix.isatty(fd2) - fd3 = 1 - posix.dup2(fd2, fd3) - assert not posix.isatty(fd3) s = posix.read(fd, 1) assert s == 't' posix.lseek(fd, 5, 0) s = posix.read(fd, 1) assert s == 'i' stat = posix.fstat(fd) - assert stat # XXX + assert stat # XXX posix.close(fd2) posix.close(fd) @@ -71,24 +68,8 @@ #UMPF cpython raises IOError ex(self.posix.ftruncate, UNUSEDFD, 123) ex(self.posix.fstat, UNUSEDFD) ex(self.posix.stat, "qweqwehello") - ex(self.posix.lstat, "qweqwehello") # how can getcwd() raise? ex(self.posix.dup, UNUSEDFD) - ex(self.posix.dup2, UNUSEDFD, UNUSEDFD) - ex(self.posix.unlink, str(UNUSEDFD)) - ex(self.posix.remove, str(UNUSEDFD)) - ex(self.posix.chdir, str(UNUSEDFD)) - ex(self.posix.rmdir, str(UNUSEDFD)) - ex(self.posix.listdir, str(UNUSEDFD)) - ex(self.posix.chmod, str(UNUSEDFD), 0777) - ex(self.posix.chown, str(UNUSEDFD), -1, -1) - ex(self.posix.chroot, str(UNUSEDFD)) - ex(self.posix.fchdir, UNUSEDFD) - ex(self.posix.getpgid, UNUSEDFD) - ex(self.posix.getsid, UNUSEDFD) - ex(self.posix.link, "foo", "foo") - ex(self.posix.readlink, "foo") - ex(self.posix.ttyname, UNUSEDFD) def test_fdopen(self): path = self.path @@ -114,9 +95,8 @@ assert isinstance(self.posix.strerror(0), str) assert isinstance(self.posix.strerror(1), str) - def test_fork(self): - import os - if hasattr(__import__(os.name), "fork"): + if hasattr(__import__(os.name), "fork"): + def test_fork(self): os = self.posix pid = os.fork() if pid == 0: # child @@ -124,298 +104,16 @@ pid1, status1 = os.waitpid(pid, 0) assert pid1 == pid # XXX check status1 - else: - skip("fork not supported") - - def test_read_write(self): - path = self.path - posix = self.posix - fd = posix.open(path, posix.O_WRONLY) - posix.write(fd, "\nfoo") - posix.close(fd) - fd = posix.open(path, posix.O_RDONLY) - raises(OSError, posix.write, fd, "foo") - buf = [] - buf.append(posix.read(fd, 4)) - assert len(buf[0]) == 4 - buf.append(posix.read(fd, 255)) - assert "".join(buf) == "\nfoo is a test" - posix.close(fd) - - def test_unlink(self): - import os - posix = self.posix - path = "foo" - fd = posix.open(path, posix.O_WRONLY | posix.O_CREAT) - assert os.path.exists(path) - try: - posix.unlink(path) - except OSError: - print "can't delete '%s'" % path - else: - assert not os.path.exists(path) - posix.close(fd) - test_remove = test_unlink - - def test_getcwd_chdir(self): - import os - posix = self.posix - path = os.path.split(posix.getcwd())[1] - posix.chdir('..') - posix.chdir(path) - posix.getcwd() - - def test_mkdir_rmdir(self): - import os - posix = self.posix - path = 'foo' - try: - posix.mkdir(path) - except OSError: - print "cannot create '%s' directory" % path - else: - assert os.path.exists(path) - try: - posix.rmdir(path) - except OSError: - print "cannot remove '%s' directory" % path - else: - assert not os.path.exists(path) - - def test_pipe(self): - posix = self.posix - r, w = posix.pipe() - data = 'foobar' - amount = posix.write(w, data) - posix.close(w) - read_data = posix.read(r, amount) - posix.close(r) - assert read_data == data - - def test_rename(self): - path = self.path - posix = self.posix - new_path = "foo" - posix.rename(path, new_path) - posix.rename(new_path, path) - - def test_ftruncate(self): - import os - if hasattr(__import__(os.name), "ftruncate"): - pdir = self.pdir - posix = self.posix - path = os.path.join(pdir, 'file1') - fd = posix.open(path, posix.O_WRONLY) - posix.ftruncate(fd, 2) - assert posix.stat(path)[6] == 2 - posix.close(fd) - raises(IOError, posix.ftruncate, 123123, 1) - else: - skip("ftruncate not supported") - - def test_abort(self): - import os - if hasattr(__import__(os.name), "fork"): - posix = self.posix - pid = posix.fork() - if pid == 0: # child - posix.abort() - else: - skip("can't test abort, because fork is missing") - - def test_access(self): - posix = self.posix - assert posix.access('.', posix.W_OK) - - def test_chown(self): - import os - if hasattr(__import__(os.name), "chown"): - posix = self.posix - path = self.path - stat_info = posix.stat(path) - uid, gid = stat_info.st_uid, stat_info.st_gid - posix.chown(path, -1, -1) - posix.lchown(path, -1, -1) - stat_info = posix.stat(path) - assert uid == stat_info.st_uid - assert gid == stat_info.st_gid - else: - skip("chown not supported") - - def test_confstr(self): - import os - if hasattr(__import__(os.name), "confstr"): - posix = self.posix - assert isinstance(posix.confstr_names, dict) - name = posix.confstr_names.keys()[0] - assert isinstance(posix.confstr(name), str) - val = posix.confstr_names.values()[0] - assert isinstance(posix.confstr(val), str) - raises(ValueError, posix.confstr, 'xYz') - raises(TypeError, posix.confstr, None) - raises(TypeError, posix.confstr, dict()) - assert isinstance(posix.confstr(12345), str) - else: - skip("confstr and confstr_names not supported") - - def test_ctermid(self): - import os - if hasattr(__import__(os.name), "ctermid"): - assert isinstance(self.posix.ctermid(), str) - - def test_fchdir(self): - import os - if hasattr(__import__(os.name), "fchdir"): - pdir = self.pdir - posix = self.posix - whereami = posix.getcwd() - fd = posix.open(pdir, posix.O_RDONLY) - posix.fchdir(fd) - posix.chdir(whereami) - - def test_fpathconf(self): - import os - if hasattr(__import__(os.name), "fpathconf"): - posix = self.posix - fd = posix.open(self.path, posix.O_RDONLY) - assert isinstance(posix.pathconf_names, dict) - name = posix.pathconf_names.keys()[-1] - assert isinstance(posix.fpathconf(fd, name), int) - val = posix.pathconf_names.values()[-1] - assert isinstance(posix.fpathconf(fd, val), int) - raises(ValueError, posix.fpathconf, fd, 'xYz') - raises(TypeError, posix.fpathconf, fd, None) - raises(TypeError, posix.fpathconf, fd, dict()) - else: - skip("fpathconf and pathconf_names not supported") - - def test_pathconf(self): - import os - if hasattr(__import__(os.name), "pathconf"): - posix = self.posix - path = self.path - assert isinstance(posix.pathconf_names, dict) - name = posix.pathconf_names.keys()[-1] - assert isinstance(posix.pathconf(path, name), int) - val = posix.pathconf_names.values()[-1] - assert isinstance(posix.pathconf(path, val), int) - raises(ValueError, posix.pathconf, path, 'xYz') - raises(TypeError, posix.pathconf, path, None) - raises(TypeError, posix.pathconf, path, dict()) - else: - skip("pathconf nad pathconf_names not supported") - - def test_getcwdu(self): - assert isinstance(self.posix.getcwdu(), unicode) - - def test_get_ids(self): - import os - if hasattr(__import__(os.name), "getegid"): - posix = self.posix - assert isinstance(posix.getegid(), int) - assert isinstance(posix.geteuid(), int) - assert isinstance(posix.getgid(), int) - assert isinstance(posix.getuid(), int) - assert posix.getpgid(0) == posix.getpgrp() - assert isinstance(posix.getpid(), int) - assert isinstance(posix.getppid(), int) - assert isinstance(posix.getsid(0), int) - - def test_getlogin(self): - import os - if hasattr(__import__(os.name), "getlogin"): - posix = self.posix - assert isinstance(posix.getlogin(), str) - # assert posix.getlogin() == pwd.getpwuid(os.getuid())[0] - - def test_getgroups(self): - import os - if hasattr(__import__(os.name), "getgroups"): - assert isinstance(self.posix.getgroups(), list) - - def test_getloadavg(self): - import os - if hasattr(__import__(os.name), "getloadavg"): - posix = self.posix - load = posix.getloadavg() - assert isinstance(load, tuple) - assert len(load) == 3 - - def test_linking(self): - import os - if hasattr(__import__(os.name), "symlink"): - posix = self.posix - pdir = self.pdir - path = self.path - link = os.path.join(pdir, 'link') - posix.symlink(path, link) - hard_link = os.path.join(pdir, 'hard_link') - posix.link(path, hard_link) - assert posix.readlink(link) == path - - def test_major_minor(self): - posix = self.posix - fd = posix.open("/dev/urandom", posix.O_RDONLY) - assert isinstance(posix.major(fd), int) - assert isinstance(posix.minor(fd), int) - def test_sysconf(self): - import os - if hasattr(__import__(os.name), "sysconf"): - posix = self.posix - assert isinstance(posix.sysconf_names, dict) - name = posix.sysconf_names.keys()[0] - assert isinstance(posix.sysconf(name), int) - val = posix.sysconf_names.values()[0] - assert isinstance(posix.sysconf(val), int) - raises(ValueError, posix.sysconf, 'xYz') - raises(TypeError, posix.sysconf, None) - raises(TypeError, posix.sysconf, dict()) - else: - skip("confstr and confstr_names not supported") - - def test_wait(self): - import os - if hasattr(__import__(os.name), "wait"): - posix = self.posix - pid = posix.fork() - if pid == 0: # child - posix._exit(4) - pid1, status1 = os.wait() - assert pid1 == pid - else: - skip("wait not supported") - - def test_uname(self): - import os - if hasattr(__import__(os.name), "uname"): - uname = self.posix.uname() - assert isinstance(uname, tuple) - assert len(uname) == 5 - - def test_umask(self): - import os - if hasattr(__import__(os.name), "umask"): - assert isinstance(self.posix.umask(022), int) - - def test_ttyname(self): - import os - if hasattr(__import__(os.name), "umask"): - assert isinstance(self.posix.ttyname(0), str) - class AppTestEnvironment(object): def setup_class(cls): cls.space = space cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) cls.w_os = space.appexec([], "(): import os; return os") cls.w_path = space.wrap(str(path)) - def test_environ(self): posix = self.posix - assert posix.environ['PATH'] - del posix.environ['PATH'] - def fn(): posix.environ['PATH'] - raises(KeyError, fn) + os = self.os if hasattr(__import__(os.name), "unsetenv"): def test_unsetenv_nonexisting(self): From arigo at codespeak.net Mon Aug 21 15:32:23 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 15:32:23 +0200 (CEST) Subject: [pypy-svn] r31441 - in pypy/dist/pypy/objspace/flow: . test Message-ID: <20060821133223.514B310050@code0.codespeak.net> Author: arigo Date: Mon Aug 21 15:32:20 2006 New Revision: 31441 Modified: pypy/dist/pypy/objspace/flow/flowcontext.py pypy/dist/pypy/objspace/flow/specialcase.py pypy/dist/pypy/objspace/flow/test/test_objspace.py Log: (arre, pedronis, arigo) Show a sane error when flowing functions with broken import statements. Refactored the flow space tests a bit. Modified: pypy/dist/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/dist/pypy/objspace/flow/flowcontext.py (original) +++ pypy/dist/pypy/objspace/flow/flowcontext.py Mon Aug 21 15:32:20 2006 @@ -281,6 +281,10 @@ except OperationError, e: #print "OE", e.w_type, e.w_value + if (self.space.do_imports_immediately and + e.w_type is self.space.w_ImportError): + raise ImportError('import statement always raises %s' % ( + e,)) link = self.make_link([e.w_type, e.w_value], self.graph.exceptblock) self.recorder.crnt_block.closeblock(link) Modified: pypy/dist/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/dist/pypy/objspace/flow/specialcase.py (original) +++ pypy/dist/pypy/objspace/flow/specialcase.py Mon Aug 21 15:32:20 2006 @@ -2,6 +2,7 @@ from pypy.objspace.flow.model import Constant from pypy.objspace.flow.operation import OperationName, Arity from pypy.interpreter.gateway import ApplevelClass +from pypy.interpreter.error import OperationError from pypy.tool.cache import Cache def sc_import(space, fn, args): @@ -13,7 +14,11 @@ if space.do_imports_immediately: name, glob, loc, frm = (space.unwrap(w_name), space.unwrap(w_glob), space.unwrap(w_loc), space.unwrap(w_frm)) - return space.wrap(__import__(name, glob, loc, frm)) + try: + mod = __import__(name, glob, loc, frm) + except ImportError, e: + raise OperationError(space.w_ImportError, space.wrap(str(e))) + return space.wrap(mod) # redirect it, but avoid exposing the globals w_glob = Constant({}) return space.do_operation('simple_call', Constant(__import__), Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Mon Aug 21 15:32:20 2006 @@ -1,19 +1,18 @@ import autopath +import py from pypy.objspace.flow.model import Constant, Block, Link, Variable, traverse from pypy.objspace.flow.model import flatten from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph from pypy.objspace.flow import FlowObjSpace from pypy.objspace.flow import objspace +from pypy import conftest import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 -class TestFlowObjSpace: - def setup_class(cls): - cls.space = FlowObjSpace() - +class Base: def codetest(self, func): import inspect try: @@ -23,17 +22,17 @@ #name = func.func_name graph = self.space.build_flow(func) graph.source = inspect.getsource(func) + self.show(graph) return graph - def reallyshow(self, x): - x.show() - #import os - #from pypy.translator.tool.make_dot import make_dot - #dest = make_dot(x.name, x) - #os.system('gv %s' % str(dest)) + def show(self, graph): + if conftest.option.view: + graph.show() + - def show(self, x): - pass # or self.reallyshow(x) +class TestFlowObjSpace(Base): + def setup_class(cls): + cls.space = FlowObjSpace() def all_operations(self, graph): result = {} @@ -54,7 +53,6 @@ assert len(x.startblock.exits) == 1 link, = x.startblock.exits assert link.target == x.returnblock - self.show(x) #__________________________________________________________ def simplefunc(x): @@ -72,7 +70,6 @@ def test_simplebranch(self): x = self.codetest(self.simplebranch) - self.show(x) #__________________________________________________________ def ifthenelse(i, j): @@ -82,7 +79,6 @@ def test_ifthenelse(self): x = self.codetest(self.simplebranch) - self.show(x) #__________________________________________________________ def loop(x): @@ -102,7 +98,6 @@ def test_print(self): x = self.codetest(self.print_) - self.show(x) #__________________________________________________________ def while_(i): @@ -111,7 +106,6 @@ def test_while(self): x = self.codetest(self.while_) - self.show(x) #__________________________________________________________ def union_easy(i): @@ -123,7 +117,6 @@ def test_union_easy(self): x = self.codetest(self.union_easy) - self.show(x) #__________________________________________________________ def union_hard(i): @@ -133,7 +126,6 @@ def test_union_hard(self): x = self.codetest(self.union_hard) - self.show(x) #__________________________________________________________ def while_union(i): @@ -145,7 +137,6 @@ def test_while_union(self): x = self.codetest(self.while_union) - self.show(x) #__________________________________________________________ def simple_for(lst): @@ -156,7 +147,6 @@ def test_simple_for(self): x = self.codetest(self.simple_for) - self.show(x) #__________________________________________________________ def nested_whiles(i, j): @@ -173,7 +163,6 @@ def test_nested_whiles(self): x = self.codetest(self.nested_whiles) - self.show(x) #__________________________________________________________ def break_continue(x): @@ -193,7 +182,6 @@ def test_break_continue(self): x = self.codetest(self.break_continue) - self.show(x) #__________________________________________________________ def unpack_tuple(lst): @@ -201,7 +189,6 @@ def test_unpack_tuple(self): x = self.codetest(self.unpack_tuple) - self.show(x) #__________________________________________________________ def reverse_3(lst): @@ -214,7 +201,6 @@ def test_reverse_3(self): x = self.codetest(self.reverse_3) - self.show(x) #__________________________________________________________ def finallys(lst): @@ -237,7 +223,6 @@ def test_finallys(self): x = self.codetest(self.finallys) - self.show(x) #__________________________________________________________ def const_pow(): @@ -245,7 +230,6 @@ def test_const_pow(self): x = self.codetest(self.const_pow) - self.show(x) #__________________________________________________________ def implicitException(lst): @@ -400,7 +384,6 @@ def test_freevar(self): x = self.codetest(self.freevar(3)) - self.show(x) #__________________________________________________________ def raise1(msg): @@ -408,8 +391,8 @@ def test_raise1(self): x = self.codetest(self.raise1) - self.show(x) simplify_graph(x) + self.show(x) ops = x.startblock.operations assert len(ops) == 2 assert ops[0].opname == 'simple_call' @@ -425,7 +408,6 @@ def test_raise2(self): x = self.codetest(self.raise2) - self.show(x) # XXX can't check the shape of the graph, too complicated... #__________________________________________________________ @@ -434,7 +416,6 @@ def test_raise3(self): x = self.codetest(self.raise3) - self.show(x) # XXX can't check the shape of the graph, too complicated... #__________________________________________________________ @@ -443,7 +424,6 @@ def test_raise4(self): x = self.codetest(self.raise4) - self.show(x) #__________________________________________________________ def raisez(z, tb): @@ -451,7 +431,6 @@ def test_raisez(self): x = self.codetest(self.raisez) - self.show(x) #__________________________________________________________ def raise_and_catch_1(exception_instance): @@ -463,7 +442,6 @@ def test_raise_and_catch_1(self): x = self.codetest(self.raise_and_catch_1) - self.show(x) #__________________________________________________________ def catch_simple_call(): @@ -475,7 +453,6 @@ def test_catch_simple_call(self): x = self.codetest(self.catch_simple_call) - self.show(x) #__________________________________________________________ def dellocal(): @@ -486,7 +463,6 @@ def test_dellocal(self): x = self.codetest(self.dellocal) - self.show(x) #__________________________________________________________ def globalconstdict(name): @@ -496,7 +472,6 @@ def test_globalconstdict(self): x = self.codetest(self.globalconstdict) - self.show(x) #__________________________________________________________ @@ -539,7 +514,6 @@ def test_jump_target_specialization(self): x = self.codetest(self.jump_target_specialization) - self.show(x) def visitor(node): if isinstance(node, Block): for op in node.operations: @@ -592,7 +566,6 @@ def test_highly_branching_example(self): x = self.codetest(self.highly_branching_example) - self.show(x) assert len(flatten(x)) < 60 # roughly 20 blocks + 30 links #__________________________________________________________ @@ -691,24 +664,49 @@ call_args.append(op) traverse(visit, graph) assert not call_args - -class TestFlowObjSpaceDelay: + def test_catch_importerror_1(self): + def f(): + try: + import pypy.this_does_not_exist + except ImportError: + return 1 + graph = self.codetest(f) + simplify_graph(graph) + self.show(graph) + assert not graph.startblock.operations + assert len(graph.startblock.exits) == 1 + assert graph.startblock.exits[0].target is graph.returnblock + + def test_catch_importerror_2(self): + def f(): + try: + from pypy import this_does_not_exist + except ImportError: + return 1 + graph = self.codetest(f) + simplify_graph(graph) + self.show(graph) + assert not graph.startblock.operations + assert len(graph.startblock.exits) == 1 + assert graph.startblock.exits[0].target is graph.returnblock + + def test_importerror_1(self): + def f(): + import pypy.this_does_not_exist + py.test.raises(ImportError, 'self.codetest(f)') + + def test_importerror_2(self): + def f(): + from pypy import this_does_not_exist + py.test.raises(ImportError, 'self.codetest(f)') + + +class TestFlowObjSpaceDelay(Base): def setup_class(cls): cls.space = FlowObjSpace() cls.space.do_imports_immediately = False - def codetest(self, func): - import inspect - try: - func = func.im_func - except AttributeError: - pass - #name = func.func_name - graph = self.space.build_flow(func) - graph.source = inspect.getsource(func) - return graph - def test_import_something(self): def f(): from some.unknown.module import stuff From arigo at codespeak.net Mon Aug 21 15:33:19 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 15:33:19 +0200 (CEST) Subject: [pypy-svn] r31442 - pypy/dist/pypy/tool Message-ID: <20060821133319.130B210050@code0.codespeak.net> Author: arigo Date: Mon Aug 21 15:33:17 2006 New Revision: 31442 Modified: pypy/dist/pypy/tool/error.py Log: (people) Improve error message. Modified: pypy/dist/pypy/tool/error.py ============================================================================== --- pypy/dist/pypy/tool/error.py (original) +++ pypy/dist/pypy/tool/error.py Mon Aug 21 15:33:17 2006 @@ -69,7 +69,7 @@ msg.append('-+' * 30) from pypy.annotation import model if model.DEBUG: - msg.append("Operation cannot succeed") + msg.append("Blocked block -- operation cannot succeed") _, _, operindex = annotator.why_not_annotated[block][1].break_at oper = block.operations[operindex] msg.append(" " + str(oper)) From ericvrp at codespeak.net Mon Aug 21 16:13:28 2006 From: ericvrp at codespeak.net (ericvrp at codespeak.net) Date: Mon, 21 Aug 2006 16:13:28 +0200 (CEST) Subject: [pypy-svn] r31444 - pypy/dist/pypy/bin Message-ID: <20060821141328.DC57A10074@code0.codespeak.net> Author: ericvrp Date: Mon Aug 21 16:13:27 2006 New Revision: 31444 Modified: pypy/dist/pypy/bin/jscompile.py Log: very slight refactoring to make usage from outside cleaner Modified: pypy/dist/pypy/bin/jscompile.py ============================================================================== --- pypy/dist/pypy/bin/jscompile.py (original) +++ pypy/dist/pypy/bin/jscompile.py Mon Aug 21 16:13:27 2006 @@ -29,12 +29,12 @@ l.append("NonConstant(%s)" % repr(func_data.func_defaults[i])) return "(%s)" % ",".join(l) -def _main(argv): - if len(argv) < 3: +def rpython2javascript(argv): + if len(argv) < 2: print __doc__ sys.exit(0) - module_name = argv[1] - function_names = argv[2:] + module_name = argv[0] + function_names = argv[1:] mod = __import__(module_name, None, None, ["Module"]) for func_name in function_names: if func_name not in mod.__dict__: @@ -57,4 +57,4 @@ debug(driver) if __name__ == '__main__': - _main(sys.argv) + rpython2javascript(sys.argv[1:]) From arigo at codespeak.net Mon Aug 21 17:04:11 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 17:04:11 +0200 (CEST) Subject: [pypy-svn] r31445 - in pypy/dist/pypy/annotation: . test Message-ID: <20060821150411.1BE3410077@code0.codespeak.net> Author: arigo Date: Mon Aug 21 17:04:07 2006 New Revision: 31445 Modified: pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: (pedronis, arre, arigo) Annotation bug and fix: confusion between multiple prebuilt '{}' or '[]' because they compare equal in the .const :-( Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Mon Aug 21 17:04:07 2006 @@ -330,6 +330,8 @@ self.immutable_cache[key] = result for e in x: result.listdef.generalize(self.immutablevalue(e)) + result.const_box = key + return result elif tp is dict or tp is r_dict: key = Constant(x) try: @@ -355,6 +357,8 @@ pass else: done = True + result.const_box = key + return result elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Mon Aug 21 17:04:07 2006 @@ -55,11 +55,10 @@ funcgraph.source = inspect.getsource(func) return funcgraph - def reallyshow(self, graph): - import os - from pypy.translator.tool.make_dot import make_dot - dest = make_dot('b', graph) - os.system('gv %s' % str(dest)) + def show(self, a): + from pypy import conftest + if conftest.option.view: + a.translator.view() def test_simple_func(self): """ @@ -2213,6 +2212,49 @@ assert isinstance(s, annmodel.SomeInteger) assert s.nonneg + def test_prebuilt_mutables(self): + class A: + pass + class B: + pass + a1 = A() + a2 = A() + a1.d = {} # this tests confusion between the two '{}', which + a2.d = {} # compare equal + a1.l = [] + a2.l = [] + b = B() + b.d1 = a1.d + b.d2 = a2.d + b.l1 = a1.l + b.l2 = a2.l + + def dmutate(d): + d[123] = 321 + + def lmutate(l): + l.append(42) + + def readout(d, l): + return len(d) + len(l) + + def f(): + dmutate(b.d1) + dmutate(b.d2) + dmutate(a1.d) + dmutate(a2.d) + lmutate(b.l1) + lmutate(b.l2) + lmutate(a1.l) + lmutate(a2.l) + return readout(a1.d, a1.l) + readout(a2.d, a2.l) + + a = self.RPythonAnnotator() + a.build_types(f, []) + self.show(a) + v1, v2 = graphof(a, readout).getargs() + assert not a.bindings[v1].is_constant() + assert not a.bindings[v2].is_constant() def g(n): From arigo at codespeak.net Mon Aug 21 17:28:44 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 17:28:44 +0200 (CEST) Subject: [pypy-svn] r31446 - pypy/dist/pypy/rpython Message-ID: <20060821152844.20FDF10075@code0.codespeak.net> Author: arigo Date: Mon Aug 21 17:28:42 2006 New Revision: 31446 Modified: pypy/dist/pypy/rpython/llinterp.py Log: (arre, pedronis, arigo) Make llop.debug_print() print to the html log files as well. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Mon Aug 21 17:28:42 2006 @@ -399,12 +399,17 @@ def op_debug_print(self, *ll_args): from pypy.rpython.lltypesystem.rstr import STR + line = [] for arg in ll_args: T = lltype.typeOf(arg) if T == lltype.Ptr(STR): arg = ''.join(arg.chars) - print arg, - print + line.append(str(arg)) + line = ' '.join(line) + print line + tracer = self.llinterpreter.tracer + if tracer: + tracer.dump('\n[debug] %s\n' % (line,)) def op_debug_pdb(self, *ll_args): if self.llinterpreter.tracer: From arigo at codespeak.net Mon Aug 21 17:32:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 17:32:59 +0200 (CEST) Subject: [pypy-svn] r31447 - in pypy/dist/pypy/jit/timeshifter: . test Message-ID: <20060821153259.DA30110078@code0.codespeak.net> Author: arigo Date: Mon Aug 21 17:32:57 2006 New Revision: 31447 Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/timeshift.py Log: (pedronis, arre, arigo) Make the simple calling test pass. Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtyper.py Mon Aug 21 17:32:57 2006 @@ -219,6 +219,13 @@ return v else: bk = self.annotator.bookkeeper + # first close the current block + ts = self.timeshifter + v_jitstate = hop.llops.getjitstate() + hop.llops.genmixlevelhelpercall(rtimeshift.leave_block, + [ts.s_JITState], + [v_jitstate], + ts.s_JITState) hop.r_s_popfirstarg() args_hs = hop.args_s[:] # fixed is always false here @@ -227,11 +234,15 @@ args_v = hop.inputargs(*args_r) fnptr = self.getcallable(graph) self.timeshifter.schedule_graph(graph) - v_jitstate = hop.llops.getjitstate() - args_v.insert(0, v_jitstate) + v_builder = hop.llops.getcurbuilder() + args_v.insert(0, v_builder) args_v.insert(0, hop.llops.genconst(fnptr)) - v = hop.genop('direct_call', args_v, hop.r_result.lowleveltype) - return v + v = hop.genop('direct_call', args_v, + resulttype = ts.r_ResidualGraphBuilder.lowleveltype) + + c_name = inputconst(lltype.Void, 'inst_valuebox') + return hop.genop('getfield', [v, c_name], + resulttype = hop.r_result.lowleveltype) def handle_highlevel_operation(self, fnobj, hop): from pypy.jit.timeshifter.oop import OopSpecDesc, Index @@ -320,12 +331,19 @@ assert self.originalblock is not None return self.timeshifter.block2jitstate[self.originalblock] + def getcurbuilder(self): + v_jitstate = self.getjitstate() + c_name = inputconst(lltype.Void, 'inst_curbuilder') + return self.genop('getfield', [v_jitstate, c_name], + self.timeshifter.r_ResidualGraphBuilder.lowleveltype) + # ____________________________________________________________ class __extend__(pairtype(HintTypeSystem, hintmodel.SomeLLAbstractConstant)): def rtyper_makerepr((ts, hs_c), hrtyper): - if hs_c.is_fixed() or hs_c.eager_concrete: + if (hs_c.is_fixed() or hs_c.eager_concrete or + hs_c.concretetype is lltype.Void): return hrtyper.getgreenrepr(hs_c.concretetype) else: return hrtyper.getredrepr(hs_c.concretetype) Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Mon Aug 21 17:32:57 2006 @@ -562,7 +562,6 @@ assert insns == {'int_is_true': 1, 'int_add': 1} def test_call_simple(): - py.test.skip("in-progress") def ll_add_one(x): return x + 1 def ll_function(y): Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Mon Aug 21 17:32:57 2006 @@ -22,8 +22,6 @@ self.rtyper = rtyper self.hrtyper = HintRTyper(hannotator, self) self.latestexitindex = -1 - self.block2jitstate = {} - self.return_cache = None self.annhelper = annlowlevel.MixLevelHelperAnnotator(rtyper) self.s_ResidualGraphBuilder, self.r_ResidualGraphBuilder = self.s_r_instanceof(ResidualGraphBuilder) @@ -161,9 +159,12 @@ self.annhelper.finish() def timeshift_graph(self, graph): + #print 'timeshift_graph START', graph self.graph = graph self.dispatch_to = [] self.statecaches = [] + self.block2jitstate = {} + self.return_cache = None entering_links = flowmodel.mkentrymap(graph) originalblocks = list(graph.iterblocks()) @@ -219,6 +220,7 @@ annmodel.s_None) self.insert_start_setup() + #print 'timeshift_graph END', graph def insert_start_setup(self): newstartblock = self.insert_before_block(self.graph.startblock, None, closeblock=True) From arigo at codespeak.net Mon Aug 21 17:54:31 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 17:54:31 +0200 (CEST) Subject: [pypy-svn] r31448 - pypy/dist/pypy/jit/timeshifter Message-ID: <20060821155431.5B4381007A@code0.codespeak.net> Author: arigo Date: Mon Aug 21 17:54:29 2006 New Revision: 31448 Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/rtyper.py Log: (pedronis, arre, arigo) Fix & refactor our preliminary direct_call support. Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Mon Aug 21 17:54:29 2006 @@ -273,6 +273,14 @@ def ll_gvar_from_constant(ll_value): return rgenop.genconst(ll_value) +def before_call(jitstate): + leave_block(jitstate) + return jitstate.curbuilder + +def after_call(jitstate, newbuilder): + jitstate.curbuilder = newbuilder + return newbuilder.valuebox + # ____________________________________________________________ class ResidualGraphBuilder(rgenop.LowLevelOpBuilder): Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtyper.py Mon Aug 21 17:54:29 2006 @@ -222,10 +222,10 @@ # first close the current block ts = self.timeshifter v_jitstate = hop.llops.getjitstate() - hop.llops.genmixlevelhelpercall(rtimeshift.leave_block, - [ts.s_JITState], - [v_jitstate], - ts.s_JITState) + v_builder = hop.llops.genmixlevelhelpercall(rtimeshift.before_call, + [ts.s_JITState], + [v_jitstate], + ts.s_ResidualGraphBuilder) hop.r_s_popfirstarg() args_hs = hop.args_s[:] # fixed is always false here @@ -234,15 +234,14 @@ args_v = hop.inputargs(*args_r) fnptr = self.getcallable(graph) self.timeshifter.schedule_graph(graph) - v_builder = hop.llops.getcurbuilder() args_v.insert(0, v_builder) args_v.insert(0, hop.llops.genconst(fnptr)) - v = hop.genop('direct_call', args_v, - resulttype = ts.r_ResidualGraphBuilder.lowleveltype) - - c_name = inputconst(lltype.Void, 'inst_valuebox') - return hop.genop('getfield', [v, c_name], - resulttype = hop.r_result.lowleveltype) + v_newbuilder = hop.genop('direct_call', args_v, + ts.r_ResidualGraphBuilder.lowleveltype) + return hop.llops.genmixlevelhelpercall(rtimeshift.after_call, + [ts.s_JITState, ts.s_ResidualGraphBuilder], + [v_jitstate, v_newbuilder], + ts.s_RedBox) def handle_highlevel_operation(self, fnobj, hop): from pypy.jit.timeshifter.oop import OopSpecDesc, Index From arigo at codespeak.net Mon Aug 21 19:50:30 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 19:50:30 +0200 (CEST) Subject: [pypy-svn] r31450 - in pypy/dist/pypy/jit/timeshifter: . test Message-ID: <20060821175030.3BE1310075@code0.codespeak.net> Author: arigo Date: Mon Aug 21 19:50:27 2006 New Revision: 31450 Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/rtyper.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/timeshift.py Log: (pedronis, arre, arigo) More support for calls: saving local variables around calls. Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Mon Aug 21 19:50:27 2006 @@ -145,10 +145,13 @@ # ____________________________________________________________ # other jitstate/graph level operations -def enter_graph(builder): - return builder.build_jitstate() +def enter_graph(builder, backstate=None): + return builder.build_jitstate(backstate) def retrieve_jitstate_for_merge(states_dic, jitstate, key, redboxes): + mylocalredboxes = redboxes + redboxes = list(redboxes) + jitstate.extend_with_parent_locals(redboxes) if key not in states_dic: memo = rvalue.freeze_memo() frozens = [redbox.freeze(memo) for redbox in redboxes] @@ -196,8 +199,9 @@ box = box.copy(replace_memo) # copy to avoid patching the original box.genvar = rgenop.geninputarg(newblock, box.gv_type) if replace_memo.boxes: - for i in range(len(redboxes)): - redboxes[i] = redboxes[i].replace(replace_memo) + for i in range(len(mylocalredboxes)): + newbox = redboxes[i].replace(replace_memo) + mylocalredboxes[i] = redboxes[i] = newbox jitstate.curbuilder.leave_block() jitstate.curbuilder.enter_block(linkargs, newblock) memo = rvalue.freeze_memo() @@ -207,15 +211,25 @@ retrieve_jitstate_for_merge._annspecialcase_ = "specialize:arglltype(2)" def enter_block(jitstate, redboxes): + # 'redboxes' is a fixed-size list (s_box_list) of the current red boxes newblock = rgenop.newblock() incoming = [] memo = rvalue.enter_block_memo() - for i in range(len(redboxes)): - redboxes[i].enter_block(newblock, incoming, memo) + for redbox in redboxes: + redbox.enter_block(newblock, incoming, memo) + js = jitstate.backstate + while js is not None: + lrb = js.localredboxes + assert lrb is not None + for redbox in lrb: + redbox.enter_block(newblock, incoming, memo) + js = js.backstate jitstate.curbuilder.enter_block(incoming, newblock) return jitstate def dyn_enter_block(jitstate, redboxes): + # 'redboxes' is a var-sized list (s_box_accum) of *all* the boxes + # including the ones from the callers' locals newblock = rgenop.newblock() incoming = [] memo = rvalue.enter_block_memo() @@ -273,6 +287,9 @@ def ll_gvar_from_constant(ll_value): return rgenop.genconst(ll_value) +def save_locals(jitstate, redboxes): + jitstate.localredboxes = redboxes + def before_call(jitstate): leave_block(jitstate) return jitstate.curbuilder @@ -289,8 +306,8 @@ self.outgoinglink = link self.valuebox = None - def build_jitstate(self): - return JITState(self) + def build_jitstate(self, backstate=None): + return JITState(self, backstate) def enter_block(self, linkargs, newblock): rgenop.closelink(self.outgoinglink, linkargs, newblock) @@ -343,10 +360,18 @@ class JITState(object): # XXX obscure interface + localredboxes = None - def __init__(self, builder): + def __init__(self, builder, backstate=None): self.split_queue = [] self.return_queue = [] self.curbuilder = builder + self.backstate = backstate - + def extend_with_parent_locals(self, redboxes): + js = self.backstate + while js is not None: + lrb = js.localredboxes + assert lrb is not None + redboxes.extend(lrb) + js = js.backstate Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtyper.py Mon Aug 21 19:50:27 2006 @@ -234,8 +234,7 @@ args_v = hop.inputargs(*args_r) fnptr = self.getcallable(graph) self.timeshifter.schedule_graph(graph) - args_v.insert(0, v_builder) - args_v.insert(0, hop.llops.genconst(fnptr)) + args_v[:0] = [hop.llops.genconst(fnptr), v_builder, v_jitstate] v_newbuilder = hop.genop('direct_call', args_v, ts.r_ResidualGraphBuilder.lowleveltype) return hop.llops.genmixlevelhelpercall(rtimeshift.after_call, @@ -243,6 +242,19 @@ [v_jitstate, v_newbuilder], ts.s_RedBox) + def translate_op_save_locals(self, hop): + ts = self.timeshifter + v_jitstate = hop.llops.getjitstate() + boxes_v = [] + for r, v in zip(hop.args_r, hop.args_v): + if isinstance(r, RedRepr): + boxes_v.append(v) + v_boxes = ts.build_box_list(hop.llops, boxes_v) + hop.llops.genmixlevelhelpercall(rtimeshift.save_locals, + [ts.s_JITState, ts.s_box_list], + [v_jitstate, v_boxes], + annmodel.s_None) + def handle_highlevel_operation(self, fnobj, hop): from pypy.jit.timeshifter.oop import OopSpecDesc, Index oopspecdesc = OopSpecDesc(fnobj) Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Mon Aug 21 19:50:27 2006 @@ -80,10 +80,10 @@ graph1 = ha.translator.graphs[0] llinterp = LLInterpreter(rtyper) builder = llinterp.eval_graph(htshift.ll_make_builder_graph, []) - graph1args = [builder] + graph1args = [builder, lltype.nullptr(htshift.r_JITState.lowleveltype.TO)] residual_graph_args = [] - assert len(graph1.getargs()) == 1 + len(values) - for i, (v, llvalue) in enumerate(zip(graph1.getargs()[1:], values)): + assert len(graph1.getargs()) == 2 + len(values) + for i, (v, llvalue) in enumerate(zip(graph1.getargs()[2:], values)): r = htshift.hrtyper.bindingrepr(v) residual_v = r.residual_values(llvalue) if len(residual_v) == 0: @@ -569,3 +569,12 @@ insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) assert res == 6 assert insns == {'int_add': 1} + +def test_call_2(): + def ll_add_one(x): + return x + 1 + def ll_function(y): + return ll_add_one(y) + y + insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + assert res == 11 + assert insns == {'int_add': 2} Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Mon Aug 21 19:50:27 2006 @@ -225,15 +225,16 @@ def insert_start_setup(self): newstartblock = self.insert_before_block(self.graph.startblock, None, closeblock=True) v_builder = varoftype(self.r_ResidualGraphBuilder.lowleveltype, 'builder') + v_backstate = varoftype(self.r_JITState.lowleveltype, 'backstate') v_jitstate = newstartblock.inputargs[0] - newstartblock.inputargs[0] = v_builder + newstartblock.inputargs[:1] = [v_builder, v_backstate] llops = HintLowLevelOpList(self, None) llops.genop('direct_call', [self.c_ll_clearcaches_ptr]) v_jitstate1 = llops.genmixlevelhelpercall(rtimeshift.enter_graph, - [self.s_ResidualGraphBuilder], - [v_builder], - self.s_JITState) + [self.s_ResidualGraphBuilder, self.s_JITState], + [v_builder, v_backstate], + self.s_JITState) llops.append(flowmodel.SpaceOperation('same_as', [v_jitstate1], v_jitstate)) newstartblock.operations = list(llops) @@ -691,6 +692,20 @@ if op.opname == 'direct_call': link = support.split_block_with_keepalive(block, i+1, annotator=self.hannotator) + + # the 'save_locals' pseudo-operation is used to save all + # alive local variables into the current JITState + args = list(link.args) + while op.result in args: + args.remove(op.result) + assert op is block.operations[i] + v_dummy = varoftype(lltype.Void) + self.hannotator.setbinding(v_dummy, annmodel.s_ImpossibleValue) + extraop = flowmodel.SpaceOperation('save_locals', + args, + v_dummy) + block.operations.insert(i, extraop) + block = link.target entering_links[block] = [link] blocks.append(block) From arigo at codespeak.net Mon Aug 21 19:53:21 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Aug 2006 19:53:21 +0200 (CEST) Subject: [pypy-svn] r31451 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20060821175321.A8DB810060@code0.codespeak.net> Author: arigo Date: Mon Aug 21 19:53:19 2006 New Revision: 31451 Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Log: Another test, already passing. Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Mon Aug 21 19:53:19 2006 @@ -578,3 +578,14 @@ insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) assert res == 11 assert insns == {'int_add': 2} + +def test_call_3(): + def ll_add_one(x): + return x + 1 + def ll_two(x): + return ll_add_one(ll_add_one(x)) - x + def ll_function(y): + return ll_two(y) * y + insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + assert res == 10 + assert insns == {'int_add': 2, 'int_sub': 1, 'int_mul': 1} From hpk at codespeak.net Tue Aug 22 12:24:46 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 22 Aug 2006 12:24:46 +0200 (CEST) Subject: [pypy-svn] r31460 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060822102446.85D6110036@code0.codespeak.net> Author: hpk Date: Tue Aug 22 12:24:44 2006 New Revision: 31460 Added: pypy/extradoc/sprintinfo/ireland-2006/planning.txt (contents, props changed) Log: planning session Added: pypy/extradoc/sprintinfo/ireland-2006/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/ireland-2006/planning.txt Tue Aug 22 12:24:44 2006 @@ -0,0 +1,36 @@ +Ireland sprint planning +-------------------------- + +timings: + monday, tuesday, wednesday full days + thursday half + friday 2pm- + 5pm tutorial + Q & A session + bea arranges a dinner table friday evening + +* source optimizations + (mwh, arre) + - builtin lookups + - caching method lookups + - profiling + +* ext compiler + maybe a topic for the weekend + +* JIT + - so far generating a jit for simple interpreters kind of + works + (armin, samuele) + - produce machine code in-memory from running the generated CFGs + - maybe think about refining the interface to generating + I386 machine code (also considering a PPC asm backend) + + +* distributed testing + (maciej, hpk) + - use "/usr/bin/rsync" to sync to remote + - master does collection of items + - "leaf" items are asynchronously send to "Executors" (running on nodes) + - super-simple reporting + + From arigo at codespeak.net Tue Aug 22 12:36:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 12:36:59 +0200 (CEST) Subject: [pypy-svn] r31461 - in pypy/dist/pypy/jit/codegen/i386: . test Message-ID: <20060822103659.E087310076@code0.codespeak.net> Author: arigo Date: Tue Aug 22 12:36:56 2006 New Revision: 31461 Added: pypy/dist/pypy/jit/codegen/i386/codebuf.py - copied unchanged from r31450, pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py - copied, changed from r31450, pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Removed: pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Log: (pedronis, arigo) Renamed a module. From bea at codespeak.net Tue Aug 22 15:58:48 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Tue, 22 Aug 2006 15:58:48 +0200 (CEST) Subject: [pypy-svn] r31471 - pypy/dist/pypy/doc Message-ID: <20060822135848.D27681005A@code0.codespeak.net> Author: bea Date: Tue Aug 22 15:58:47 2006 New Revision: 31471 Modified: pypy/dist/pypy/doc/dev_method.txt Log: updated text with all sprints done prior to Limerick one + added people that had participated and contributed Modified: pypy/dist/pypy/doc/dev_method.txt ============================================================================== --- pypy/dist/pypy/doc/dev_method.txt (original) +++ pypy/dist/pypy/doc/dev_method.txt Tue Aug 22 15:58:47 2006 @@ -31,7 +31,7 @@ What is a sprint and why are we sprinting? Originally the sprint methodology used in the Python community grew from -practices within Zope Corporation. The definition of a sprint is "two-day or +practices within Zope3 development. The definition of a sprint is "two-day or three-day focused development session, in which developers pair off together in a room and focus on building a particular subsystem". @@ -41,9 +41,9 @@ noted to have more than that. This is the recommendation and it is probably based on the idea of having a critical mass of people who can interact/communicate and work without adding the need for more than just - the absolute necessary coordination time. The sprints during 2005 have + the absolute necessary coordination time. The sprints during 2005 and 2006 have been having ca 13-14 people per sprint, the highest number of participants - during a PyPy sprint has been 20 developers) + during a PyPy sprint has been 24 developers) * a coach (the coach is the "manager" of the sprint, he/she sets the goals, prepares, leads and coordinate the work and track progress and makes this @@ -94,8 +94,8 @@ As always with methodologies you have to adapt them to fit your project (and not the other way around which is much too common). The PyPy team have been -sprinting since early 2003 and have done 11 sprints so far, 10 in Europe and 1 -in the USA. Certain practices have proven to be more successful within this +sprinting since early 2003 and have done 21 sprints so far, 18 in Europe, 2 +in the USA and 1 in Asia. Certain practices have proven to be more successful within this team and those are the one we are summarizing here. @@ -146,12 +146,12 @@ has the venue were the sprint is planned to be weird rules for access to their network etc etc? - Whiteboards are useful tools and good to have. Beamers (1-2) are very - useful for the status meetings and should be available, at least 1. The + Whiteboards are useful tools and good to have. Beamers (PyPy jargon for a projector) + are very useful for the status meetings and should be available, at least 1. The project also owns one beamer - specifically for sprint purposes. The person making sure that the requirements for a good sprint venue is - being met should therefore have very good local conncetions or, preferrably + being met should therefore have very good local connections or, preferrably live there. 3. Information - discussions about content and goals (pre announcements) are @@ -176,7 +176,7 @@ One very important strategy when planning the venue is cost efficiency. Keeping accomodation and food/travel costs as low as possible makes sure that more people can afford to visit or join the sprint - fully. The EU funded parts of the project do have a so called sprint budget + fully. The partially EU funded parts of the project do have a so called sprint budget which we use to try to help developers to participate in our sprints (travel expenses and accomodation) and because most of the funding is so called matched funding we pay for most of our expenses in our own @@ -251,14 +251,26 @@ Amsterdam Dec 2003 Europython /Gothenburg June 2004 - Vilnius 2004-11-15 - 2004-11-23 - Leysin 2005-01-22 - 2005-01-29 + Vilnius November 2004 + Leysin January 2005 PyCon - /Washington 2005-03-19 - 2005-03-22 + /Washington March 2005 Europython - /Gothenburg 2005-07-01 - 2005-07-07 - Hildesheim 2005-07-25 - 2005-07-31 - Heidelberg 2005-08-22 - 2005-08-29 + /Gothenburg June 2005 + Hildesheim July 2005 + Heidelberg August 2005 + Paris October 2005 + Gothenburg December 2005 + Mallorca January 2006 + PyCon/Dallas February 2006 + LouvainLaNeuve March 2006 + Leysin April 2006 + Tokyo April 2006 + D?sseldorf June 2006 + Europython/ + Geneva July 2006 + Limerick August 2006 + People who have participated and contributed during our sprints and thus contributing to PyPy:: @@ -306,5 +318,40 @@ Lutz Paelike Michael Chermside Beatrice D??ring + Bert Freudenberg + Boris Feigin + Amaury Forgeot d'Arc + Andrew Thompson + Valentino Volonghi + Aurelien Campeas + Stephan Busemann + Johan Hahn + Gerald Klix + Gene Oden + Josh Gilbert + Geroge Paci + Pat Maupin + Martin Blais + Stuart Williams + Jiwon Seo + Michael Twomey + Wanja Saatkamp + Alexandre Fayolle + Rapha?l Collet + Gr?goire Dooms + Guido Wesdorp + Maciej Fijalkowski + Antonio Cuni + Lawrence Oluyede + Fabrizio Milo + Alexander Schremmer + David Douard + Michele Frettoli + Simon Burton + Aaron Bingham + Pieter Zieschang + Sad Rejeb + Brian Sutherland + From bea at codespeak.net Tue Aug 22 16:02:09 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Tue, 22 Aug 2006 16:02:09 +0200 (CEST) Subject: [pypy-svn] r31472 - pypy/dist/pypy/doc Message-ID: <20060822140209.30EEB1005A@code0.codespeak.net> Author: bea Date: Tue Aug 22 16:02:06 2006 New Revision: 31472 Modified: pypy/dist/pypy/doc/dev_method.txt Log: formatting? Modified: pypy/dist/pypy/doc/dev_method.txt ============================================================================== --- pypy/dist/pypy/doc/dev_method.txt (original) +++ pypy/dist/pypy/doc/dev_method.txt Tue Aug 22 16:02:06 2006 @@ -245,31 +245,31 @@ The PyPy team have been sprinting on the following occasions:: - Hildesheim Feb 2003 - Gothenburg May 2003 - LovainLaNeuve June 2003 - Amsterdam Dec 2003 + Hildesheim Feb 2003 + Gothenburg May 2003 + LovainLaNeuve June 2003 + Amsterdam Dec 2003 Europython - /Gothenburg June 2004 - Vilnius November 2004 - Leysin January 2005 + /Gothenburg June 2004 + Vilnius November 2004 + Leysin January 2005 PyCon - /Washington March 2005 + /Washington March 2005 Europython - /Gothenburg June 2005 - Hildesheim July 2005 - Heidelberg August 2005 - Paris October 2005 - Gothenburg December 2005 - Mallorca January 2006 - PyCon/Dallas February 2006 - LouvainLaNeuve March 2006 - Leysin April 2006 - Tokyo April 2006 - D?sseldorf June 2006 + /Gothenburg June 2005 + Hildesheim July 2005 + Heidelberg August 2005 + Paris October 2005 + Gothenburg December 2005 + Mallorca January 2006 + PyCon/Dallas February 2006 + LouvainLaNeuve March 2006 + Leysin April 2006 + Tokyo April 2006 + D?sseldorf June 2006 Europython/ - Geneva July 2006 - Limerick August 2006 + Geneva July 2006 + Limerick August 2006 People who have participated and contributed during our sprints and thus @@ -341,17 +341,17 @@ Gr?goire Dooms Guido Wesdorp Maciej Fijalkowski - Antonio Cuni - Lawrence Oluyede - Fabrizio Milo - Alexander Schremmer - David Douard - Michele Frettoli - Simon Burton - Aaron Bingham - Pieter Zieschang - Sad Rejeb - Brian Sutherland + Antonio Cuni + Lawrence Oluyede + Fabrizio Milo + Alexander Schremmer + David Douard + Michele Frettoli + Simon Burton + Aaron Bingham + Pieter Zieschang + Sad Rejeb + Brian Sutherland From bea at codespeak.net Tue Aug 22 16:12:09 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Tue, 22 Aug 2006 16:12:09 +0200 (CEST) Subject: [pypy-svn] r31475 - pypy/dist/pypy/doc Message-ID: <20060822141209.263771005A@code0.codespeak.net> Author: bea Date: Tue Aug 22 16:12:06 2006 New Revision: 31475 Modified: pypy/dist/pypy/doc/dev_method.txt Log: fixing formatting and wrongly spelled names Modified: pypy/dist/pypy/doc/dev_method.txt ============================================================================== --- pypy/dist/pypy/doc/dev_method.txt (original) +++ pypy/dist/pypy/doc/dev_method.txt Tue Aug 22 16:12:06 2006 @@ -250,26 +250,26 @@ LovainLaNeuve June 2003 Amsterdam Dec 2003 Europython - /Gothenburg June 2004 - Vilnius November 2004 - Leysin January 2005 + /Gothenburg June 2004 + Vilnius Nov 2004 + Leysin Jan 2005 PyCon - /Washington March 2005 + /Washington March 2005 Europython - /Gothenburg June 2005 - Hildesheim July 2005 - Heidelberg August 2005 - Paris October 2005 - Gothenburg December 2005 - Mallorca January 2006 - PyCon/Dallas February 2006 - LouvainLaNeuve March 2006 - Leysin April 2006 - Tokyo April 2006 - D?sseldorf June 2006 + /Gothenburg June 2005 + Hildesheim July 2005 + Heidelberg Aug 2005 + Paris Oct 2005 + Gothenburg Dec 2005 + Mallorca Jan 2006 + PyCon/Dallas Feb 2006 + LouvainLaNeuve March 2006 + Leysin April 2006 + Tokyo April 2006 + D?sseldorf June 2006 Europython/ - Geneva July 2006 - Limerick August 2006 + Geneva July 2006 + Limerick Aug 2006 People who have participated and contributed during our sprints and thus @@ -288,7 +288,7 @@ Ludovic Aubry Adrien DiMascio Nicholas Chauvat - Niklaus Landemann + Niklaus Haldimann Anders Lehmann Carl Friedrich Bolz Eric Van Riet Paap @@ -318,7 +318,6 @@ Lutz Paelike Michael Chermside Beatrice D??ring - Bert Freudenberg Boris Feigin Amaury Forgeot d'Arc Andrew Thompson @@ -330,7 +329,6 @@ Gene Oden Josh Gilbert Geroge Paci - Pat Maupin Martin Blais Stuart Williams Jiwon Seo @@ -339,6 +337,11 @@ Alexandre Fayolle Rapha?l Collet Gr?goire Dooms + Sanghyeon Seo + Yutaka Niibe + Yusei Tahara + George Toshida + Koichi Sasada Guido Wesdorp Maciej Fijalkowski Antonio Cuni From arigo at codespeak.net Tue Aug 22 16:32:35 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 16:32:35 +0200 (CEST) Subject: [pypy-svn] r31478 - in pypy/dist/pypy: jit/llabstractinterp rpython Message-ID: <20060822143235.4ECC610069@code0.codespeak.net> Author: arigo Date: Tue Aug 22 16:32:32 2006 New Revision: 31478 Modified: pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py pypy/dist/pypy/rpython/rgenop.py Log: (pedronis, arigo) Removed unused argument. Modified: pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py ============================================================================== --- pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py (original) +++ pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py Tue Aug 22 16:32:32 2006 @@ -597,8 +597,7 @@ def residual_direct_call(self, FUNCTYPE, name, target, args_a): T = FUNCTYPE.RESULT - gen_fn_const = rgenop.gencallableconst(self.newblock, - name, + gen_fn_const = rgenop.gencallableconst(name, target, rgenop.constTYPE(FUNCTYPE)) retvar = self.genop('direct_call', [gen_fn_const] + Modified: pypy/dist/pypy/rpython/rgenop.py ============================================================================== --- pypy/dist/pypy/rpython/rgenop.py (original) +++ pypy/dist/pypy/rpython/rgenop.py Tue Aug 22 16:32:32 2006 @@ -69,7 +69,7 @@ block.operations.append(op) return to_opaque_object(v) -def gencallableconst(blockcontainer, name, targetcontainer, gv_FUNCTYPE): +def gencallableconst(name, targetcontainer, gv_FUNCTYPE): # is name useful, is it runtime variable? target = from_opaque_object(targetcontainer.obj) FUNCTYPE = from_opaque_object(gv_FUNCTYPE).value @@ -297,6 +297,7 @@ setannotation(initblock, None) setannotation(geninputarg, s_ConstOrVar) setannotation(genop, s_ConstOrVar) +setannotation(gencallableconst, s_ConstOrVar) setannotation(genconst, s_ConstOrVar) setannotation(revealconst, lambda s_T, s_gv: annmodel.lltype_to_annotation( s_T.const)) From arigo at codespeak.net Tue Aug 22 16:33:34 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 16:33:34 +0200 (CEST) Subject: [pypy-svn] r31479 - pypy/dist/pypy/translator/c/test Message-ID: <20060822143334.9549510069@code0.codespeak.net> Author: arigo Date: Tue Aug 22 16:33:32 2006 New Revision: 31479 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: (pedronis, arigo) Refactored test a bit. Well, we're not using it so far. We should really do so and kill this func_default hack. Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Tue Aug 22 16:33:32 2006 @@ -15,17 +15,19 @@ class CompilationTestCase: - def annotatefunc(self, func): + def annotatefunc(self, func, argtypes=None): t = TranslationContext(simplifying=True) - # builds starting-types from func_defs - argstypelist = [] - if func.func_defaults: - for spec in func.func_defaults: - if isinstance(spec, tuple): - spec = spec[0] # use the first type only for the tests - argstypelist.append(spec) + if argtypes is None: + # builds starting-types from func_defs + # XXX kill kill kill! + argtypes = [] + if func.func_defaults: + for spec in func.func_defaults: + if isinstance(spec, tuple): + spec = spec[0] # use the first type only for the tests + argtypes.append(spec) a = t.buildannotator() - a.build_types(func, argstypelist) + a.build_types(func, argtypes) a.simplify() return t @@ -37,9 +39,9 @@ builder.import_module() return builder.get_entry_point() - def getcompiled(self, func, view=False): + def getcompiled(self, func, argtypes=None, view=False): from pypy.translator.transform import insert_ll_stackcheck - t = self.annotatefunc(func) + t = self.annotatefunc(func, argtypes) self.process(t) if view or conftest.option.view: t.view() From arigo at codespeak.net Tue Aug 22 16:34:20 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 16:34:20 +0200 (CEST) Subject: [pypy-svn] r31480 - pypy/dist/pypy/translator/tool/pygame Message-ID: <20060822143420.BF46310076@code0.codespeak.net> Author: arigo Date: Tue Aug 22 16:34:17 2006 New Revision: 31480 Modified: pypy/dist/pypy/translator/tool/pygame/graphdisplay.py Log: (pedronis, arigo) Fix a pygame viewer bug with mixed strings-with-non-ascii characters and unicodes. Modified: pypy/dist/pypy/translator/tool/pygame/graphdisplay.py ============================================================================== --- pypy/dist/pypy/translator/tool/pygame/graphdisplay.py (original) +++ pypy/dist/pypy/translator/tool/pygame/graphdisplay.py Tue Aug 22 16:34:17 2006 @@ -287,7 +287,7 @@ if e.key == K_ESCAPE: return None elif e.key == K_RETURN: - return text + return text.encode('latin-1') # XXX do better elif e.key == K_BACKSPACE: text = text[:-1] elif e.unicode and ord(e.unicode) >= ord(' '): From arigo at codespeak.net Tue Aug 22 16:36:38 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 16:36:38 +0200 (CEST) Subject: [pypy-svn] r31481 - in pypy/dist/pypy: annotation rpython rpython/test Message-ID: <20060822143638.309B010075@code0.codespeak.net> Author: arigo Date: Tue Aug 22 16:36:33 2006 New Revision: 31481 Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/rpython/extregistry.py pypy/dist/pypy/rpython/test/test_extregistry.py Log: (pedronis, arigo) An extension to extregistry: you can pass a policy object to the various lookup functions, and register entries with conditions based on the policy. See test. Purpose: special-casing rgenop when compiling to C on a specific processor; but this could also be used e.g. to select between lltypesystem and ootypesystem versions of built-in functions. Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Tue Aug 22 16:36:33 2006 @@ -54,16 +54,15 @@ self.reflowcounter = {} self.return_bindings = {} # map return Variables to their graphs # --- end of debugging information --- - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper self.frozen = False - # user-supplied annotation logic for functions we don't want to flow into if policy is None: from pypy.annotation.policy import AnnotatorPolicy self.policy = AnnotatorPolicy() else: self.policy = policy + if bookkeeper is None: + bookkeeper = Bookkeeper(self) + self.bookkeeper = bookkeeper def __getstate__(self): attrs = """translator pendingblocks bindings annotated links_followed Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Tue Aug 22 16:36:33 2006 @@ -156,6 +156,7 @@ def __init__(self, annotator): self.annotator = annotator + self.policy = annotator.policy self.descs = {} # map Python objects to their XxxDesc wrappers self.methoddescs = {} # map (funcdesc, classdef) to the MethodDesc self.classdefs = [] # list of all ClassDefs @@ -363,8 +364,9 @@ elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) - elif extregistry.is_registered(x): - result = extregistry.lookup(x).compute_annotation() + elif extregistry.is_registered(x, self.policy): + entry = extregistry.lookup(x, self.policy) + result = entry.compute_annotation_bk(self) ## elif hasattr(x, "compute_result_annotation"): ## result = SomeBuiltin(x.compute_result_annotation, methodname=x.__name__) ## elif hasattr(tp, "compute_annotation"): @@ -473,8 +475,9 @@ elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) - elif extregistry.is_registered(x): - result = extregistry.lookup(x).compute_annotation() + elif extregistry.is_registered(x, self.policy): + entry = extregistry.lookup(x, self.policy) + result = entry.compute_annotation_bk(self) ## elif hasattr(x, "compute_result_annotation"): ## result = SomeBuiltin(x.compute_result_annotation, methodname=x.__name__) ## elif hasattr(tp, "compute_annotation"): @@ -638,8 +641,9 @@ return SomeExternalObject(t) ## elif hasattr(t, "compute_annotation"): ## return t.compute_annotation() - elif extregistry.is_registered_type(t): - return extregistry.lookup_type(t).compute_annotation() + elif extregistry.is_registered_type(t, self.policy): + entry = extregistry.lookup_type(t, self.policy) + return entry.compute_annotation_bk(self) elif t.__module__ != '__builtin__' and t not in self.pbctypes: classdef = self.getuniqueclassdef(t) return SomeInstance(classdef) Modified: pypy/dist/pypy/rpython/extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extregistry.py (original) +++ pypy/dist/pypy/rpython/extregistry.py Tue Aug 22 16:36:33 2006 @@ -18,29 +18,45 @@ selfcls._register_metatype(dict['_metatype_']) del selfcls._metatype_ - def _register_value(selfcls, key): + def _register(selfcls, dict, key): if isinstance(key, tuple): for k in key: - selfcls._register_value(k) + selfcls._register(dict, k) else: - assert key not in EXT_REGISTRY_BY_VALUE - EXT_REGISTRY_BY_VALUE[key] = selfcls + for basecls in selfcls.__mro__: + if '_condition_' in basecls.__dict__: + cond = basecls.__dict__['_condition_'] + break + else: + cond = None + try: + family = dict[key] + except KeyError: + family = dict[key] = ClassFamily() + family.add(selfcls, cond) + + def _register_value(selfcls, key): + selfcls._register(EXT_REGISTRY_BY_VALUE, key) def _register_type(selfcls, key): - if isinstance(key, tuple): - for k in key: - selfcls._register_type(k) - else: - assert key not in EXT_REGISTRY_BY_TYPE - EXT_REGISTRY_BY_TYPE[key] = selfcls + selfcls._register(EXT_REGISTRY_BY_TYPE, key) def _register_metatype(selfcls, key): - if isinstance(key, tuple): - for k in key: - selfcls._register_metatype(k) + selfcls._register(EXT_REGISTRY_BY_METATYPE, key) + +class ClassFamily(object): + + def __init__(self): + self.default = None + self.conditionals = [] + + def add(self, cls, cond=None): + if cond is None: + assert self.default is None, ( + "duplicate extregistry entry %r" % (cls,)) + self.default = cls else: - assert key not in EXT_REGISTRY_BY_METATYPE - EXT_REGISTRY_BY_METATYPE[key] = selfcls + self.conditionals.append((cls, cond)) class ExtRegistryEntry(object): @@ -64,6 +80,9 @@ def __hash__(self): return hash((self.__class__, self.type, Hashable(self.instance))) + def compute_annotation_bk(self, bk): + return self.compute_annotation() + def compute_annotation(self): # default implementation useful for built-in functions, # can be overriden @@ -127,36 +146,49 @@ # ____________________________________________________________ # Public interface to access the registry -def _lookup_type_cls(tp): +def _lookup_from(dict, key, config): + family = dict[key] + if config is not None: + matches = [cls for cls, cond in family.conditionals + if cond(config)] + if matches: + assert len(matches) == 1, "multiple extregistry matches: %r" % ( + matches,) + return matches[0] + if family.default: + return family.default + raise KeyError(key) + +def _lookup_type_cls(tp, config): try: - return EXT_REGISTRY_BY_TYPE[tp] + return _lookup_from(EXT_REGISTRY_BY_TYPE, tp, config) except (KeyError, TypeError): - return EXT_REGISTRY_BY_METATYPE[type(tp)] + return _lookup_from(EXT_REGISTRY_BY_METATYPE, type(tp), config) -def lookup_type(tp): - Entry = _lookup_type_cls(tp) +def lookup_type(tp, config=None): + Entry = _lookup_type_cls(tp, config) return Entry(tp) -def is_registered_type(tp): +def is_registered_type(tp, config=None): try: - _lookup_type_cls(tp) + _lookup_type_cls(tp, config) except KeyError: return False return True -def _lookup_cls(instance): +def _lookup_cls(instance, config): try: - return EXT_REGISTRY_BY_VALUE[instance] + return _lookup_from(EXT_REGISTRY_BY_VALUE, instance, config) except (KeyError, TypeError): - return _lookup_type_cls(type(instance)) + return _lookup_type_cls(type(instance), config) -def lookup(instance): - Entry = _lookup_cls(instance) +def lookup(instance, config=None): + Entry = _lookup_cls(instance, config) return Entry(type(instance), instance) -def is_registered(instance): +def is_registered(instance, config=None): try: - _lookup_cls(instance) + _lookup_cls(instance, config) except KeyError: return False return True Modified: pypy/dist/pypy/rpython/test/test_extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_extregistry.py (original) +++ pypy/dist/pypy/rpython/test/test_extregistry.py Tue Aug 22 16:36:33 2006 @@ -177,3 +177,26 @@ _about_ = n1 assert isinstance(extregistry.lookup(n1), Entry) assert isinstance(extregistry.lookup(n2), Entry) + +def test_condition(): + stuff = object() + class Entry(ExtRegistryEntry): + _about_ = stuff + _condition_ = lambda n: n == 'yes' + assert isinstance(extregistry.lookup(stuff, 'yes'), Entry) + py.test.raises(KeyError, "extregistry.lookup(stuff, 'no')") + py.test.raises(KeyError, "extregistry.lookup(stuff)") + + class Entry2(ExtRegistryEntry): + _about_ = stuff + assert isinstance(extregistry.lookup(stuff, 'yes'), Entry) + assert isinstance(extregistry.lookup(stuff, 'no'), Entry2) + assert isinstance(extregistry.lookup(stuff), Entry2) + + otherstuff = object() + class Entry3(Entry): + _about_ = otherstuff + # _condition_ is inherited from Entry + assert isinstance(extregistry.lookup(otherstuff, 'yes'), Entry3) + py.test.raises(KeyError, "extregistry.lookup(otherstuff, 'no')") + py.test.raises(KeyError, "extregistry.lookup(otherstuff)") From arigo at codespeak.net Tue Aug 22 16:55:44 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 16:55:44 +0200 (CEST) Subject: [pypy-svn] r31486 - pypy/dist/pypy/rpython Message-ID: <20060822145544.79FE61005A@code0.codespeak.net> Author: arigo Date: Tue Aug 22 16:55:41 2006 New Revision: 31486 Modified: pypy/dist/pypy/rpython/extregistry.py Log: (pedronis, arigo) Minor {clean,speed}-up. Modified: pypy/dist/pypy/rpython/extregistry.py ============================================================================== --- pypy/dist/pypy/rpython/extregistry.py (original) +++ pypy/dist/pypy/rpython/extregistry.py Tue Aug 22 16:55:41 2006 @@ -58,6 +58,18 @@ else: self.conditionals.append((cls, cond)) + def match(self, config): + if config is not None: + matches = [cls for cls, cond in self.conditionals + if cond(config)] + if matches: + assert len(matches) == 1, ( + "multiple extregistry matches: %r" % (matches,)) + return matches[0] + if self.default: + return self.default + raise KeyError("no default extregistry entry") + class ExtRegistryEntry(object): __metaclass__ = AutoRegisteringType @@ -146,24 +158,11 @@ # ____________________________________________________________ # Public interface to access the registry -def _lookup_from(dict, key, config): - family = dict[key] - if config is not None: - matches = [cls for cls, cond in family.conditionals - if cond(config)] - if matches: - assert len(matches) == 1, "multiple extregistry matches: %r" % ( - matches,) - return matches[0] - if family.default: - return family.default - raise KeyError(key) - def _lookup_type_cls(tp, config): try: - return _lookup_from(EXT_REGISTRY_BY_TYPE, tp, config) + return EXT_REGISTRY_BY_TYPE[tp].match(config) except (KeyError, TypeError): - return _lookup_from(EXT_REGISTRY_BY_METATYPE, type(tp), config) + return EXT_REGISTRY_BY_METATYPE[type(tp)].match(config) def lookup_type(tp, config=None): Entry = _lookup_type_cls(tp, config) @@ -178,7 +177,7 @@ def _lookup_cls(instance, config): try: - return _lookup_from(EXT_REGISTRY_BY_VALUE, instance, config) + return EXT_REGISTRY_BY_VALUE[instance].match(config) except (KeyError, TypeError): return _lookup_type_cls(type(instance), config) From mwh at codespeak.net Tue Aug 22 17:08:37 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 22 Aug 2006 17:08:37 +0200 (CEST) Subject: [pypy-svn] r31487 - in pypy/dist/pypy/interpreter: . test Message-ID: <20060822150837.65EAB10050@code0.codespeak.net> Author: mwh Date: Tue Aug 22 17:08:18 2006 New Revision: 31487 Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/test/test_raise.py Log: (ac, mwh) fix abstract_getclass to cope with old-style classes -- see test. this broke "./pypy-c-oldstyle translate.py"... Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Tue Aug 22 17:08:18 2006 @@ -638,7 +638,12 @@ return self.w_True def abstract_getclass(self, w_obj): - return self.getattr(w_obj, self.wrap('__class__')) + try: + return self.getattr(w_obj, self.wrap('__class__')) + except OperationError, e: + if e.match(self, self.w_TypeError) or e.match(self, self.w_AttributeError): + return self.type(w_obj) + raise def eval(self, expression, w_globals, w_locals): "NOT_RPYTHON: For internal debugging." Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Tue Aug 22 17:08:18 2006 @@ -303,14 +303,6 @@ return space.wrap(Method(space, w_function, None, w_cls)) -def _getclass(space, w_obj): - try: - return space.abstract_getclass(w_obj) - except OperationError, e: - if e.match(space, space.w_AttributeError): - return space.type(w_obj) - raise - class Method(Wrappable): """A method is a function bound to a specific instance or class.""" @@ -353,7 +345,7 @@ if w_firstarg is None: instdescr = "nothing" else: - instname = _getclass(space, w_firstarg).getname(space,"") + instname = self.abstract_getclass(space, w_firstarg).getname(space,"") if instname: instname += " " instdescr = "%sinstance" %instname Modified: pypy/dist/pypy/interpreter/test/test_raise.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_raise.py (original) +++ pypy/dist/pypy/interpreter/test/test_raise.py Tue Aug 22 17:08:18 2006 @@ -123,3 +123,11 @@ except B, b: assert b.__class__ == B assert b.x == 42 + + def test_it(self): + C = _classobj('C', (), {}) + # this used to explode in the exception normalization step: + try: + {}[C] + except KeyError: + pass From arigo at codespeak.net Tue Aug 22 18:41:14 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 18:41:14 +0200 (CEST) Subject: [pypy-svn] r31494 - in pypy/dist/pypy/rpython: lltypesystem rctypes rctypes/test Message-ID: <20060822164114.80A9F10036@code0.codespeak.net> Author: arigo Date: Tue Aug 22 18:41:10 2006 New Revision: 31494 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/rctypes/avoid_p.py pypy/dist/pypy/rpython/rctypes/rvoid_p.py pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py Log: (pedronis, arigo) Support for c_void_p().value to cast a pointer or address to an int. Started support for c_void_p(int), but it's a mess. Delayed until we really need it. Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Tue Aug 22 18:41:10 2006 @@ -340,6 +340,7 @@ 'cast_weakadr_to_ptr': LLOp(canfold=True), 'cast_weakadr_to_int': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), + #'cast_int_to_adr': LLOp(canfold=True), # __________ GC operations __________ Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Tue Aug 22 18:41:10 2006 @@ -279,6 +279,10 @@ checkadr(adr) return llmemory.cast_adr_to_int(adr) +##def op_cast_int_to_adr(x): +## assert type(x) is int +## return llmemory.cast_int_to_adr(x) + def op_unichar_eq(x, y): assert isinstance(x, unicode) and len(x) == 1 Modified: pypy/dist/pypy/rpython/rctypes/avoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/avoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/avoid_p.py Tue Aug 22 18:41:10 2006 @@ -1,6 +1,6 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rctypes.implementation import CTypesCallEntry, CTypesObjEntry -from pypy.annotation.model import SomeCTypesObject +from pypy.annotation.model import SomeInteger, SomeCTypesObject from ctypes import c_void_p, c_int, POINTER, cast, c_char, c_char_p from pypy.rpython.rctypes.astringbuf import StringBufferType @@ -17,6 +17,13 @@ r_void_p = hop.r_result hop.exception_cannot_occur() v_result = r_void_p.allocate_instance(hop.llops) + if hop.args_r: + raise NotImplementedError("cast_int_to_adr") +## from pypy.rpython.lltypesystem import lltype, llmemory +## [v_intadr] = hop.inputargs(lltype.Signed) # xxx id-sized +## v_adr = hop.genop('cast_int_to_adr', [v_intadr], +## resulttype = llmemory.Address) +## r_void_p.setvalue(hop.llops, v_result, v_adr) return v_result @@ -24,6 +31,10 @@ "Annotation and rtyping of c_void_p instances." _type_ = c_void_p + def get_field_annotation(self, s_void_p, fieldname): + assert fieldname == "value" + return SomeInteger() # xxx id-sized + def get_repr(self, rtyper, s_void_p): from pypy.rpython.rctypes.rvoid_p import CVoidPRepr from pypy.rpython.lltypesystem import llmemory Modified: pypy/dist/pypy/rpython/rctypes/rvoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/rvoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/rvoid_p.py Tue Aug 22 18:41:10 2006 @@ -14,6 +14,16 @@ return super(CVoidPRepr, self).convert_const(value) raise NotImplementedError("XXX constant pointer passed to void* arg") + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + assert s_attr.is_constant() + assert s_attr.const == 'value' + v_box = hop.inputarg(self, 0) + v_c_adr = self.getvalue(hop.llops, v_box) + hop.exception_cannot_occur() + return hop.genop('cast_adr_to_int', [v_c_adr], + resulttype = lltype.Signed) + class __extend__(pairtype(CCharPRepr, CVoidPRepr), pairtype(PointerRepr, CVoidPRepr)): Modified: pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py ============================================================================== --- pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py (original) +++ pypy/dist/pypy/rpython/rctypes/test/test_rvoid_p.py Tue Aug 22 18:41:10 2006 @@ -5,6 +5,7 @@ import py.test import pypy.rpython.rctypes.implementation from pypy.annotation.annrpython import RPythonAnnotator +from pypy.annotation import model as annmodel from pypy.translator.translator import TranslationContext from pypy.translator.c.test.test_genc import compile from pypy import conftest @@ -32,6 +33,26 @@ if conftest.option.view: t.view() + def test_annotate_addr2int(self): + def fn(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + return p1.value + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_annotate_int2addr(self): + def fn(): + return c_void_p(123) + + t = TranslationContext() + a = t.buildannotator() + s = a.build_types(fn, []) + assert s.knowntype == c_void_p + class Test_specialization: def test_specialize_c_void_p(self): def func(): @@ -82,6 +103,20 @@ interpret(func, [65]) + def test_specialize_addr2int(self): + def fn(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + return p1.value + res = interpret(fn, []) + assert lltype.typeOf(res) == lltype.Signed # xxx + +## def test_annotate_int2addr(self): XXX cast_int_to_adr() not implemented +## def fn(n): +## return c_void_p(n) +## res = interpret(fn, [123]) +## assert llmemory.cast_adr_to_int(res.c_data[0]) == 123 + class Test_compilation: def test_compile_c_char_p(self): def func(): From arigo at codespeak.net Tue Aug 22 18:42:36 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 18:42:36 +0200 (CEST) Subject: [pypy-svn] r31495 - in pypy/dist/pypy: jit/timeshifter rpython Message-ID: <20060822164236.3999510036@code0.codespeak.net> Author: arigo Date: Tue Aug 22 18:42:29 2006 New Revision: 31495 Modified: pypy/dist/pypy/jit/timeshifter/rcontainer.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/rpython/rgenop.py Log: (pedronis, arigo) Kill VARLIST. No reason at all for it, plain lists work. Modified: pypy/dist/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/dist/pypy/jit/timeshifter/rcontainer.py Tue Aug 22 18:42:29 2006 @@ -143,14 +143,12 @@ gv_sub = genvar genop = builder.genop for i in range(len(self.accessptrtype_gv)-1): - op_args = lltype.malloc(rgenop.VARLIST.TO, 2) - op_args[0] = gv_sub - op_args[1] = self.fieldname_gv[i] + op_args = [gv_sub, + self.fieldname_gv[i]] gv_sub = genop('getsubstruct', op_args, self.accessptrtype_gv[i+1]) - op_args = lltype.malloc(rgenop.VARLIST.TO, 3) - op_args[0] = gv_sub - op_args[1] = self.fieldname_gv[-1] - op_args[2] = box.getgenvar(builder) + op_args = [gv_sub, + self.fieldname_gv[-1], + box.getgenvar(builder)] genop('setfield', op_args, rgenop.gv_Void) # ____________________________________________________________ @@ -210,14 +208,12 @@ typedesc = self.typedesc boxes = self.content_boxes self.content_boxes = None - op_args = lltype.malloc(rgenop.VARLIST.TO, 1) - op_args[0] = typedesc.gv_type + op_args = [typedesc.gv_type] genvar = genop('malloc', op_args, typedesc.gv_ptrtype) # force all the boxes pointing to this VirtualStruct for box in self.substruct_boxes: # XXX using getsubstruct would be nicer - op_args = lltype.malloc(rgenop.VARLIST.TO, 1) - op_args[0] = genvar + op_args = [genvar] box.genvar = genop('cast_pointer', op_args, box.gv_type) box.content = None self.substruct_boxes = None Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Tue Aug 22 18:42:29 2006 @@ -61,8 +61,7 @@ arg = rvalue.ll_getvalue(argbox, ARG0) res = opdesc.llop(RESULT, arg) return rvalue.ll_fromvalue(res) - op_args = lltype.malloc(rgenop.VARLIST.TO, 1) - op_args[0] = argbox.getgenvar(jitstate.curbuilder) + op_args = [argbox.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop(opdesc.opname, op_args, opdesc.gv_RESULT) return opdesc.redboxcls(opdesc.gv_RESULT, genvar) @@ -78,9 +77,8 @@ arg1 = rvalue.ll_getvalue(argbox1, ARG1) res = opdesc.llop(RESULT, arg0, arg1) return rvalue.ll_fromvalue(res) - op_args = lltype.malloc(rgenop.VARLIST.TO, 2) - op_args[0] = argbox0.getgenvar(jitstate.curbuilder) - op_args[1] = argbox1.getgenvar(jitstate.curbuilder) + op_args = [argbox0.getgenvar(jitstate.curbuilder), + argbox1.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop(opdesc.opname, op_args, opdesc.gv_RESULT) return opdesc.redboxcls(opdesc.gv_RESULT, genvar) @@ -92,9 +90,8 @@ return rvalue.ll_fromvalue(res) assert isinstance(argbox, rvalue.PtrRedBox) if argbox.content is None: - op_args = lltype.malloc(rgenop.VARLIST.TO, 2) - op_args[0] = argbox.getgenvar(jitstate.curbuilder) - op_args[1] = fielddesc.fieldname_gv[-1] + op_args = [argbox.getgenvar(jitstate.curbuilder), + fielddesc.fieldname_gv[-1]] genvar = jitstate.curbuilder.genop('getfield', op_args, fielddesc.gv_resulttype) return fielddesc.redboxcls(fielddesc.gv_resulttype, genvar) @@ -104,10 +101,9 @@ def ll_generate_setfield(jitstate, fielddesc, destbox, valuebox): assert isinstance(destbox, rvalue.PtrRedBox) if destbox.content is None: - op_args = lltype.malloc(rgenop.VARLIST.TO, 3) - op_args[0] = destbox.getgenvar(jitstate.curbuilder) - op_args[1] = fielddesc.fieldname_gv[-1] - op_args[2] = valuebox.getgenvar(jitstate.curbuilder) + op_args = [destbox.getgenvar(jitstate.curbuilder), + fielddesc.fieldname_gv[-1], + valuebox.getgenvar(jitstate.curbuilder)] jitstate.curbuilder.genop('setfield', op_args, rgenop.gv_Void) else: @@ -120,9 +116,8 @@ return rvalue.ll_fromvalue(res) assert isinstance(argbox, rvalue.PtrRedBox) if argbox.content is None: - op_args = lltype.malloc(rgenop.VARLIST.TO, 2) - op_args[0] = argbox.getgenvar(jitstate.curbuilder) - op_args[1] = fielddesc.gv_fieldname + op_args = [argbox.getgenvar(jitstate.curbuilder), + fielddesc.gv_fieldname] genvar = jitstate.curbuilder.genop('getsubstruct', op_args, fielddesc.gv_resulttype) return fielddesc.redboxcls(fielddesc.gv_resulttype, genvar) @@ -135,9 +130,8 @@ array = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) res = array[rvalue.ll_getvalue(indexbox, lltype.Signed)] return rvalue.ll_fromvalue(res) - op_args = lltype.malloc(rgenop.VARLIST.TO, 2) - op_args[0] = argbox.getgenvar(jitstate.curbuilder) - op_args[1] = indexbox.getgenvar(jitstate.curbuilder) + op_args = [argbox.getgenvar(jitstate.curbuilder), + indexbox.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop('getarrayitem', op_args, fielddesc.gv_resulttype) return fielddesc.redboxcls(fielddesc.gv_resulttype, genvar) @@ -254,8 +248,6 @@ jitstate.split_queue.append((exitindex, later_builder, redboxcopies)) return True -novars = lltype.malloc(rgenop.VARLIST.TO, 0) - def dispatch_next(jitstate, outredboxes): split_queue = jitstate.split_queue if split_queue: Modified: pypy/dist/pypy/rpython/rgenop.py ============================================================================== --- pypy/dist/pypy/rpython/rgenop.py (original) +++ pypy/dist/pypy/rpython/rgenop.py Tue Aug 22 18:42:29 2006 @@ -52,17 +52,18 @@ res.append(v) return res -# is opname a runtime value? -def genop(blockcontainer, opname, vars, resulttype): +def genop(blockcontainer, opname, vars_gv, gv_RESULT_TYPE): + # 'opname' is a constant string + # gv_RESULT_TYPE comes from constTYPE if not isinstance(opname, str): opname = LLSupport.from_rstr(opname) block = from_opaque_object(blockcontainer.obj) assert block.exits == [], "block already closed" - if isinstance(resulttype, lltype.LowLevelType): - RESULT_TYPE = resulttype + if isinstance(gv_RESULT_TYPE, lltype.LowLevelType): + RESULT_TYPE = gv_RESULT_TYPE else: - RESULT_TYPE = from_opaque_object(resulttype).value - opvars = _inputvars(vars) + RESULT_TYPE = from_opaque_object(gv_RESULT_TYPE).value + opvars = _inputvars(vars_gv) v = flowmodel.Variable() v.concretetype = RESULT_TYPE op = flowmodel.SpaceOperation(opname, opvars, v) @@ -245,19 +246,6 @@ nulllink = lltype.nullptr(LINK.TO) gv_Void = constTYPE(lltype.Void) -# VARLIST -def ll_fixed_items(l): - return l - -def ll_fixed_length(l): - return len(l) - -VARLIST = lltype.Ptr(lltype.GcArray(CONSTORVAR, - adtmeths = { - "ll_items": ll_fixed_items, - "ll_length": ll_fixed_length - })) - # helpers def setannotation(func, annotation, specialize_as_constant=False): From arigo at codespeak.net Tue Aug 22 18:48:31 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 18:48:31 +0200 (CEST) Subject: [pypy-svn] r31496 - in pypy/dist/pypy/jit/codegen/i386: . test Message-ID: <20060822164831.8BB2E1005A@code0.codespeak.net> Author: arigo Date: Tue Aug 22 18:48:28 2006 New Revision: 31496 Modified: pypy/dist/pypy/jit/codegen/i386/ri386.py pypy/dist/pypy/jit/codegen/i386/ri386setup.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386.py Log: (pedronis, arigo) Reintroduced some support for REL32, used in jumps and calls. Modified: pypy/dist/pypy/jit/codegen/i386/ri386.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386.py Tue Aug 22 18:48:28 2006 @@ -46,6 +46,10 @@ class MODRM8(MODRM): pass +class REL32(OPERAND): + def __init__(self, absolute_target): + self.absolute_target = absolute_target + class MISSING(OPERAND): pass @@ -78,6 +82,7 @@ imm32 = IMM32 imm8 = IMM8 imm16 = IMM16 +rel32 = REL32 def memregister(register): assert register.width == 4 @@ -159,5 +164,8 @@ def write(self, data): raise NotImplementedError + def tell(self): + raise NotImplementedError + import ri386setup # side-effect: add methods to AbstractCodeBuilder Modified: pypy/dist/pypy/jit/codegen/i386/ri386setup.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386setup.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386setup.py Tue Aug 22 18:48:28 2006 @@ -40,6 +40,8 @@ IMM16: [(IMM16, None)], # only for RET IMM8: [(IMM8, None), (IMM32, None)], + REL32: [(REL32, None)], + MODRM: [(MODRM, None)], MODRM8: [(MODRM8, None)], @@ -100,6 +102,15 @@ lines.append('builder.write(%s(arg%d.value))' % (packer, self.op)) return False +class relative(operand): + def eval(self, lines, has_orbyte): + assert not has_orbyte, "malformed bytecode" + assert self.width == 'i', "only REL32 supported at the moment" + lines.append('offset = arg%d.absolute_target - (builder.tell()+4)' % ( + self.op,)) + lines.append('builder.write(packimm32(offset))') + return False + def consolidate(code1): for i in range(len(code1)-1, 0, -1): @@ -277,16 +288,16 @@ RET.mode0(['\xC3']) RET.mode1(IMM16, ['\xC2', immediate(1,'h')]) -#CALL = Instruction() -#CALL.mode1(REL32, ['\xE8', immediate(1)]) -#CALL.mode1(MODRM, ['\xFF', orbyte(2<<3), modrm(1)]) -#CALL.indirect = 1 +CALL = Instruction() +CALL.mode1(REL32, ['\xE8', relative(1)]) +CALL.mode1(MODRM, ['\xFF', orbyte(2<<3), modrm(1)]) +CALL.indirect = 1 -#JMP = Instruction() +JMP = Instruction() #JMP.mode1(REL8, ['\xEB', immediate(1,'b')]) -#JMP.mode1(REL32, ['\xE9', immediate(1)]) -#JMP.mode1(MODRM, ['\xFF', orbyte(4<<3), modrm(1)]) -#JMP.indirect = 1 +JMP.mode1(REL32, ['\xE9', relative(1)]) +JMP.mode1(MODRM, ['\xFF', orbyte(4<<3), modrm(1)]) +JMP.indirect = 1 PUSH = Instruction() PUSH.mode1(IMM8, ['\x6A', immediate(1,'b')]) @@ -396,7 +407,7 @@ instr.indirect = indirect #define_cond('J', 1, (REL8,), [None,'\x70', immediate(1,'b')]) -#define_cond('J', 1, (REL32,), ['\x0F', None,'\x80', immediate(1)]) +define_cond('J', 1, (REL32,), ['\x0F', None,'\x80', relative(1)]) define_cond('SET', 0, (MODRM8,), ['\x0F', None,'\x90',orbyte(0<<3),modrm(1,'b')]) define_cond('CMOV',0,(REG,MODRM),['\x0F', None,'\x40', register(1,8), modrm(2)]) # note: CMOVxx are Pentium-class instructions, unknown to the 386 and 486 Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386.py Tue Aug 22 18:48:28 2006 @@ -9,6 +9,9 @@ for c in data: self.buffer.append(c) # extend the list of characters + def tell(self): + return len(self.buffer) + def getvalue(self): return ''.join(self.buffer) @@ -47,6 +50,8 @@ # mov eax, [8*ecx] yield check, '\x89\x04\xcd\x00\x00\x00\x00', \ 'MOV', memSIB(None,ecx,3,0), eax + # call +17 + yield check, '\xE8\x11\x00\x00\x00', 'CALL', rel32(22) def test_translate(): From arigo at codespeak.net Tue Aug 22 19:09:59 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 19:09:59 +0200 (CEST) Subject: [pypy-svn] r31500 - in pypy/dist/pypy/jit/codegen/i386: . test Message-ID: <20060822170959.A918D10069@code0.codespeak.net> Author: arigo Date: Tue Aug 22 19:09:55 2006 New Revision: 31500 Added: pypy/dist/pypy/jit/codegen/i386/ri386genop.py (contents, props changed) pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/i386/codebuf.py pypy/dist/pypy/jit/codegen/i386/ri386setup.py pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py Log: (pedronis, arigo) First real progress towards a back-end for the JIT. Yay! Modified: pypy/dist/pypy/jit/codegen/i386/codebuf.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/codebuf.py (original) +++ pypy/dist/pypy/jit/codegen/i386/codebuf.py Tue Aug 22 19:09:55 2006 @@ -1,7 +1,7 @@ import mmap from pypy.module.mmap import interp_mmap from ctypes import * -from ri386 import * +from ri386 import AbstractCodeBuilder libcmmap = interp_mmap.libc.mmap libcmunmap = interp_mmap.libc.munmap @@ -19,7 +19,7 @@ def __init__(self, map_size): flags = mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS prot = mmap.PROT_EXEC | mmap.PROT_READ | mmap.PROT_WRITE - res = libcmmap(c_void_p(0), map_size, prot, flags, -1, 0) + res = libcmmap(c_void_p(), map_size, prot, flags, -1, 0) if not res: raise MemoryError self._data = cast(res, POINTER(c_char * map_size)) @@ -38,6 +38,10 @@ p += 1 self._pos = p + def tell(self): + baseaddr = cast(self._data, c_void_p).value + return baseaddr + self._pos + def execute(self, arg1, arg2): fnptr = cast(self._data, binaryfn) return fnptr(arg1, arg2) Added: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Tue Aug 22 19:09:55 2006 @@ -0,0 +1,123 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.jit.codegen.i386.codebuf import MachineCodeBlock +from pypy.jit.codegen.i386.ri386 import * + + +WORD = 4 + + +class VarOrConst(object): + pass + + +class Var(VarOrConst): + + def __init__(self, stackpos): + # 'stackpos' is an index relative to the pushed arguments: + # 0 = 1st arg, + # 1 = 2nd arg, + # ... + # return address, + # local var, ... + # ... <--- esp+4 + # local var <--- esp + # + self.stackpos = stackpos + + def operand(self, block): + return mem(esp, WORD * (block.stackdepth-1 - self.stackpos)) + + +class TypeConst(VarOrConst): + + def __init__(self, kind): + self.kind = kind + + +class IntConst(VarOrConst): + + def __init__(self, value): + self.value = value + + def operand(self, block): + if single_byte(self.value): + return IMM8(self.value) + else: + return IMM32(self.value) + + +class Block(object): + def __init__(self, mc): + self.argcount = 0 + self.stackdepth = 0 + self.mc = mc + self.startaddr = mc.tell() + + def geninputarg(self, gv_TYPE): + res = Var(self.argcount) + self.argcount += 1 + self.stackdepth += 1 + return res + + def push(self, reg): + self.mc.PUSH(reg) + res = Var(self.stackdepth) + self.stackdepth += 1 + return res + + def op_int_add(self, (gv_x, gv_y), gv_RESTYPE): + self.mc.MOV(eax, gv_x.operand(self)) + self.mc.ADD(eax, gv_y.operand(self)) + return self.push(eax) + + +class RI386GenOp(object): + gv_IntWord = TypeConst('IntWord') + gv_Void = TypeConst('Void') + + def __init__(self): + self.mc = MachineCodeBlock(65536) # XXX!!! + + def newblock(self): + # XXX concurrently-open Blocks cannot use the same mc + return Block(self.mc) + + def closeblock1(self, block): + return block + + def closereturnlink(self, link, gv_result): + link.mc.MOV(eax, gv_result.operand(link)) + link.mc.ADD(esp, IMM32(WORD * link.stackdepth)) + link.mc.RET() + + def geninputarg(self, block, gv_TYPE): + return block.geninputarg(gv_TYPE) + + def genconst(llvalue): + T = lltype.typeOf(llvalue) + assert T is lltype.Signed + return IntConst(llvalue) + genconst._annspecialcase_ = 'specialize:argtype(0)' # XXX arglltype(0)? + genconst = staticmethod(genconst) + + def constTYPE(T): + if T is lltype.Void: + return RI386GenOp.gv_Void + else: + return RI386GenOp.gv_IntWord # XXX for now + constTYPE._annspecialcase_ = 'specialize:memo' + constTYPE = staticmethod(constTYPE) + + def genop(self, block, opname, args_gv, gv_RESTYPE): + genmethod = getattr(block, 'op_' + opname) + return genmethod(args_gv, gv_RESTYPE) + genop._annspecialcase_ = 'specialize:arg(2)' + + def gencallableconst(self, name, block, gv_FUNCTYPE): + prologue = self.newblock() + #prologue.mc.BREAKPOINT() + operand = mem(esp, WORD * block.argcount) + for i in range(block.argcount): + prologue.mc.PUSH(operand) + prologue.mc.JMP(rel32(block.startaddr)) + return IntConst(prologue.startaddr) Modified: pypy/dist/pypy/jit/codegen/i386/ri386setup.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386setup.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386setup.py Tue Aug 22 19:09:55 2006 @@ -376,6 +376,12 @@ TEST.mode2(EAX, IMM32, ['\xA9', immediate(2)]) TEST.mode2(MODRM, IMM32, ['\xF7', orbyte(0<<3), modrm(1), immediate(2)]) +INT = Instruction() +INT.mode1(IMM8, ['\xCD', immediate(1, 'b')]) + +BREAKPOINT = Instruction() # INT 3 +BREAKPOINT.mode0(['\xCC']) + Conditions = { 'O': 0, Modified: pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_codebuf.py Tue Aug 22 19:09:55 2006 @@ -1,13 +1,14 @@ -from pypy.jit.codegen.i386.codebuf import * +from pypy.jit.codegen.i386.ri386 import * +from pypy.jit.codegen.i386.codebuf import MachineCodeBlock def test_machinecodeblock(): mc = MachineCodeBlock(4096) mc.MOV(eax, mem(esp, 4)) - mc.ADD(eax, mem(esp, 8)) + mc.SUB(eax, mem(esp, 8)) mc.RET() - res = mc.execute(40, 2) + res = mc.execute(44, 2) assert res == 42 return res Added: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Tue Aug 22 19:09:55 2006 @@ -0,0 +1,48 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.rpython import llinterp +from pypy.rpython.test.test_llinterp import interpret +from pypy.translator.c.test.test_genc import compile +from pypy.jit.codegen.i386.ri386genop import RI386GenOp + +from ctypes import c_void_p, cast, CFUNCTYPE, c_int + +# ____________________________________________________________ + +FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + +def make_adder(rgenop, n): + # 'return x+n' + gv_SIGNED = rgenop.constTYPE(lltype.Signed) + block = rgenop.newblock() + gv_x = rgenop.geninputarg(block, gv_SIGNED) + args_gv = [gv_x, + rgenop.genconst(n)] + gv_result = rgenop.genop(block, "int_add", args_gv, gv_SIGNED) + link = rgenop.closeblock1(block) + rgenop.closereturnlink(link, gv_result) + + gv_FUNC = rgenop.constTYPE(FUNC) + gv_add_one = rgenop.gencallableconst("adder", block, gv_FUNC) + return gv_add_one + +def runner(x, y): + rgenop = RI386GenOp() + gv_add_x = make_adder(rgenop, x) + add_x = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_x) + return add_x(y) + +# ____________________________________________________________ + +def test_adder_direct(): + rgenop = RI386GenOp() + gv_add_5 = make_adder(rgenop, 5) + print gv_add_5.value + fnptr = cast(c_void_p(gv_add_5.value), CFUNCTYPE(c_int, c_int)) + res = fnptr(37) # <== the segfault is here + assert res == 42 + +def test_adder_compile(): + import py; py.test.skip("in-progress") + fn = compile(runner, [int, int]) + res = fn(9080983, -9080941) + assert res == 42 From arigo at codespeak.net Tue Aug 22 19:26:16 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 19:26:16 +0200 (CEST) Subject: [pypy-svn] r31501 - in pypy/dist/pypy: annotation jit/codegen/i386 jit/codegen/i386/test Message-ID: <20060822172616.BBEEF10060@code0.codespeak.net> Author: arigo Date: Tue Aug 22 19:26:13 2006 New Revision: 31501 Modified: pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Log: (pedronis, arigo) Passing the first test that produces a C program doing run-time code generation. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Tue Aug 22 19:26:13 2006 @@ -334,7 +334,11 @@ return SomeString(can_be_None=str1.can_be_None or str2.can_be_None) def add((str1, str2)): - return SomeString() + # propagate const-ness to help getattr(obj, 'prefix' + const_name) + result = SomeString() + if str1.is_immutable_constant() and str2.is_immutable_constant(): + result.const = str1.const + str2.const + return result class __extend__(pairtype(SomeChar, SomeChar)): Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Tue Aug 22 19:26:13 2006 @@ -46,6 +46,12 @@ return IMM32(self.value) +class FnPtrConst(IntConst): + def __init__(self, value, mc): + self.value = value + self.mc = mc # to keep it alive + + class Block(object): def __init__(self, mc): self.argcount = 0 @@ -120,4 +126,10 @@ for i in range(block.argcount): prologue.mc.PUSH(operand) prologue.mc.JMP(rel32(block.startaddr)) - return IntConst(prologue.startaddr) + return FnPtrConst(prologue.startaddr, prologue.mc) + + def revealconst(T, gv_const): + assert isinstance(gv_const, IntConst) # for now + return lltype.cast_int_to_ptr(T, gv_const.value) + revealconst._annspecialcase_ = 'specialize:arg(0)' + revealconst = staticmethod(revealconst) Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Tue Aug 22 19:26:13 2006 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython import llinterp from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.objectmodel import keepalive_until_here from pypy.translator.c.test.test_genc import compile from pypy.jit.codegen.i386.ri386genop import RI386GenOp @@ -29,7 +30,9 @@ rgenop = RI386GenOp() gv_add_x = make_adder(rgenop, x) add_x = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_x) - return add_x(y) + res = add_x(y) + keepalive_until_here(gv_add_x) # to keep the 'add_x' fnptr alive + return res # ____________________________________________________________ @@ -42,7 +45,6 @@ assert res == 42 def test_adder_compile(): - import py; py.test.skip("in-progress") fn = compile(runner, [int, int]) res = fn(9080983, -9080941) assert res == 42 From arigo at codespeak.net Tue Aug 22 21:19:45 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Aug 2006 21:19:45 +0200 (CEST) Subject: [pypy-svn] r31509 - in pypy/dist/pypy/translator/c: . test Message-ID: <20060822191945.C68C91005A@code0.codespeak.net> Author: arigo Date: Tue Aug 22 21:19:42 2006 New Revision: 31509 Modified: pypy/dist/pypy/translator/c/support.py pypy/dist/pypy/translator/c/test/test_support.py Log: (pedronis, arigo) gen_assignments() was completely broken, but you need completely obscure graphs to see the difference anyway. Fixed. Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Tue Aug 22 21:19:42 2006 @@ -111,44 +111,79 @@ return '{%s}' % ', '.join(lines) +##def gen_assignments(assignments): +## # Generate a sequence of assignments that is possibly reordered +## # to avoid clashes -- i.e. do the equivalent of a tuple assignment, +## # reading all sources first, writing all targets next, but optimized + +## allsources = [] +## src2dest = {} +## types = {} +## for typename, dest, src in assignments: +## if src != dest: # ignore 'v=v;' +## allsources.append(src) +## src2dest.setdefault(src, []).append(dest) +## types[dest] = typename + +## for starting in allsources: +## # starting from some starting variable, follow a chain of assignments +## # 'vn=vn-1; ...; v3=v2; v2=v1; v1=starting;' +## v = starting +## srcchain = [] +## while src2dest.get(v): +## srcchain.append(v) +## v = src2dest[v].pop(0) +## if v == starting: +## break # loop +## if not srcchain: +## continue # already done in a previous chain +## srcchain.reverse() # ['vn-1', ..., 'v2', 'v1', 'starting'] +## code = [] +## for pair in zip([v] + srcchain[:-1], srcchain): +## code.append('%s = %s;' % pair) +## if v == starting: +## # assignment loop 'starting=vn-1; ...; v2=v1; v1=starting;' +## typename = types[starting] +## tmpdecl = cdecl(typename, 'tmp') +## code.insert(0, '{ %s = %s;' % (tmpdecl, starting)) +## code[-1] = '%s = tmp; }' % (srcchain[-2],) +## yield ' '.join(code) + def gen_assignments(assignments): # Generate a sequence of assignments that is possibly reordered # to avoid clashes -- i.e. do the equivalent of a tuple assignment, # reading all sources first, writing all targets next, but optimized - allsources = [] - src2dest = {} - types = {} - assignments = list(assignments) + srccount = {} + dest2src = {} for typename, dest, src in assignments: if src != dest: # ignore 'v=v;' - allsources.append(src) - src2dest.setdefault(src, []).append(dest) - types[dest] = typename - - for starting in allsources: - # starting from some starting variable, follow a chain of assignments - # 'vn=vn-1; ...; v3=v2; v2=v1; v1=starting;' - v = starting - srcchain = [] - while src2dest.get(v): - srcchain.append(v) - v = src2dest[v].pop(0) - if v == starting: - break # loop - if not srcchain: - continue # already done in a previous chain - srcchain.reverse() # ['vn-1', ..., 'v2', 'v1', 'starting'] - code = [] - for pair in zip([v] + srcchain[:-1], srcchain): - code.append('%s = %s;' % pair) - if v == starting: - # assignment loop 'starting=vn-1; ...; v2=v1; v1=starting;' - typename = types[starting] - tmpdecl = cdecl(typename, 'tmp') - code.insert(0, '{ %s = %s;' % (tmpdecl, starting)) - code[-1] = '%s = tmp; }' % (srcchain[-2],) - yield ' '.join(code) + srccount[src] = srccount.get(src, 0) + 1 + dest2src[dest] = src, typename + + while dest2src: + progress = False + for dst in dest2src.keys(): + if dst not in srccount: + src, typename = dest2src.pop(dst) + yield '%s = %s;' % (dst, src) + srccount[src] -= 1 + if not srccount[src]: + del srccount[src] + progress = True + if not progress: + # we are left with only pure disjoint cycles; break them + while dest2src: + dst, (src, typename) = dest2src.popitem() + startingpoint = dst + tmpdecl = cdecl(typename, 'tmp') + code = ['{ %s = %s;' % (tmpdecl, dst)] + while src is not startingpoint: + code.append('%s = %s;' % (dst, src)) + dst = src + src, typename = dest2src.pop(dst) + code.append('%s = tmp; }' % (dst,)) + yield ' '.join(code) # logging Modified: pypy/dist/pypy/translator/c/test/test_support.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_support.py (original) +++ pypy/dist/pypy/translator/c/test/test_support.py Tue Aug 22 21:19:42 2006 @@ -1,20 +1,39 @@ +import random from pypy.translator.c.support import gen_assignments -def gen_assign(input, expected): - assert ' '.join(gen_assignments(input)) == expected +def test_gen_simple_assignments(): + yield gen_check, [('int @', 'a', 'a')] + yield gen_check, [('int @', 'a', 'b')] + yield gen_check, [('int @', 'a', 'b'), + ('int @', 'c', 'b')] + yield gen_check, [('int @', 'a', 'b'), + ('int @', 'b', 'c')] + yield gen_check, [('int @', 'b', 'c'), + ('int @', 'a', 'b')] + yield gen_check, [('int @', 'a', 'b'), + ('int @', 'b', 'a')] + yield gen_check, [('int @', 'a', 'b'), + ('int @', 'b', 'c'), + ('int @', 'd', 'b')] -def test_gen_assignments(): - yield gen_assign, [('int @', 'a', 'a')], '' - yield gen_assign, [('int @', 'a', 'b')], 'a = b;' - yield gen_assign, [('int @', 'a', 'b'), - ('int @', 'c', 'b')], 'a = b; c = b;' - yield gen_assign, [('int @', 'a', 'b'), - ('int @', 'b', 'c')], 'a = b; b = c;' - yield gen_assign, [('int @', 'b', 'c'), - ('int @', 'a', 'b')], 'a = b; b = c;' - yield gen_assign, [('int @', 'a', 'b'), - ('int @', 'b', 'a')], '{ int tmp = b; b = a; a = tmp; }' - yield gen_assign, [('int @', 'a', 'b'), - ('int @', 'b', 'c'), - ('int @', 'd', 'b')], 'a = b; d = b; b = c;' +def gen_check(input): + for _, dst, src in input: + print 'input:', dst, src + result = ' '.join(gen_assignments(input)) + print result + result = result.replace('{ int', '').replace('}', '').strip() + d = {} + for _, dst, src in input: + d[src] = '' % (src,) + exec result in d + for _, dst, src in input: + assert d[dst] == '' % (src,) + +def test_gen_check(): + varlist = list('abcdefg') + for i in range(100): + random.shuffle(varlist) + input = [('int @', varlist[n], random.choice(varlist)) + for n in range(random.randrange(1, 7))] + yield gen_check, input From mwh at codespeak.net Tue Aug 22 21:20:21 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Tue, 22 Aug 2006 21:20:21 +0200 (CEST) Subject: [pypy-svn] r31510 - pypy/dist/pypy/interpreter Message-ID: <20060822192021.363301005A@code0.codespeak.net> Author: mwh Date: Tue Aug 22 21:20:19 2006 New Revision: 31510 Modified: pypy/dist/pypy/interpreter/function.py Log: oops, fix silly breakage Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Tue Aug 22 21:20:19 2006 @@ -345,7 +345,7 @@ if w_firstarg is None: instdescr = "nothing" else: - instname = self.abstract_getclass(space, w_firstarg).getname(space,"") + instname = space.abstract_getclass(w_firstarg).getname(space,"") if instname: instname += " " instdescr = "%sinstance" %instname From hpk at codespeak.net Wed Aug 23 11:41:53 2006 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 23 Aug 2006 11:41:53 +0200 (CEST) Subject: [pypy-svn] r31515 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060823094153.70D2A10034@code0.codespeak.net> Author: hpk Date: Wed Aug 23 11:41:50 2006 New Revision: 31515 Modified: pypy/extradoc/sprintinfo/ireland-2006/planning.txt Log: planning session ends Modified: pypy/extradoc/sprintinfo/ireland-2006/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/planning.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/planning.txt Wed Aug 23 11:41:50 2006 @@ -8,19 +8,20 @@ 5pm tutorial + Q & A session bea arranges a dinner table friday evening -* source optimizations +* core optimizations (mwh, arre) - - builtin lookups - - caching method lookups - - profiling + - (done) did a lot of profiling, main finding: GC + - (next) optimising GC + - experiment with optimising builtin lookups + e.g caching method lookups or callback-dict * ext compiler maybe a topic for the weekend * JIT - - so far generating a jit for simple interpreters kind of - works (armin, samuele) + - (done) first machine code generation tests are passing + (next) more of the same - produce machine code in-memory from running the generated CFGs - maybe think about refining the interface to generating I386 machine code (also considering a PPC asm backend) @@ -28,9 +29,12 @@ * distributed testing (maciej, hpk) - - use "/usr/bin/rsync" to sync to remote - - master does collection of items - - "leaf" items are asynchronously send to "Executors" (running on nodes) - - super-simple reporting + more or less done, but many things : + - use "/usr/bin/rsync" to sync to remote + - master does collection of items + - "leaf" items are asynchronously send to "Executors" (running on nodes) + - super-simple reporting + + (arigo) wrote a generic interface that works via greenlets or threads interchangeable From arigo at codespeak.net Wed Aug 23 11:44:01 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 11:44:01 +0200 (CEST) Subject: [pypy-svn] r31516 - in pypy/dist/pypy/jit/codegen/i386: . test Message-ID: <20060823094401.C8F2710050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 11:43:59 2006 New Revision: 31516 Modified: pypy/dist/pypy/jit/codegen/i386/ri386.py pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Log: (pedronis, arigo) Some link support. Fun copying values from a "previous stack layout" to a "new stack layout" overlapping it. Modified: pypy/dist/pypy/jit/codegen/i386/ri386.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386.py Wed Aug 23 11:43:59 2006 @@ -84,6 +84,12 @@ imm16 = IMM16 rel32 = REL32 +def imm(value): + if single_byte(value): + return imm8(value) + else: + return imm32(value) + def memregister(register): assert register.width == 4 return MODRM(0xC0 | register.op, '') Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Wed Aug 23 11:43:59 2006 @@ -25,7 +25,7 @@ self.stackpos = stackpos def operand(self, block): - return mem(esp, WORD * (block.stackdepth-1 - self.stackpos)) + return block.stack_access(self.stackpos) class TypeConst(VarOrConst): @@ -40,10 +40,7 @@ self.value = value def operand(self, block): - if single_byte(self.value): - return IMM8(self.value) - else: - return IMM32(self.value) + return imm(self.value) class FnPtrConst(IntConst): @@ -65,6 +62,9 @@ self.stackdepth += 1 return res + def stack_access(self, stackpos): + return mem(esp, WORD * (self.stackdepth-1 - stackpos)) + def push(self, reg): self.mc.PUSH(reg) res = Var(self.stackdepth) @@ -76,25 +76,97 @@ self.mc.ADD(eax, gv_y.operand(self)) return self.push(eax) + def op_int_sub(self, (gv_x, gv_y), gv_RESTYPE): + self.mc.MOV(eax, gv_x.operand(self)) + self.mc.SUB(eax, gv_y.operand(self)) + return self.push(eax) + class RI386GenOp(object): gv_IntWord = TypeConst('IntWord') gv_Void = TypeConst('Void') def __init__(self): - self.mc = MachineCodeBlock(65536) # XXX!!! + self.mcs = [] # machine code blocks where no-one is currently writing + + def open_mc(self): + if self.mcs: + # XXX think about inserting NOPS for alignment + return self.mcs.pop() + else: + return MachineCodeBlock(65536) # XXX supposed infinite for now + + def close_mc(self, mc): + self.mcs.append(mc) def newblock(self): - # XXX concurrently-open Blocks cannot use the same mc - return Block(self.mc) + return Block(self.open_mc()) def closeblock1(self, block): - return block + return block # NB. links and blocks are the same for us def closereturnlink(self, link, gv_result): link.mc.MOV(eax, gv_result.operand(link)) - link.mc.ADD(esp, IMM32(WORD * link.stackdepth)) + link.mc.ADD(esp, imm32(WORD * link.stackdepth)) link.mc.RET() + self.close_mc(link.mc) + + def closelink(self, link, outputargs_gv, targetblock): + N = len(outputargs_gv) + if link.stackdepth < N: + link.mc.SUB(esp, imm(WORD * (N - link.stackdepth))) + link.stackdepth = N + + pending_dests = N + srccount = [0] * N + for i in range(N): + gv = outputargs_gv[i] + if isinstance(gv, Var): + p = gv.stackpos + if 0 <= p < N: + if p == i: + srccount[p] = -N # ignore 'v=v' + pending_dests -= 1 + else: + srccount[p] += 1 + + while pending_dests: + progress = False + for i in range(N): + if srccount[i] == 0: + srccount[i] = -1 + pending_dests -= 1 + gv_src = outputargs_gv[i] + link.mc.MOV(eax, gv_src.operand(link)) + link.mc.MOV(link.stack_access(i), eax) + progress = True + if not progress: + # we are left with only pure disjoint cycles; break them + for i in range(N): + if srccount[i] >= 0: + dst = i + link.mc.MOV(edx, link.stack_access(dst)) + while True: + assert srccount[dst] == 1 + srccount[dst] = -1 + pending_dests -= 1 + gv_src = outputargs_gv[dst] + assert isinstance(gv_src, Var) + src = gv_src.stackpos + assert 0 <= src < N + if src == i: + break + link.mc.MOV(eax, link.stack_access(src)) + link.mc.MOV(link.stack_access(dst), eax) + dst = src + link.mc.MOV(link.stack_access(dst), edx) + assert pending_dests == 0 + + if link.stackdepth > N: + link.mc.ADD(esp, imm(WORD * (link.stackdepth - N))) + link.stackdepth = N + link.mc.JMP(rel32(targetblock.startaddr)) + self.close_mc(link.mc) def geninputarg(self, block, gv_TYPE): return block.geninputarg(gv_TYPE) @@ -122,10 +194,13 @@ def gencallableconst(self, name, block, gv_FUNCTYPE): prologue = self.newblock() #prologue.mc.BREAKPOINT() - operand = mem(esp, WORD * block.argcount) + # re-push the arguments so that they are after the return value + # and in the correct order for i in range(block.argcount): + operand = mem(esp, WORD * (2*i+1)) prologue.mc.PUSH(operand) prologue.mc.JMP(rel32(block.startaddr)) + self.close_mc(prologue.mc) return FnPtrConst(prologue.startaddr, prologue.mc) def revealconst(T, gv_const): Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Wed Aug 23 11:43:59 2006 @@ -1,6 +1,5 @@ from pypy.rpython.lltypesystem import lltype -from pypy.rpython import llinterp -from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.objectmodel import keepalive_until_here from pypy.translator.c.test.test_genc import compile from pypy.jit.codegen.i386.ri386genop import RI386GenOp @@ -31,10 +30,16 @@ gv_add_x = make_adder(rgenop, x) add_x = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_x) res = add_x(y) - keepalive_until_here(gv_add_x) # to keep the 'add_x' fnptr alive + keepalive_until_here(rgenop) # to keep the code blocks alive return res -# ____________________________________________________________ +def test_adder_interpret(): + from pypy.rpython import rgenop + gv_add_5 = make_adder(rgenop, 5) + add_5 = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_5) + llinterp = LLInterpreter(None) + res = llinterp.eval_graph(add_5._obj.graph, [12]) + assert res == 17 def test_adder_direct(): rgenop = RI386GenOp() @@ -48,3 +53,63 @@ fn = compile(runner, [int, int]) res = fn(9080983, -9080941) assert res == 42 + +# ____________________________________________________________ + +FUNC2 = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + +def make_dummy(rgenop): + # 'return x - (y - (x-1))' + gv_SIGNED = rgenop.constTYPE(lltype.Signed) + block = rgenop.newblock() + gv_x = rgenop.geninputarg(block, gv_SIGNED) + gv_y = rgenop.geninputarg(block, gv_SIGNED) + args_gv = [gv_x, rgenop.genconst(1)] + gv_z = rgenop.genop(block, "int_sub", args_gv, gv_SIGNED) + link = rgenop.closeblock1(block) + + block2 = rgenop.newblock() + gv_y2 = rgenop.geninputarg(block2, gv_SIGNED) + gv_z2 = rgenop.geninputarg(block2, gv_SIGNED) + gv_x2 = rgenop.geninputarg(block2, gv_SIGNED) + rgenop.closelink(link, [gv_y, gv_z, gv_x], block2) + + args_gv = [gv_y2, gv_z2] + gv_s2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) + args_gv = [gv_x2, gv_s2] + gv_t2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) + link2 = rgenop.closeblock1(block2) + + rgenop.closereturnlink(link2, gv_t2) + gv_FUNC2 = rgenop.constTYPE(FUNC2) + gv_dummyfn = rgenop.gencallableconst("dummy", block, gv_FUNC2) + return gv_dummyfn + +def dummy_runner(x, y): + rgenop = RI386GenOp() + gv_dummyfn = make_dummy(rgenop) + dummyfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_dummyfn) + res = dummyfn(x, y) + keepalive_until_here(rgenop) # to keep the code blocks alive + return res + +def test_dummy_interpret(): + from pypy.rpython import rgenop + gv_dummyfn = make_dummy(rgenop) + dummyfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_dummyfn) + llinterp = LLInterpreter(None) + res = llinterp.eval_graph(dummyfn._obj.graph, [30, 17]) + assert res == 42 + +def test_dummy_direct(): + rgenop = RI386GenOp() + gv_dummyfn = make_dummy(rgenop) + print gv_dummyfn.value + fnptr = cast(c_void_p(gv_dummyfn.value), CFUNCTYPE(c_int, c_int, c_int)) + res = fnptr(30, 17) # <== the segfault is here + assert res == 42 + +def test_dummy_compile(): + fn = compile(dummy_runner, [int, int]) + res = fn(40, 37) + assert res == 42 From arigo at codespeak.net Wed Aug 23 11:44:41 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 11:44:41 +0200 (CEST) Subject: [pypy-svn] r31517 - pypy/dist/pypy/translator/c Message-ID: <20060823094441.E8FA910050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 11:44:40 2006 New Revision: 31517 Modified: pypy/dist/pypy/translator/c/support.py Log: (pedronis, arigo) Add asserts. Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Wed Aug 23 11:44:40 2006 @@ -175,6 +175,7 @@ # we are left with only pure disjoint cycles; break them while dest2src: dst, (src, typename) = dest2src.popitem() + assert srccount[dst] == 1 startingpoint = dst tmpdecl = cdecl(typename, 'tmp') code = ['{ %s = %s;' % (tmpdecl, dst)] @@ -182,6 +183,7 @@ code.append('%s = %s;' % (dst, src)) dst = src src, typename = dest2src.pop(dst) + assert srccount[dst] == 1 code.append('%s = tmp; }' % (dst,)) yield ' '.join(code) From arigo at codespeak.net Wed Aug 23 12:09:12 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 12:09:12 +0200 (CEST) Subject: [pypy-svn] r31520 - in pypy/dist/pypy: jit/llabstractinterp jit/timeshifter rpython rpython/test Message-ID: <20060823100912.B00E810050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 12:09:00 2006 New Revision: 31520 Modified: pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/rpython/rgenop.py pypy/dist/pypy/rpython/test/test_rgenop.py Log: (pedronis, arigo) Evil plan to remove LINKPAIR and return a tuple directly from closeblock2(). Modified: pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py ============================================================================== --- pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py (original) +++ pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py Wed Aug 23 12:09:00 2006 @@ -530,8 +530,8 @@ assert len(newlinkstates) == 2 - exitspair = rgenop.closeblock2(b, newexitswitch.getgenvar(self)) - false_link, true_link = exitspair.item0, exitspair.item1 + v = newexitswitch.getgenvar(self) + false_link, true_link = rgenop.closeblock2(b, v) cases = {False: false_link, True: true_link} for ls in newlinkstates: Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Wed Aug 23 12:09:00 2006 @@ -310,8 +310,7 @@ self.outgoinglink = rgenop.closeblock1(self.block) def leave_block_split(self, exitgvar): - linkpair = rgenop.closeblock2(self.block, exitgvar) - false_link, true_link = linkpair.item0, linkpair.item1 + false_link, true_link = rgenop.closeblock2(self.block, exitgvar) later_builder = ResidualGraphBuilder(link=false_link) self.outgoinglink = true_link return later_builder Modified: pypy/dist/pypy/rpython/rgenop.py ============================================================================== --- pypy/dist/pypy/rpython/rgenop.py (original) +++ pypy/dist/pypy/rpython/rgenop.py Wed Aug 23 12:09:00 2006 @@ -126,7 +126,7 @@ block.closeblock(link) return to_opaque_object(link) -def closeblock2into(blockcontainer, exitswitch, linkpair): +def closeblock2(blockcontainer, exitswitch): block = from_opaque_object(blockcontainer.obj) exitswitch = from_opaque_object(exitswitch) assert isinstance(exitswitch, flowmodel.Variable) @@ -138,13 +138,19 @@ true_link.exitcase = True true_link.llexitcase = True block.closeblock(false_link, true_link) - linkpair.item0 = to_opaque_object(false_link) - linkpair.item1 = to_opaque_object(true_link) + return pseudotuple(to_opaque_object(false_link), + to_opaque_object(true_link)) -def closeblock2(blockcontainer, exitswitch): - linkpair = lltype.malloc(LINKPAIR) - closeblock2into(blockcontainer, exitswitch, linkpair) - return linkpair +class pseudotuple(object): + # something that looks both like a hl and a ll tuple + def __init__(self, *items): + self._TYPE = rtupletype.TUPLE_TYPE( + [lltype.typeOf(item) for item in items]) + for i, item in enumerate(items): + setattr(self, 'item%d' % i, item) + self._items = items + def __iter__(self): + return iter(self._items) def _closelink(link, vars, targetblock): if isinstance(link, flowmodel.Link): @@ -237,7 +243,6 @@ BLOCKCONTAINERTYPE = blocktypeinfo.get_lltype() BLOCK = lltype.Ptr(BLOCKCONTAINERTYPE) LINK = lltype.Ptr(linktypeinfo.get_lltype()) -LINKPAIR = rtupletype.TUPLE_TYPE([LINK, LINK]).TO # support constants and types @@ -280,7 +285,7 @@ s_ConstOrVar = annmodel.SomePtr(CONSTORVAR)#annmodel.SomeExternalObject(flowmodel.Variable) s_Link = annmodel.SomePtr(LINK)#annmodel.SomeExternalObject(flowmodel.Link) -s_LinkPair = annmodel.SomePtr(lltype.Ptr(LINKPAIR)) +s_LinkPair = annmodel.SomeTuple([s_Link, s_Link]) setannotation(initblock, None) setannotation(geninputarg, s_ConstOrVar) Modified: pypy/dist/pypy/rpython/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rgenop.py (original) +++ pypy/dist/pypy/rpython/test/test_rgenop.py Wed Aug 23 12:09:00 2006 @@ -52,8 +52,7 @@ v0 = geninputarg(block, constTYPE(Signed)) const0 = genconst(0) v1 = genop(block, 'int_lt', [v0, const0], constTYPE(Bool)) - exitspair = closeblock2(block, v1) - false_link, true_link = exitspair.item0, exitspair.item1 + false_link, true_link = closeblock2(block, v1) closereturnlink(true_link, const0) closereturnlink(false_link, v0) return block @@ -95,8 +94,7 @@ result1 = genop(loopblock, 'int_mul', [result0, i0], constTYPE(Signed)) i1 = genop(loopblock, 'int_add', [i0, const1], constTYPE(Signed)) v2 = genop(loopblock, 'int_le', [i1, v1], constTYPE(Bool)) - exitspair = closeblock2(loopblock, v2) - false_link, true_link = exitspair.item0, exitspair.item1 + false_link, true_link = closeblock2(loopblock, v2) closereturnlink(false_link, result1) closelink(true_link, [result1, i1, v1], loopblock) return block From arigo at codespeak.net Wed Aug 23 12:29:20 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 12:29:20 +0200 (CEST) Subject: [pypy-svn] r31522 - in pypy/dist/pypy/jit/codegen/i386: . test Message-ID: <20060823102920.DD98210050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 12:29:17 2006 New Revision: 31522 Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Log: (pedronis, arigo) Support for branches. Easy. Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Wed Aug 23 12:29:17 2006 @@ -81,6 +81,13 @@ self.mc.SUB(eax, gv_y.operand(self)) return self.push(eax) + def op_int_gt(self, (gv_x, gv_y), gv_RESTYPE): + self.mc.MOV(eax, gv_x.operand(self)) + self.mc.CMP(eax, gv_y.operand(self)) + self.mc.SETG(al) + self.mc.MOVZX(eax, al) + return self.push(eax) + class RI386GenOp(object): gv_IntWord = TypeConst('IntWord') @@ -105,9 +112,16 @@ def closeblock1(self, block): return block # NB. links and blocks are the same for us + def closeblock2(self, block, gv_condition): + false_block = self.newblock() + false_block.stackdepth = block.stackdepth + block.mc.CMP(gv_condition.operand(block), imm8(0)) + block.mc.JE(rel32(false_block.startaddr)) + return false_block, block + def closereturnlink(self, link, gv_result): link.mc.MOV(eax, gv_result.operand(link)) - link.mc.ADD(esp, imm32(WORD * link.stackdepth)) + link.mc.ADD(esp, imm(WORD * link.stackdepth)) link.mc.RET() self.close_mc(link.mc) Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Wed Aug 23 12:29:17 2006 @@ -113,3 +113,67 @@ fn = compile(dummy_runner, [int, int]) res = fn(40, 37) assert res == 42 + +# ____________________________________________________________ + +def make_branching(rgenop): + # 'if x > 5: return x-1 + # else: return y' + gv_SIGNED = rgenop.constTYPE(lltype.Signed) + gv_BOOL = rgenop.constTYPE(lltype.Bool) + block = rgenop.newblock() + gv_x = rgenop.geninputarg(block, gv_SIGNED) + gv_y = rgenop.geninputarg(block, gv_SIGNED) + args_gv = [gv_x, rgenop.genconst(5)] + gv_cond = rgenop.genop(block, "int_gt", args_gv, gv_BOOL) + link_false, link_true = rgenop.closeblock2(block, gv_cond) + + block2 = rgenop.newblock() + gv_x2 = rgenop.geninputarg(block2, gv_SIGNED) + rgenop.closelink(link_true, [gv_x], block2) + + args_gv = [gv_x2, rgenop.genconst(1)] + gv_s2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) + link2 = rgenop.closeblock1(block2) + rgenop.closereturnlink(link2, gv_s2) + + rgenop.closereturnlink(link_false, gv_y) + + gv_FUNC2 = rgenop.constTYPE(FUNC2) + gv_branchingfn = rgenop.gencallableconst("branching", block, gv_FUNC2) + return gv_branchingfn + +def branching_runner(x, y): + rgenop = RI386GenOp() + gv_branchingfn = make_branching(rgenop) + branchingfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_branchingfn) + res = branchingfn(x, y) + keepalive_until_here(rgenop) # to keep the code blocks alive + return res + +def test_branching_interpret(): + from pypy.rpython import rgenop + gv_branchingfn = make_branching(rgenop) + branchingfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_branchingfn) + llinterp = LLInterpreter(None) + res = llinterp.eval_graph(branchingfn._obj.graph, [30, 17]) + assert res == 29 + res = llinterp.eval_graph(branchingfn._obj.graph, [3, 17]) + assert res == 17 + +def test_branching_direct(): + rgenop = RI386GenOp() + gv_branchingfn = make_branching(rgenop) + print gv_branchingfn.value + fnptr = cast(c_void_p(gv_branchingfn.value), CFUNCTYPE(c_int, c_int, c_int)) + res = fnptr(30, 17) # <== the segfault is here + assert res == 29 + res = fnptr(3, 17) # <== or here + assert res == 17 + +def test_branching_compile(): + fn = compile(branching_runner, [int, int]) + res = fn(30, 17) + assert res == 29 + res = fn(3, 17) + assert res == 17 From mwh at codespeak.net Wed Aug 23 12:42:35 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 12:42:35 +0200 (CEST) Subject: [pypy-svn] r31524 - pypy/branch/no-zeroing-assumption Message-ID: <20060823104235.C1AE410050@code0.codespeak.net> Author: mwh Date: Wed Aug 23 12:42:34 2006 New Revision: 31524 Added: pypy/branch/no-zeroing-assumption/ - copied from r31523, pypy/dist/ Log: a branch for another go at the zeroing assumption From bea at codespeak.net Wed Aug 23 12:53:40 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 12:53:40 +0200 (CEST) Subject: [pypy-svn] r31525 - pypy/extradoc/sprintinfo/mallorca Message-ID: <20060823105340.3A48010060@code0.codespeak.net> Author: bea Date: Wed Aug 23 12:53:38 2006 New Revision: 31525 Added: pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt (contents, props changed) Log: creating a separate file from two different pypy-dev postings (doing this because of creating an overview page with "all" sprint reports 2003-2006) Added: pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt Wed Aug 23 12:53:38 2006 @@ -0,0 +1,140 @@ +Hi all! + +This is the first report of the challenging Mallorca sprint. On the one +hand getting here proved to be quite an ordeal for some of the sprinters +due to bad weather at airports and uncooperative ferry companies. On the +other hand it seems that we have finally run out of even vaguely easy +tasks to do and are thus involved in serious head-scratching and the +making of strange noises as we contemplate what we have to do. + +The sprint is hosted in the GNU/Linux lab of the Universitat de les +Illes Balears in Palma de Mallorca. We are very thankful to them for +letting us disturb their undergraduates in their attempts to use the +computers. + +On the first day some of the strangest noises were produced by Samuele +who started together with Gerald and Stephan to work on a static version +of ctypes for rpython. On Tuesday evening, after lots of tedious work on +all levels of the translation toolchain, they managed (with some +cheating) to got an example calling atoi to run. + +Michael and Arre started extending the l3interpreter to support +operations involving heap-structures in a way that could be run on top +of the llinterpreter and after translation to C. After some argument +with Samuele, who, as usual, turned out to be right, they were +successful in their goals. This means that the l3intepreter can now +perform reasonably complex operations involving pointers but not +subtract two integers. + +Carl Friedrich and Richard started refactoring the gc framework to use +the new facilities Michael and Arre had written. Progress there was +extreeeemely slow, so that in the evening of the first day only marginal +progress was achieved. + +Christian, Eric and later Jacob started exposing PyPy's stackless +features to application level. While doing this they found a few +problems in the interp-level code and refactored it a bit. On the second + and third day Christian and Jacob continued with this. They managed to +expose simple coroutines on Wednesday and are now continuing to create +nicer interfaces following the stackless module of stackless Python. One +of the problems there turned out to be testing turnaround time, since +the functionality can only be tested after a full translation has been +run. There might be some ways around this involving slimy hacks. + +On the second day most pairs continued as before. Armin and Carl +Friedrich worked on the JIT. They started defining an interface to +generate code at runtime (see +http://codespeak.net/svn/pypy/dist/pypy/doc/discussion/draft-jit-ideas.txt) +and then using this same interface in the low level abstract +interpreter. This turned out to be quite difficult in the sense that +most of the time was spent re-redesigning the interface yet another time. + +On the third day Armin swapped with Samuele and continued with Gerald +and Stephan to work on rctypes, with slow but steady progress. They are +now able to annotate simple cases that use structures. + +Arre and Samuele tied themselves to the JIT treadmill and continued the +work of Armin and Carl. A lot of "wuah!"'s and "arrgh!"'s can be heared +from their direction -- we assume they are having fun. + +Carl Friedrich and Michael thought about how to integrate the currently +only simulated GC framework with the backends. This looked and still +looks like it will be *a lot of* work but they were able to at least +clarify some issues and get somewhat started. + +Eric and Richard tried to finally remove the dependency on pyrex (!) in +genllvm which is a relict from the old days of the first LLVM backend +(which was pre-rtyper!). They are also working on removing some +hand-written C code from genllvm. + +So while we're not seeing super-rapid progress there are still +interesting changes continuously arriving. Tomorrow is a day off, but +given the bunch of obsessives we are there might still be some checkins. +There will still be three full days of sprinting afterwards. + +Cheers from sunny Palma, + +Michael & Carl Friedrich + +We greet you once again from the Aula Linux lab of the UIB in Palma de +Mallorca. There is one more day of sprinting, but as many people have +had to leave slightly early, it's probably safe to attempt some kind +of summary. + +Thursday was the break day, which didn't stop some mad people from +doing a little work on rctypes and the JIT. The sane ones among us +went hiking or to artists' studios instead :) + +Friday was not the most productive of days, partly becase we gave a +talk to some people from the university and the local free software +user group in the afternoon. Also contributing to the lack of results +were the general difficulty of the tasks and some amazing getting-lost +skills ("getting lost" here refers to the physical world for a change). + +Armin, Arre and Samuele worked on the JIT, the idea being to write an +"abstract abstract interpreter", which would specialize an abstract +interpreter of low-level graphs for a particular set of low-level +graphs. At lunch they told a hilarious joke that they weren't going +to do this at all and would instead re-use the annotator to extract +the needed information. This then turned out to not be a joke at all, +and is exactly what they did for the next two days (apparently they +plan to re-use the rtyper as well, but this is just preposterous). + +Christian worked on coroutines and other stackless-like functionality, +with good results: you can now build a version of pypy that can use +and freely intermingle raw coroutines, tasklets and greenlets all at +once. Apparently this is all fairly natural in the implementation, +but must allow for the possibility of some *deeply* obscure user +code... (obfuscated pypy contests, anyone?) + +Michael and Carl repeatedly bashed their heads onto the topic of gc +integration, the desk, reference counting details, walls, genc and so +on. The basic goal was to express the logic behind placing incref and +decref operations as a nice and fairly clear transformation of the +flow graphs, as opposed to backend-specific incomprehensible hacks. +This task was made considerably harder by exceptions (who uses them, +anyway?) but hopefully by the time we've finished writing this report +we'll have built a pypy-c in the new style. This work should allow +(finally) the integration of the GC framework Carl wrote as his Summer +of Code project, and mere mortals to understand genc. + +Gerald and Stephan continued to work on "rctypes" -- a static version +of ctypes for RPython. Reasonably complex ctypes declarations can now +be annotated. + +Eric and Richard worked on a transformation to replace some of the +operation that can raise exceptions with direct_call operations -- +something that will be very useful for the GC work mentioned above, +because operations that can raise an exception need special treatment +-- so the fewer the better! + +And finally, some really interesting news: Armin plans to release +Bub-n-Bros 1.5, the play testing of which delayed this morning's +status meeting considerably :) + +To sum up, this sprint has seen a fair amount of work on very +challenging tasks. As usual we all need to sleep for a week, so it's +time to leave this wonderful island (also, it's raining). + +Cheers, +mwh and Carl Friedrich \ No newline at end of file From mwh at codespeak.net Wed Aug 23 13:17:19 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 13:17:19 +0200 (CEST) Subject: [pypy-svn] r31528 - in pypy/branch/no-zeroing-assumption/pypy: annotation rpython rpython/lltypesystem rpython/lltypesystem/test rpython/ootypesystem rpython/test Message-ID: <20060823111719.C4C6910036@code0.codespeak.net> Author: mwh Date: Wed Aug 23 13:17:08 2006 New Revision: 31528 Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_picklelltype.py pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py pypy/branch/no-zeroing-assumption/pypy/rpython/rcpy.py pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_llinterp.py Log: beginnings of removing the zeroing assumption from lltype. rdict is still broken (and boy is that going to be fun to fix) and this explains the majority of the 50-odd failures in py.test rpython/test. Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py Wed Aug 23 13:17:08 2006 @@ -669,7 +669,7 @@ example = p.ll_ptrtype._example() if example[0] is not None: # ignore Void s_value v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() + example[0] = v_lltype._defl(example=True) setitem.can_only_throw = [] class __extend__(pairtype(SomePtr, SomeObject)): Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py Wed Aug 23 13:17:08 2006 @@ -414,7 +414,7 @@ def cast_primitive(T, s_v): assert T.is_constant() - return ll_to_annotation(lltype.cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(lltype.cast_primitive(T.const, annotation_to_lltype(s_v)._defl(example=True))) def nullptr(T): assert T.is_constant() @@ -424,13 +424,13 @@ def cast_pointer(PtrT, s_p): assert isinstance(s_p, SomePtr), "casting of non-pointer: %r" % s_p assert PtrT.is_constant() - cast_p = lltype.cast_pointer(PtrT.const, s_p.ll_ptrtype._defl()) + cast_p = lltype.cast_pointer(PtrT.const, s_p.ll_ptrtype._defl(example=True)) return SomePtr(ll_ptrtype=lltype.typeOf(cast_p)) def cast_opaque_ptr(PtrT, s_p): assert isinstance(s_p, SomePtr), "casting of non-pointer: %r" % s_p assert PtrT.is_constant() - cast_p = lltype.cast_opaque_ptr(PtrT.const, s_p.ll_ptrtype._defl()) + cast_p = lltype.cast_opaque_ptr(PtrT.const, s_p.ll_ptrtype._defl(example=True)) return SomePtr(ll_ptrtype=lltype.typeOf(cast_p)) def direct_fieldptr(s_p, s_fieldname): Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py Wed Aug 23 13:17:08 2006 @@ -605,10 +605,10 @@ example = p.ll_ptrtype._example() if getattr(example, s_attr.const) is not None: # ignore Void s_value v_lltype = annotation_to_lltype(s_value) - setattr(example, s_attr.const, v_lltype._defl()) + setattr(example, s_attr.const, v_lltype._defl(example=True)) def simple_call(p, *args_s): - llargs = [annotation_to_lltype(arg_s)._defl() for arg_s in args_s] + llargs = [annotation_to_lltype(arg_s)._defl(example=True) for arg_s in args_s] v = p.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py Wed Aug 23 13:17:08 2006 @@ -14,6 +14,8 @@ TLS = tlsobject() +Uninitialized = object() + def saferecursive(func, defl): def safe(*args): try: @@ -113,7 +115,7 @@ def _short_name(self): return str(self) - def _defl(self, parent=None, parentindex=None): + def _defl(self, parent=None, parentindex=None, example=False): raise NotImplementedError def _freeze_(self): @@ -252,15 +254,16 @@ def _short_name(self): return "%s %s" % (self.__class__.__name__, self._name) - def _defl(self, parent=None, parentindex=None): - return _struct(self, parent=parent, parentindex=parentindex) + def _defl(self, parent=None, parentindex=None, example=False): + return _struct(self, parent=parent, parentindex=parentindex, + example=example) def _container_example(self): if self._arrayfld is None: n = None else: n = 1 - return _struct(self, n) + return _struct(self, n, example=True) class RttiStruct(Struct): _runtime_type_info = None @@ -356,7 +359,7 @@ _short_name = saferecursive(_short_name, '...') def _container_example(self): - return _array(self, 1) + return _array(self, 1, example=True) class GcArray(Array): _gckind = 'gc' @@ -422,7 +425,7 @@ def _container_example(self): def ex(*args): - return self.RESULT._defl() + return self.RESULT._defl(example=True) return _func(self, _callable=ex) def _trueargs(self): @@ -445,7 +448,7 @@ def _container_example(self): return _opaque(self) - def _defl(self, parent=None, parentindex=None): + def _defl(self, parent=None, parentindex=None, example=False): return _opaque(self, parent=parent, parentindex=parentindex) RuntimeTypeInfo = OpaqueType("RuntimeTypeInfo") @@ -466,7 +469,7 @@ return "PyObject" def _inline_is_varsize(self, last): return False - def _defl(self, parent=None, parentindex=None): + def _defl(self, parent=None, parentindex=None, example=False): return _pyobjheader(parent, parentindex) PyObject = PyObjectType() @@ -505,13 +508,16 @@ def __str__(self): return self._name - def _defl(self, parent=None, parentindex=None): + def _defl(self, parent=None, parentindex=None, example=False): + if not example and self is not Void: + return Uninitialized return self._default def _is_atomic(self): return True - _example = _defl + def _example(self, parent=None, parentindex=None, example=False): + return self._default class Number(Primitive): @@ -574,8 +580,11 @@ def _is_atomic(self): return self.TO._gckind == 'raw' - def _defl(self, parent=None, parentindex=None): - return _ptr(self, None) + def _defl(self, parent=None, parentindex=None, example=False): + if example or self._needsgc: + return _ptr(self, None) + else: + return Uninitialized def _example(self): o = self.TO._container_example() @@ -589,6 +598,8 @@ try: return val._TYPE except AttributeError: + if val is Uninitialized: + raise UninitializedMemoryAccess("typeOf uninitialized value") tp = type(val) if tp is NoneType: return Void # maybe @@ -826,6 +837,9 @@ class DelayedPointer(Exception): pass +class UninitializedMemoryAccess(Exception): + pass + class _ptr(object): __slots__ = ('_TYPE', '_T', '_weak', '_solid', @@ -920,6 +934,9 @@ if isinstance(self._T, Struct): if field_name in self._T._flds: o = getattr(self._obj, field_name) + if o is Uninitialized: + raise UninitializedMemoryAccess( + "%r->%s"%(self, field_name)) return _expose(o, self._solid) if isinstance(self._T, ContainerType): try: @@ -1193,11 +1210,11 @@ __slots__ = () - def __new__(self, TYPE, n=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, parent=None, parentindex=None, example=False): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, parent=None, parentindex=None, example=False): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1206,9 +1223,9 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, parent=self, parentindex=fld) + value = _array(typ, n, parent=self, parentindex=fld, example=example) else: - value = typ._defl(parent=self, parentindex=fld) + value = typ._defl(parent=self, parentindex=fld, example=example) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1227,7 +1244,7 @@ for name in names: T = self._TYPE._flds[name] if isinstance(T, Primitive): - reprvalue = repr(getattr(self, name)) + reprvalue = repr(getattr(self, name, '')) else: reprvalue = '...' fields.append('%s=%s' % (name, reprvalue)) @@ -1270,13 +1287,13 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, parent=None, parentindex=None): + def __init__(self, TYPE, n, parent=None, parentindex=None, example=False): if not isinstance(n, int): raise TypeError, "array length must be an int" if n < 0: raise ValueError, "negative array length" _parentable.__init__(self, TYPE) - self.items = [TYPE.OF._defl(parent=self, parentindex=j) + self.items = [TYPE.OF._defl(parent=self, parentindex=j, example=example) for j in range(n)] if parent is not None: self._setparentstructure(parent, parentindex) @@ -1285,6 +1302,8 @@ return '<%s>' % (self,) def _str_item(self, item): + if type(item) is object: + return '#' if isinstance(self._TYPE.OF, Struct): of = self._TYPE.OF if self._TYPE._anonym_struct: @@ -1319,7 +1338,10 @@ def getitem(self, index): try: - return self.items[index] + v = self.items[index] + if v is Uninitialized: + raise UninitializedMemoryAccess("%r[%s]"%(self, index)) + return v except IndexError: if (self._TYPE._hints.get('isrpystring', False) and index == len(self.items)): @@ -1502,11 +1524,11 @@ return "pyobjheader of type %r" % (getattr(self, 'ob_type', '???'),) -def malloc(T, n=None, flavor='gc', immortal=False, extra_args=()): +def malloc(T, n=None, flavor='gc', immortal=False, extra_args=(), zero=False): if isinstance(T, Struct): - o = _struct(T, n) + o = _struct(T, n, example=zero) elif isinstance(T, Array): - o = _array(T, n) + o = _array(T, n, example=zero) else: raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py Wed Aug 23 13:17:08 2006 @@ -551,7 +551,7 @@ def ll_newdict(DICT): d = lltype.malloc(DICT) d.entries = lltype.malloc(DICT.entries.TO, DICT_INITSIZE) - #d.num_items = 0 -- defaults + d.num_items = 0 d.num_pristine_entries = DICT_INITSIZE return d @@ -562,7 +562,7 @@ n *= 2 d = lltype.malloc(DICT) d.entries = lltype.malloc(DICT.entries.TO, n) - #d.num_items = 0 -- defaults + d.num_items = 0 d.num_pristine_entries = DICT_INITSIZE return d Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py Wed Aug 23 13:17:08 2006 @@ -48,6 +48,7 @@ p = malloc(STR, len(value)) for i in range(len(value)): p.chars[i] = value[i] + p.hash = 0 self.ll.ll_strhash(p) # precompute the hash CONST_STR_CACHE[value] = p return p Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_lltype.py Wed Aug 23 13:17:08 2006 @@ -10,13 +10,12 @@ s0 = malloc(S0) print s0 assert typeOf(s0) == Ptr(S0) - assert s0.a == 0 - assert s0.b == 0 - assert typeOf(s0.a) == Signed + py.test.raises(UninitializedMemoryAccess, "s0.a") s0.a = 1 s0.b = s0.a assert s0.a == 1 assert s0.b == 1 + assert typeOf(s0.a) == Signed # simple array Ar = GcArray(('v', Signed)) x = malloc(Ar,0) @@ -26,11 +25,10 @@ print x assert typeOf(x) == Ptr(Ar) assert isweak(x[0], Ar.OF) - assert typeOf(x[0].v) == Signed - assert x[0].v == 0 x[0].v = 1 x[1].v = 2 x[2].v = 3 + assert typeOf(x[0].v) == Signed assert [x[z].v for z in range(3)] == [1, 2, 3] # def define_list(T): @@ -94,10 +92,12 @@ S1 = GcStruct("s1", ('a', Signed), ('rest', Array(('v', Signed)))) py.test.raises(TypeError, "malloc(S1)") s1 = malloc(S1, 4) + s1.a = 0 assert s1.a == 0 assert isweak(s1.rest, S1.rest) assert len(s1.rest) == 4 assert isweak(s1.rest[0], S1.rest.OF) + s1.rest[0].v = 0 assert typeOf(s1.rest[0].v) == Signed assert s1.rest[0].v == 0 py.test.raises(IndexError, "s1.rest[4]") @@ -206,7 +206,6 @@ S1 = GcStruct("s1", ('sub1', S2), ('sub2', S2), ('tail', Array(('e', Signed)))) p1 = malloc(S1, 1) p2 = p1.sub2 - assert p2.a == 0 p3 = p1.tail p3[0].e = 1 assert p3[0].e == 1 @@ -220,8 +219,6 @@ A1 = GcArray(('v', Signed)) p1 = malloc(A1, 10) p1[5].v=3 - assert p1[0].v == 0 - assert p1[9].v == 0 assert p1[5].v == 3 p1_5 = p1[5] del p1 @@ -327,9 +324,7 @@ As = GcArray(Signed) a = malloc(As, 3) assert typeOf(a) == Ptr(As) - assert a[0] == 0 - assert a[1] == 0 - assert a[2] == 0 + py.test.raises(UninitializedMemoryAccess, "a[0]") a[1] = 3 assert a[1] == 3 S = GcStruct('s', ('x', Signed)) @@ -398,10 +393,13 @@ S = GcStruct('s', ('x', Signed)) attachRuntimeTypeInfo(S) s = malloc(S) + s.x = 0 assert runtime_type_info(s) == getRuntimeTypeInfo(S) S1 = GcStruct('s1', ('sub', S), ('x', Signed)) attachRuntimeTypeInfo(S1) s1 = malloc(S1) + s1.sub.x = 0 + s1.x = 0 assert runtime_type_info(s1) == getRuntimeTypeInfo(S1) assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1) assert runtime_type_info(cast_pointer(Ptr(S), s1)) == getRuntimeTypeInfo(S1) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_picklelltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_picklelltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_picklelltype.py Wed Aug 23 13:17:08 2006 @@ -92,6 +92,9 @@ S1 = GcStruct("s1", ('a', Signed), ('rest', Array(('v', Signed)))) py.test.raises(TypeError, "malloc(S1)") s1 = malloc(S1, 4) + s1.a = 0 + for i in range(4): + s1.rest[i].v = 0 p_s1 = pickle.dumps(s1) r_s1 = pickle.loads(p_s1) assert r_s1.a == 0 @@ -166,6 +169,8 @@ def test_best_effort_gced_parent_for_arrays(): A1 = GcArray(('v', Signed)) p1 = malloc(A1, 10) + for i in range(10): + p1[i].v = 0 p1[5].v = 3 p1_5 = p1[5] r_p1_5 = pickle.loads(pickle.dumps(p1_5)) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py Wed Aug 23 13:17:08 2006 @@ -30,7 +30,7 @@ class Class(OOType): - def _defl(self): + def _defl(self, example=False): return nullruntimeclass Class = Class() @@ -66,7 +66,7 @@ def __hash__(self): return object.__hash__(self) - def _defl(self): + def _defl(self, example=False): return self._null def _example(self): return new(self) @@ -92,7 +92,7 @@ if isinstance(defn, Meth): raise TypeError("Attempting to store method in field") - fields[name] = (defn, defn._defl()) + fields[name] = (defn, defn._defl(example=True)) else: ootype, default = defn @@ -195,7 +195,7 @@ _retval = self.RESULT._example() return _static_meth(self, _callable=lambda *args: _retval) - def _defl(self): + def _defl(self, example=False): return null(self) def __repr__(self): @@ -219,7 +219,7 @@ def _example(self): return new(self) - def _defl(self): + def _defl(self, example=False): return self._null def _get_interp_class(self): @@ -233,10 +233,10 @@ def __init__(self, fields): self._fields = frozendict() for name, ITEMTYPE in fields.items(): - self._fields[name] = ITEMTYPE, ITEMTYPE._defl() + self._fields[name] = ITEMTYPE, ITEMTYPE._defl(example=True) self._null = _null_record(self) - def _defl(self): + def _defl(self, example=False): return self._null def _get_interp_class(self): @@ -321,7 +321,7 @@ return BuiltinADTType._enforce(self, value) # TODO: should it return _null or ''? - def _defl(self): + def _defl(self, example=False): return make_string("") def _example(self): return self._defl() @@ -344,7 +344,7 @@ }) self._setup_methods({}) - def _defl(self): + def _defl(self, example=False): return self._null def _get_interp_class(self): @@ -430,7 +430,7 @@ ITEMTYPE = self._specialize_type(self._ITEMTYPE, generic_types) return self.__class__(ITEMTYPE) - def _defl(self): + def _defl(self, example=False): return self._null def _set_itemtype(self, ITEMTYPE): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/rcpy.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/rcpy.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/rcpy.py Wed Aug 23 13:17:08 2006 @@ -255,7 +255,8 @@ # build the PyTypeObject structure pytypeobj = lltype.malloc(PY_TYPE_OBJECT, flavor='cpy', - extra_args=(typetype,)) + extra_args=(typetype,), + zero=True) name = cpytype.name T = lltype.FixedSizeArray(lltype.Char, len(name)+1) p = lltype.malloc(T, immortal=True) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_llinterp.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_llinterp.py Wed Aug 23 13:17:08 2006 @@ -1,4 +1,3 @@ - import py from pypy.rpython.lltypesystem.lltype import typeOf, pyobjectptr, Ptr, PyObject, Void from pypy.rpython.lltypesystem.lloperation import llop From wanja at codespeak.net Wed Aug 23 13:24:06 2006 From: wanja at codespeak.net (wanja at codespeak.net) Date: Wed, 23 Aug 2006 13:24:06 +0200 (CEST) Subject: [pypy-svn] r31529 - pypy/dist/pypy/doc/image Message-ID: <20060823112406.5B8BA10036@code0.codespeak.net> Author: wanja Date: Wed Aug 23 13:23:48 2006 New Revision: 31529 Added: pypy/dist/pypy/doc/image/bram.jpg (contents, props changed) pypy/dist/pypy/doc/image/guido.jpg (contents, props changed) Log: added thumbnails for torrents of bram's interview and guido's keynote speech at pycon Added: pypy/dist/pypy/doc/image/bram.jpg ============================================================================== Binary file. No diff available. Added: pypy/dist/pypy/doc/image/guido.jpg ============================================================================== Binary file. No diff available. From bea at codespeak.net Wed Aug 23 13:27:59 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 13:27:59 +0200 (CEST) Subject: [pypy-svn] r31530 - pypy/extradoc/sprintinfo Message-ID: <20060823112759.08E5110036@code0.codespeak.net> Author: bea Date: Wed Aug 23 13:27:58 2006 New Revision: 31530 Added: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt Log: creating a txt file from various pypy-dev postings (for the summary sprint report page) Added: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/ep2005-sprintreport.txt Wed Aug 23 13:27:58 2006 @@ -0,0 +1,136 @@ +Nothing official, but as I'm stting here waiting for the others to +arrive, I thought I'd wirte a little bit about what we did yesterday +(all my own opninions and recollections, corrections welcome :-). + +The people present and hacking were: Armin, Samuele, Christian, +Holger, Michael, Anders (C) and Anders (L). + +We started by writing: + +http://codespeak.net/svn/pypy/extradoc/sprintinfo/pre-ep2005-planning.txt + +(which actually started as a mail Armin sent to pypy-dev a week or so +ago, and was edited during the day to reflect progress). + +Then in a pretty informal way we paired or tripled up and got +cracking. Generally speaking, it was encouraging how quickly everyone +became productive, it says good things about the current design of +codebase (and also indicates how much work there is left to do...). + +Almost all of the work happened at the rtyper level: + +http://codespeak.net/pypy/index.cgi?doc/translation.html#the-rpython-typer + +that sits between the annotator and the language backends. + +Samuele and Michael mostly worked on string operations in rpython, +implementing conversions from ints to strings and limited string +formatting operations, did a couple of easy builtins and worked out +which were going to be hard. + +Anders (L) and Christian worked mostly on rpython lists and tuples, +and also did string slicing. + +Anders (C), Holger and Armin mostly worked on rpython dictionaries +(the ones that will hold keyword arguments during argument processing, +among a few other things), and nearly finished it. + +Right, the others are hear now, so it's time to start hacking again... + +Cheers, +mwh + +Yesterday was less obviously productive than day 1, possibly because +you now have to go quite a lot higher up the tree to find the lowest +fruit by now... + +Arre and Holger finished off string-keyed dictionary stuff. + +Anders and Samuele did the list builtin. + +Armin and Michael did isinstance. + +Michael and Armin wrote some grotty float parsing code and removed the +only use of the float builtin on a string (if anyone has some nice and +less numerically naive float parsing code or wants to write some, feel +free -- it's a fairly self contained task). + +Christian worked on reducing the use of import * in rpython/ (though +really we should have less 'from module import thing ... thing' and +more 'import module ... module.thing', but this still an improvement). + +Holger and Armin removed some of the uses of dictionaries in the +to-be-translated code. + +Michael and Samuele implemented a very limited str for instances. + +Arre and Anders implemented a 'unicode character' for the rtyper and c +backend. + +Michael and Samuele investigated issues preventing the translation of +demo/bpnn.py. + +Holger and Armin worked on functions like Cache.getorbuild which need +to be treated specially by the annotator and rtyper (because the +'build' part is not allowed to happen at runtime). + +Then Michael, Samuele and the two Anders went to a midsomar party, and +at 1pm the day after, Samuele hasn't been seen since... + +Cheers, +mwh +Yesterday was another good day at the pre-EP sprint. + +We mostly started by polishing stuff from the day before -- unicode +characters, pre-built caches. + +Holger and Michael removed by writing boring code a use of +specialize:memo we weren't feeling intelligent enought to support in +the rtyper. + +Armin and Anders made yet another pass through the list of partially +supported 'builtins'[1] and made some decisions about whether support +should be removed, finished or ignored for now. + +The we had bit of a planning session and decided that (Christian and +Arre) and (Anders and Armin) would pair on supporting builtins and +Holger, Samuele and Michael would work on translator issues. + +Holger and Michael implemented for the C translator the few remaining +unsupported float operations the rtyper could emit. + +Arre and Christian implemented a few more operations on rdicts. + +Armin and Anders worked on the list of builtins in the usual fashion: +sometimes removing code now deemed to be not rpython, sometimes by +removing special-casing code that was no longer necessary and +sometimes by adding code to the rtyper. + +Holger, Michael and Samuele attempted to translate demo/bpnn.py and +fixed the problems they ran into; this included obscure behaviour when +the rtyper hit code calling a statically known bound method of a +Constant, the same when the Constant was of a class that was only seen +by the RTyper as a class of such a Constant and not elsewhere. + +Holger and Armin refactored some of the rtyper code in the area of +equality. + +Michael and Samuele fixed some broken code in the c translator in the +area of calling C functions not implemented by us. + +Anders with help from Armin implemented is_true for PBC and fixed some +bugs. + +Holger and Armin implemented yet another sort of dictionary: +'constant' dicts that are built at initialization time and only +queried thereafter. + +Then we called it a day and went for dinner. + +Today, we plan to carry on the above and do a little planning for the +post-EP sprint (which I won't be at, as I'll be involved in the +much-more-entertaining task of moving house) and maybe, maybe, writing +a talk for that conference we hear is happening quite soon. + +Cheers, +mwh \ No newline at end of file From bea at codespeak.net Wed Aug 23 13:28:34 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 13:28:34 +0200 (CEST) Subject: [pypy-svn] r31531 - pypy/extradoc/sprintinfo Message-ID: <20060823112834.33D4310036@code0.codespeak.net> Author: bea Date: Wed Aug 23 13:28:33 2006 New Revision: 31531 Modified: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt (contents, props changed) Log: eol style Modified: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt ============================================================================== --- pypy/extradoc/sprintinfo/ep2005-sprintreport.txt (original) +++ pypy/extradoc/sprintinfo/ep2005-sprintreport.txt Wed Aug 23 13:28:33 2006 @@ -1,136 +1,136 @@ -Nothing official, but as I'm stting here waiting for the others to -arrive, I thought I'd wirte a little bit about what we did yesterday -(all my own opninions and recollections, corrections welcome :-). - -The people present and hacking were: Armin, Samuele, Christian, -Holger, Michael, Anders (C) and Anders (L). - -We started by writing: - -http://codespeak.net/svn/pypy/extradoc/sprintinfo/pre-ep2005-planning.txt - -(which actually started as a mail Armin sent to pypy-dev a week or so -ago, and was edited during the day to reflect progress). - -Then in a pretty informal way we paired or tripled up and got -cracking. Generally speaking, it was encouraging how quickly everyone -became productive, it says good things about the current design of -codebase (and also indicates how much work there is left to do...). - -Almost all of the work happened at the rtyper level: - -http://codespeak.net/pypy/index.cgi?doc/translation.html#the-rpython-typer - -that sits between the annotator and the language backends. - -Samuele and Michael mostly worked on string operations in rpython, -implementing conversions from ints to strings and limited string -formatting operations, did a couple of easy builtins and worked out -which were going to be hard. - -Anders (L) and Christian worked mostly on rpython lists and tuples, -and also did string slicing. - -Anders (C), Holger and Armin mostly worked on rpython dictionaries -(the ones that will hold keyword arguments during argument processing, -among a few other things), and nearly finished it. - -Right, the others are hear now, so it's time to start hacking again... - -Cheers, -mwh - -Yesterday was less obviously productive than day 1, possibly because -you now have to go quite a lot higher up the tree to find the lowest -fruit by now... - -Arre and Holger finished off string-keyed dictionary stuff. - -Anders and Samuele did the list builtin. - -Armin and Michael did isinstance. - -Michael and Armin wrote some grotty float parsing code and removed the -only use of the float builtin on a string (if anyone has some nice and -less numerically naive float parsing code or wants to write some, feel -free -- it's a fairly self contained task). - -Christian worked on reducing the use of import * in rpython/ (though -really we should have less 'from module import thing ... thing' and -more 'import module ... module.thing', but this still an improvement). - -Holger and Armin removed some of the uses of dictionaries in the -to-be-translated code. - -Michael and Samuele implemented a very limited str for instances. - -Arre and Anders implemented a 'unicode character' for the rtyper and c -backend. - -Michael and Samuele investigated issues preventing the translation of -demo/bpnn.py. - -Holger and Armin worked on functions like Cache.getorbuild which need -to be treated specially by the annotator and rtyper (because the -'build' part is not allowed to happen at runtime). - -Then Michael, Samuele and the two Anders went to a midsomar party, and -at 1pm the day after, Samuele hasn't been seen since... - -Cheers, -mwh -Yesterday was another good day at the pre-EP sprint. - -We mostly started by polishing stuff from the day before -- unicode -characters, pre-built caches. - -Holger and Michael removed by writing boring code a use of -specialize:memo we weren't feeling intelligent enought to support in -the rtyper. - -Armin and Anders made yet another pass through the list of partially -supported 'builtins'[1] and made some decisions about whether support -should be removed, finished or ignored for now. - -The we had bit of a planning session and decided that (Christian and -Arre) and (Anders and Armin) would pair on supporting builtins and -Holger, Samuele and Michael would work on translator issues. - -Holger and Michael implemented for the C translator the few remaining -unsupported float operations the rtyper could emit. - -Arre and Christian implemented a few more operations on rdicts. - -Armin and Anders worked on the list of builtins in the usual fashion: -sometimes removing code now deemed to be not rpython, sometimes by -removing special-casing code that was no longer necessary and -sometimes by adding code to the rtyper. - -Holger, Michael and Samuele attempted to translate demo/bpnn.py and -fixed the problems they ran into; this included obscure behaviour when -the rtyper hit code calling a statically known bound method of a -Constant, the same when the Constant was of a class that was only seen -by the RTyper as a class of such a Constant and not elsewhere. - -Holger and Armin refactored some of the rtyper code in the area of -equality. - -Michael and Samuele fixed some broken code in the c translator in the -area of calling C functions not implemented by us. - -Anders with help from Armin implemented is_true for PBC and fixed some -bugs. - -Holger and Armin implemented yet another sort of dictionary: -'constant' dicts that are built at initialization time and only -queried thereafter. - -Then we called it a day and went for dinner. - -Today, we plan to carry on the above and do a little planning for the -post-EP sprint (which I won't be at, as I'll be involved in the -much-more-entertaining task of moving house) and maybe, maybe, writing -a talk for that conference we hear is happening quite soon. - -Cheers, +Nothing official, but as I'm stting here waiting for the others to +arrive, I thought I'd wirte a little bit about what we did yesterday +(all my own opninions and recollections, corrections welcome :-). + +The people present and hacking were: Armin, Samuele, Christian, +Holger, Michael, Anders (C) and Anders (L). + +We started by writing: + +http://codespeak.net/svn/pypy/extradoc/sprintinfo/pre-ep2005-planning.txt + +(which actually started as a mail Armin sent to pypy-dev a week or so +ago, and was edited during the day to reflect progress). + +Then in a pretty informal way we paired or tripled up and got +cracking. Generally speaking, it was encouraging how quickly everyone +became productive, it says good things about the current design of +codebase (and also indicates how much work there is left to do...). + +Almost all of the work happened at the rtyper level: + +http://codespeak.net/pypy/index.cgi?doc/translation.html#the-rpython-typer + +that sits between the annotator and the language backends. + +Samuele and Michael mostly worked on string operations in rpython, +implementing conversions from ints to strings and limited string +formatting operations, did a couple of easy builtins and worked out +which were going to be hard. + +Anders (L) and Christian worked mostly on rpython lists and tuples, +and also did string slicing. + +Anders (C), Holger and Armin mostly worked on rpython dictionaries +(the ones that will hold keyword arguments during argument processing, +among a few other things), and nearly finished it. + +Right, the others are hear now, so it's time to start hacking again... + +Cheers, +mwh + +Yesterday was less obviously productive than day 1, possibly because +you now have to go quite a lot higher up the tree to find the lowest +fruit by now... + +Arre and Holger finished off string-keyed dictionary stuff. + +Anders and Samuele did the list builtin. + +Armin and Michael did isinstance. + +Michael and Armin wrote some grotty float parsing code and removed the +only use of the float builtin on a string (if anyone has some nice and +less numerically naive float parsing code or wants to write some, feel +free -- it's a fairly self contained task). + +Christian worked on reducing the use of import * in rpython/ (though +really we should have less 'from module import thing ... thing' and +more 'import module ... module.thing', but this still an improvement). + +Holger and Armin removed some of the uses of dictionaries in the +to-be-translated code. + +Michael and Samuele implemented a very limited str for instances. + +Arre and Anders implemented a 'unicode character' for the rtyper and c +backend. + +Michael and Samuele investigated issues preventing the translation of +demo/bpnn.py. + +Holger and Armin worked on functions like Cache.getorbuild which need +to be treated specially by the annotator and rtyper (because the +'build' part is not allowed to happen at runtime). + +Then Michael, Samuele and the two Anders went to a midsomar party, and +at 1pm the day after, Samuele hasn't been seen since... + +Cheers, +mwh +Yesterday was another good day at the pre-EP sprint. + +We mostly started by polishing stuff from the day before -- unicode +characters, pre-built caches. + +Holger and Michael removed by writing boring code a use of +specialize:memo we weren't feeling intelligent enought to support in +the rtyper. + +Armin and Anders made yet another pass through the list of partially +supported 'builtins'[1] and made some decisions about whether support +should be removed, finished or ignored for now. + +The we had bit of a planning session and decided that (Christian and +Arre) and (Anders and Armin) would pair on supporting builtins and +Holger, Samuele and Michael would work on translator issues. + +Holger and Michael implemented for the C translator the few remaining +unsupported float operations the rtyper could emit. + +Arre and Christian implemented a few more operations on rdicts. + +Armin and Anders worked on the list of builtins in the usual fashion: +sometimes removing code now deemed to be not rpython, sometimes by +removing special-casing code that was no longer necessary and +sometimes by adding code to the rtyper. + +Holger, Michael and Samuele attempted to translate demo/bpnn.py and +fixed the problems they ran into; this included obscure behaviour when +the rtyper hit code calling a statically known bound method of a +Constant, the same when the Constant was of a class that was only seen +by the RTyper as a class of such a Constant and not elsewhere. + +Holger and Armin refactored some of the rtyper code in the area of +equality. + +Michael and Samuele fixed some broken code in the c translator in the +area of calling C functions not implemented by us. + +Anders with help from Armin implemented is_true for PBC and fixed some +bugs. + +Holger and Armin implemented yet another sort of dictionary: +'constant' dicts that are built at initialization time and only +queried thereafter. + +Then we called it a day and went for dinner. + +Today, we plan to carry on the above and do a little planning for the +post-EP sprint (which I won't be at, as I'll be involved in the +much-more-entertaining task of moving house) and maybe, maybe, writing +a talk for that conference we hear is happening quite soon. + +Cheers, mwh \ No newline at end of file From bea at codespeak.net Wed Aug 23 13:36:25 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 13:36:25 +0200 (CEST) Subject: [pypy-svn] r31535 - pypy/extradoc/sprintinfo Message-ID: <20060823113625.1F63D10036@code0.codespeak.net> Author: bea Date: Wed Aug 23 13:36:21 2006 New Revision: 31535 Added: pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt (contents, props changed) Log: again combining separate pypy-dev postings into one txt file Added: pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt Wed Aug 23 13:36:21 2006 @@ -0,0 +1,91 @@ +Just some quick (and probably confused) notes about went on at the +Hildesheim sprint day 1. Corrections welcome. +As in Gothenburg we tried to write an evolving file that listed all the +tasks: + +http://codespeak.net/svn/pypy/extradoc/sprintinfo/hildesheim2-planning.txt + +somehow we didn't get very far and decided that we just would all work +on rtyper issues. The following people are here: Hogler, Armin, Samuele, +Christian, Richard, Carl Friedrich. + +Holger and Richard paired and started removing some amount of faking +in PyPy. They did this by removing the usage of file as well as +stdout/stdin in the PyPy interpreter and resorting to the os primitives +instead, which can be more easily translated to C. + +Armin and Christian worked on various rtyper issues. They worked on +unicode, fixed hashing for numerical types, and worked on marshal. + +Samuele and me worked on external functions. We cleaned up some of the +hooks that were there before the current external function mechanism and +implemented some math and os function for the C backend so that you can +now actually use some of the primitive file operations like fdopen, +stat, write... without using the CPython API. + +Cheers, + +Carl Friedrich + +Hi folks, + +the end of the world is near ... +following up on Carl's day 1 report here are the main results +of what happened up until now at the hildesheim2 sprint (we are +in the break day currently which we spent in the city, at +Wing-Tsun lessons or at family meetings). + +The main results so far are: + +- any faking can be removed now with + + python2.4 py.py --file --nofakedmodules + + so on the trunk you should see no 'faking' messages + during startup anymore. It will take some time + mostly because of the 'file' implementation living + at app-level. Note that we have some problems + running on top of Python2.3 at the moment and + also some more problems with PYC file support. + +- Annotation can now complete without + resorting to any SomeObjects anywhere (!!) + +- RTyping has only 2 Errors remaining. The next bigger + problem is getting the compiler package integrated + and bootstrapped properly. + +- note that we are not working from the snapshot + but the trunk currently. After the sprint + we probably return to the snapshot approach + for translation. + +If you want to see more detailed daily-updated progress +reports i suggest to read and follow the commits on + +http://codespeak.net/pypy/index.cgi?extradoc/sprintinfo/hildesheim2-planning.txt +Moreover, i shot a few pictures which i put here: + + http://codespeak.net/~hpk/hildesheim2-sprint-www + +and here is a picture of the surrounding place +(Trillke-Gut, my living&working base): + + http://www.trillke.net/images/HomePagePictureSmall.jpg + +cheers, + + holger + + +P.S.: It appears that we will have to continue our + investigations regarding some of the current problems on + the astral plane. Armin and Carl has already spend some + time on assembler level yesterday to hunt down the + weirdest TCC interactions. But there are other confusing + events as well. um, we watched 'the nine lives of + Thomas Katz' yesterday night, btw. + Anyway, we are still taking bets if the first C-generated + annotated RTyped PyPy is to produce a segmentation fault + in the C-compiler or in the resulting binary. + From bea at codespeak.net Wed Aug 23 13:43:31 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 13:43:31 +0200 (CEST) Subject: [pypy-svn] r31537 - pypy/extradoc/sprintinfo/gothenburg-2005 Message-ID: <20060823114331.BC19A10036@code0.codespeak.net> Author: bea Date: Wed Aug 23 13:43:29 2006 New Revision: 31537 Added: pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt (contents, props changed) Log: again pypy-dev postings into one txt file Added: pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt Wed Aug 23 13:43:29 2006 @@ -0,0 +1,163 @@ +Hi PyPy-dev! + +Finally we have found some time to write a sprint report on what is +day three or day five of the sprint depending on how you count. + +The reason for the uncertainty is that on Monday us lifers^Wfulltimers +gathered for two days of EU-report writing ahead of our review meeting +in January. We'll spare you the details, but as most of the technical +reports are also part of the documention on our website, it's worth +mentioning that two documents: + + http://codespeak.net/pypy/dist/pypy/doc/low-level-encapsulation.html + http://codespeak.net/pypy/dist/pypy/doc/translation-aspects.html + +received significant updates in these days. The former is a more +friendly read, start with that one. Consensus opinion is that two +days of proof-reading and generally attempting to write nice prose is +even more exhausting than coding. + +It was thus with some relief that on Wednesday morning we met to plan +our programming tasks. The task that most combined enormity and +urgency was starting work on the the Just-In-Time compiler. Samuele, +Armin, Eric, Carl and Arre all worked on this. Eric and Arre +implemented yet another interpreter for a very simple toy language as +we needed something that would be simpler to apply a JIT to than the +PyPy behemoth and then moved on to looking at pypy-c's performance. + +Carl, Armin and Samuele began writing a partial specializer for +low-level graphs, which is to say a way to rewrite a graph in the +knowledge that certain values are constants. This is related to the +JIT effort because a JIT will partially specialize the interpreter +main-loop for a particular code object. The combination of the two +sub-tasks will allow us to experiment with ways of applying these +specialization techniques. + +Anders L and Nik repeatedly bashed their heads on the brick wall of +issues surrounding a working socket module, and are making good +progress although running a useful program is still a way off. The +usual platform dependent horrors are, predictably, horrific. + +Michael and the sole sprint newcomer this time, Johan, implemented +support in the translation toolchain for C's "long long" type which +involved taking Johan on a whirlwind tour of the annotator, rtyper and +C backend. By Thursday lunchtime they had a translated pypy that +supported returning a long long value to app-level from exactly one +external function (os.lseek) and declared success. + +Richard and Christian worked on exposing the raw stackless facilities +that have existed for some time in a useful way. At this point they +have written a 'tasklet' module that is usable by RPython which is +probably best considered as an experiment on ways to write a module +that can expose stackless features to application level. + +Adrien and Ludovic worked on increasing the flexibility of our parser +and compiler, in particular aiming to allow modification of the syntax +at runtime. Their first success was to allow syntax to be *removed* +from PyPy: a valuable tool for making web frameworks harder to write:: + + Clearly, the only way to cut down on the number of Web frameworks + is to make it much harder to write them. If Guido were really going + to help resolve this Web frameworks mess, he'd make Python a much + uglier and less powerful language. I can't help but feel that + that's the wrong way to go ;). + + (from Titus Brown's Advogato diary of 7 Dec 2005: + http://www.advogato.org/person/titus/diary.html?start=134) + +But they have also now allowed the definition of rules that can modify +the syntax tree on the fly. + +On Friday morning a task group reshuffle gave Michael a unique +opportunity to work with Armin on the 'JIT' and Carl and Johan became +a new '__del__' taskforce. By the end of the day, '__del__' was +supported in the genc backend when using the reference counting +garbage collector. This also involved changing details at all levels +of the translation process, so by now Johan has seen a very little bit +of very large amounts of PyPy... + +Arre and Eric (with some crucial hints from Richard) had a +successful hunt for performance problems: they changed +about 5 lines of code affecting the way we use the Boehm GC +which resulted in a remarkable 30-40% speed up in pystone after +translation. Now if they can change 10 lines to get an 80% +improvement we _will_ be impressed :) + +That's all for now. We'll write a report on the last two days, err, +sometime. :) + +Cheers, + +Michael & Carl Friedrich + +So, the sprint is over, we are on a ferry but we *still* haven't +escaped the internet... + +Christian and Richard spent the last couple of days thinking hard +about the many possible ways of exposing stackless features to +application-level and by the end of the sprint had pretty much +considered them all. This means they will have no choice but to write +some code for the mixed module soon :) + +Armin and his team of helpers (well, mostly Michael and Samuele to be +honest) continued to work on JIT-related things, and continued to +manufacture both working code and extremely strange bugs. Eventually +the Test Driven Development style was halted for a quick but useful +Cafe-cake-and-thinking-hard Driven Development moment (also attended +by Carl). By the end of the sprint there was support in the abstract +interpreter for "virtual structures" and "virtual arrays" which are +the abstract interpreter's way of handling objects that are usually +allocated on the memory heap but are sufficiently known ahead of time +for actual allocation to be unnecessary. Now that probably didn't +make much sense, so here's an example: + + def f(x): + l = [x] + return l[0] + +When CPython executes this code it allocates a list, places x into it, +extracts it again and throws the list away. When the abstract +interpeter sees the statement "l = [x]" it just records the +information that "l is a list containing x" so when it sees "l[0]" it +already knows what this is -- just "x". Then as nothing in the +function ever needed l as a list, it just evaporates. + +Anders L and Nik continued on the socket module and managed to write a +test that contained a simple server and client that could successfully +talk to each other (after fighting some mysterious problems that +with processes that refused to die). + +Carl continued his work on __del__, implementing support for it in the +C backend when using the Boehm garbage collector which had the minor +disadvantage of slowing pypy-c down by a factor of ten, as every +instance of a user-defined class was registered with the GC as having +a finalizer. This is apparently not something that the Boehm GC +appreciates, so on Sunday Carl and Samuele implemented a different +strategy. Now only instances of user-defined classes that actually +define __del__ (at class definition time, no sneaky cls.__del__ = ... +nonsense) get registered as needing finalization. Carl was also +awarded his first "obscure Python bug" medal for making CPython dump +core when he tried to test a hacky way to implement weakrefs (now SF +bug #1377858). + +Arre and Eric continued their optimization drive and implemented a +'fastcall' path for both functions and methods which accelerates the +common case of calling a Python function or method with the correct, +fixed number of arguments. This improved the results of a simple +function-call benchmark by a factor of about two. Result! + +A notable feature of this sprint is that Armin and Christian were +implementing things very much like other things they had implemented +before, painfully, in C -- namely stackless and psyco -- again, but +this time in Python, and much more enjoyably :) Even more than this, +they both managed to work their way through designs and ideas that had +taken months to sort through the first time in days or even hours. +We'd love to attribute all this to the magical qualities of Python but +practice probably counts for something too :) + +So another sprint ends, and a productive one at that. As usual, we +all need to sleep for a week, or at least until pypy-sync on +Thursday... + +Cheers, +mwh & Carl Friedrich \ No newline at end of file From ac at codespeak.net Wed Aug 23 15:01:22 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Wed, 23 Aug 2006 15:01:22 +0200 (CEST) Subject: [pypy-svn] r31543 - pypy/dist/pypy/rpython/memory Message-ID: <20060823130122.C6C0010060@code0.codespeak.net> Author: ac Date: Wed Aug 23 15:01:21 2006 New Revision: 31543 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: (mwh, arre) Move a test before a call from inside it. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Wed Aug 23 15:01:21 2006 @@ -273,7 +273,8 @@ typeid = hdr.typeid >> 1 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header - self.add_reachable_to_stack(obj, objects) + if not hdr.typeid & 1: + self.add_reachable_to_stack(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) if self.is_varsize(typeid): @@ -287,11 +288,11 @@ # stack until the stack is empty while objects.non_empty(): #mark curr = objects.pop() - self.add_reachable_to_stack(curr, objects) gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) if hdr.typeid & 1: continue + self.add_reachable_to_stack(curr, objects) hdr.typeid = hdr.typeid | 1 objects.delete() # also mark self.curpool @@ -420,8 +421,6 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - return typeid = hdr.typeid >> 1 offsets = self.offsets_to_gc_pointers(typeid) i = 0 From bea at codespeak.net Wed Aug 23 15:14:36 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Wed, 23 Aug 2006 15:14:36 +0200 (CEST) Subject: [pypy-svn] r31545 - pypy/dist/pypy/doc Message-ID: <20060823131436.4C36C10060@code0.codespeak.net> Author: bea Date: Wed Aug 23 15:14:32 2006 New Revision: 31545 Added: pypy/dist/pypy/doc/sprint-reports.txt (contents, props changed) Log: a summary page of all sprint reports being produced 2003-2006, to be put into the metadocumentation page Added: pypy/dist/pypy/doc/sprint-reports.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/sprint-reports.txt Wed Aug 23 15:14:32 2006 @@ -0,0 +1,52 @@ +Sprint reports from PyPy sprints 2003-2006 +============================== + +Here are links to sprint reports from various sprints in the PyPy project, +an informal summary of features being worked on and the results of that work. +A good summary of the progress over the years.....enjoy! + + `Hildesheim (Feb 2003)`_ + Gothenburg (May 2003) + `LovainLaNeuve (June 2003)`_ + `Berlin (Sept 2003)`_ + `Amsterdam (Dec 2003)`_ + Europython + /Gothenburg (June 2004) + `Vilnius (Nov 2004)`_ + `Leysin (Jan 2005)`_ + `PyCon + /Washington (March 2005)`_ + `Europython + /Gothenburg (June 2005)`_ + `Hildesheim (July 2005)`_ + `Heidelberg (Aug 2005)`_ + `Paris (Oct 2005)`_ + `Gothenburg (Dec 2005)`_ + `Mallorca (Jan 2006)`_ + PyCon/Dallas (Feb 2006) + `LouvainLaNeuve (March 2006)`_ + Leysin (April 2006) + `Tokyo (April 2006)`_ + `D?sseldorf (June 2006)`_ + `Europython/ + Geneva (July 2006)`_ + Limerick (Aug 2006) + + .. _Hildesheim (Feb 2003): https://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html + .. _LovainLaNeuve (June 2003): https://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt + .. _Berlin (Sept 2003): https://codespeak.net/pypy/extradoc/sprintinfo/BerlinReport.txt + .. _Amsterdam (Dec 2003): https://codespeak.net/pypy/extradoc/sprintinfo/AmsterdamReport.txt + .. _Leysin (Jan 2005): https://codespeak.net/pypy/extradoc/sprintinfo/LeysinReport.txt + .. _PyCon/Washington (March 2005): https://codespeak.net/pypy/extradoc/sprintinfo/pycon_sprint_report.txt + .. _Europython/Gothenburg (June 2005): https://codespeak.net/pypy/extradoc/sprintinfo/ep2005-sprintreport.txt + .. _Hildesheim (July 2005): https://codespeak.net/pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt + .. _Heidelberg (Aug 2005): https://codespeak.net/pypy/extradoc/sprintinfo/Heidelberg-report.txt + .. _Paris (Oct 2005): https://codespeak.net/pypy/extradoc/sprintinfo/paris/paris-report.txt + .. _Gothenburg (Dec 2005): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt + .. _Mallorca (Jan 2006): https://codespeak.net/pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt + .. _LovainLaNeuve (March 2006): https://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt + .. _Tokyo (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/tokyo/sprint-report.txt + .. _D?sseldorf (June 2006): https://codespeak.net/pypy/extradoc/sprintinfo/ddorf2006/report1.txt + .. _Europython/Geneva (July 2006): https://codespeak.net/pypy/extradoc/sprintinfo/post-ep2006/report.txt + + \ No newline at end of file From mwh at codespeak.net Wed Aug 23 15:23:58 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 15:23:58 +0200 (CEST) Subject: [pypy-svn] r31549 - pypy/dist/pypy/doc Message-ID: <20060823132358.955F910060@code0.codespeak.net> Author: mwh Date: Wed Aug 23 15:23:52 2006 New Revision: 31549 Modified: pypy/dist/pypy/doc/index.txt pypy/dist/pypy/doc/sprint-reports.txt pypy/dist/pypy/doc/translation.txt Log: (bea, mwh) reST fixes + a link from the index page. Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Wed Aug 23 15:23:52 2006 @@ -16,7 +16,10 @@ `coding guide`_ helps you to write code for PyPy. -`development methodology`_ describe our sprint-driven approach. +`development methodology`_ describe our sprint-driven approach. + +`sprint reports`_ lists reports written at most of our sprints, from +2003 to the present. `talks and related projects`_ lists presentations and related projects. @@ -113,6 +116,7 @@ .. _`PyPy video documentation`: video-index.html .. _parser: parser.html .. _`development methodology`: dev_method.html +.. _`sprint reports`: sprint-reports.html .. _`talks and related projects`: extradoc.html .. _`license`: ../../LICENSE .. _`compliance test status`: http://codespeak.net/~hpk/pypy-testresult/ Modified: pypy/dist/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/dist/pypy/doc/sprint-reports.txt (original) +++ pypy/dist/pypy/doc/sprint-reports.txt Wed Aug 23 15:23:52 2006 @@ -1,36 +1,32 @@ Sprint reports from PyPy sprints 2003-2006 -============================== +========================================== Here are links to sprint reports from various sprints in the PyPy project, an informal summary of features being worked on and the results of that work. A good summary of the progress over the years.....enjoy! - `Hildesheim (Feb 2003)`_ - Gothenburg (May 2003) - `LovainLaNeuve (June 2003)`_ - `Berlin (Sept 2003)`_ - `Amsterdam (Dec 2003)`_ - Europython - /Gothenburg (June 2004) - `Vilnius (Nov 2004)`_ - `Leysin (Jan 2005)`_ - `PyCon - /Washington (March 2005)`_ - `Europython - /Gothenburg (June 2005)`_ - `Hildesheim (July 2005)`_ - `Heidelberg (Aug 2005)`_ - `Paris (Oct 2005)`_ - `Gothenburg (Dec 2005)`_ - `Mallorca (Jan 2006)`_ - PyCon/Dallas (Feb 2006) - `LouvainLaNeuve (March 2006)`_ - Leysin (April 2006) - `Tokyo (April 2006)`_ - `D?sseldorf (June 2006)`_ - `Europython/ - Geneva (July 2006)`_ - Limerick (Aug 2006) + * `Hildesheim (Feb 2003)`_ + * Gothenburg (May 2003) + * `LovainLaNeuve (June 2003)`_ + * `Berlin (Sept 2003)`_ + * `Amsterdam (Dec 2003)`_ + * Europython/Gothenburg (June 2004) + * Vilnius (Nov 2004) + * `Leysin (Jan 2005)`_ + * `PyCon/Washington (March 2005)`_ + * `Europython/Gothenburg (June 2005)`_ + * `Hildesheim (July 2005)`_ + * `Heidelberg (Aug 2005)`_ + * `Paris (Oct 2005)`_ + * `Gothenburg (Dec 2005)`_ + * `Mallorca (Jan 2006)`_ + * PyCon/Dallas (Feb 2006) + * `LouvainLaNeuve (March 2006)`_ + * Leysin (April 2006) + * `Tokyo (April 2006)`_ + * `D?sseldorf (June 2006)`_ + * `Europython/Geneva (July 2006)`_ + * Limerick (Aug 2006) .. _Hildesheim (Feb 2003): https://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html .. _LovainLaNeuve (June 2003): https://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt @@ -44,9 +40,9 @@ .. _Paris (Oct 2005): https://codespeak.net/pypy/extradoc/sprintinfo/paris/paris-report.txt .. _Gothenburg (Dec 2005): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt .. _Mallorca (Jan 2006): https://codespeak.net/pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt - .. _LovainLaNeuve (March 2006): https://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt + .. _LouvainLaNeuve (March 2006): https://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt .. _Tokyo (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/tokyo/sprint-report.txt .. _D?sseldorf (June 2006): https://codespeak.net/pypy/extradoc/sprintinfo/ddorf2006/report1.txt .. _Europython/Geneva (July 2006): https://codespeak.net/pypy/extradoc/sprintinfo/post-ep2006/report.txt - \ No newline at end of file + Modified: pypy/dist/pypy/doc/translation.txt ============================================================================== --- pypy/dist/pypy/doc/translation.txt (original) +++ pypy/dist/pypy/doc/translation.txt Wed Aug 23 15:23:52 2006 @@ -64,11 +64,12 @@ graphs into low-level operations. 4. After RTyping there are two, rather different, `optional - transformations`_ which can be applied -- the "backend optimizations" - which are make the resulting program go faster, and the "stackless - transform" which transforms the program into a form of continuation - passing style which allows the implementation of coroutines and other - forms of non-standard control flow. + transformations`_ which can be applied -- the "backend + optimizations" which are intended to make the resulting program go + faster, and the "stackless transform" which transforms the program + into a form of continuation passing style which allows the + implementation of coroutines and other forms of non-standard + control flow. 5. The next step is `preparing the graphs for source generation`_, which involves computing the names that the various functions and types in From arigo at codespeak.net Wed Aug 23 16:05:58 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 16:05:58 +0200 (CEST) Subject: [pypy-svn] r31552 - in pypy/dist/pypy/jit/codegen/llgraph: . test Message-ID: <20060823140558.F221010050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 16:05:54 2006 New Revision: 31552 Added: pypy/dist/pypy/jit/codegen/llgraph/ (props changed) pypy/dist/pypy/jit/codegen/llgraph/__init__.py (contents, props changed) pypy/dist/pypy/jit/codegen/llgraph/llimpl.py - copied, changed from r31520, pypy/dist/pypy/rpython/rgenop.py pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (contents, props changed) pypy/dist/pypy/jit/codegen/llgraph/test/ (props changed) pypy/dist/pypy/jit/codegen/llgraph/test/__init__.py (contents, props changed) pypy/dist/pypy/jit/codegen/llgraph/test/test_llimpl.py - copied, changed from r31520, pypy/dist/pypy/rpython/test/test_rgenop.py Log: (pedronis, arigo) In-progress: move the rpython.rgenop file here, as a llgraph-producing codegen. Start using a common OO model for all codegens. (That's in model.py, coming soon.) Added: pypy/dist/pypy/jit/codegen/llgraph/__init__.py ============================================================================== Added: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Wed Aug 23 16:05:54 2006 @@ -0,0 +1,78 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.jit.codegen.model import AbstractRGenOp, CodeGenBlock +from pypy.jit.codegen.model import GenVar, GenConst +from pypy.jit.codegen.llgraph import llimpl + + +class LLVar(GenVar): + def __init__(self, v): + self.v = v + + +class LLConst(GenConst): + def __init__(self, v): + self.v = v + + def revealconst(self, T): + return llimpl.revealconst(T, self.v) + revealconst._annspecialcase_ = 'specialize:arg(1)' + + +class LLBlock(CodeGenBlock): + def __init__(self, b): + self.b = b + + def geninputarg(self, gv_TYPE): + return LLVar(llimpl.geninputarg(self.b, gv_TYPE.v)) + + def genop(self, opname, vars_gv, gv_RESULT_TYPE): + return LLVar(llimpl.genop(self.b, opname, vars_gv, gv_RESULT_TYPE.v)) + genop._annspecialcase_ = 'specialize:arg(1)' + + def close1(self): + return LLLink(llimpl.closeblock1(self.b)) + + def close2(self, gv_exitswitch): + l1, l2 = llimpl.closeblock2(self.b, gv_exitswitch.v) + return LLLink(l1), LLLink(l2) + + +class LLLink(CodeGenLink): + def __init__(self, l): + self.l = l + + def close(self, vars_gv, targetblock): + llimpl.closelink(self.l, vars_gv, targetblock.b) + + def closereturn(self, gv_returnvar): + llimpl.closereturnlink(self.l, gv_returnvar.v) + + +class RGenOp(AbstractRGenOp): + + def newblock(self): + return LLBlock(llimpl.newblock()) + + def gencallableconst(self, name, targetblock, gv_FUNCTYPE): + return LLConst(llimpl.gencallableconst(name, targetblock.b, + gv_FUNCTYPE.v)) + + def genconst(llvalue): + return LLConst(llimpl.genconst(llvalue)) + genconst._annspecialcase_ = 'specialize:ll' + genconst = staticmethod(genconst) + + def constTYPE(T): + return LLConst(llimpl.constTYPE(T)) + constTYPE._annspecialcase_ = 'specialize:memo' + constTYPE = staticmethod(constTYPE) + + def placeholder(dummy): + return LLConst(llimpl.placerholder(dummy)) + placeholder._annspecialcase_ = 'specialize:arg(0)' + placeholder = staticmethod(placeholder) + + def constFieldName(T, name): + return LLConst(llimpl.constFieldName(name)) + constFieldName._annspecialcase_ = 'specialize:memo' + constFieldName = staticmethod(constFieldName) Added: pypy/dist/pypy/jit/codegen/llgraph/test/__init__.py ============================================================================== From arigo at codespeak.net Wed Aug 23 16:07:19 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 16:07:19 +0200 (CEST) Subject: [pypy-svn] r31553 - in pypy/dist/pypy/jit/codegen: . i386 Message-ID: <20060823140719.6E49710050@code0.codespeak.net> Author: arigo Date: Wed Aug 23 16:07:17 2006 New Revision: 31553 Added: pypy/dist/pypy/jit/codegen/model.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py Log: (pedronis, arigo) Rest of the previous check-in. Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Wed Aug 23 16:07:17 2006 @@ -96,6 +96,10 @@ def __init__(self): self.mcs = [] # machine code blocks where no-one is currently writing + def get_rgenop_for_testing(): + return RI386GenOp() + get_rgenop_for_testing = staticmethod(get_rgenop_for_testing) + def open_mc(self): if self.mcs: # XXX think about inserting NOPS for alignment @@ -189,7 +193,7 @@ T = lltype.typeOf(llvalue) assert T is lltype.Signed return IntConst(llvalue) - genconst._annspecialcase_ = 'specialize:argtype(0)' # XXX arglltype(0)? + genconst._annspecialcase_ = 'specialize:ll' genconst = staticmethod(genconst) def constTYPE(T): Added: pypy/dist/pypy/jit/codegen/model.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/model.py Wed Aug 23 16:07:17 2006 @@ -0,0 +1,21 @@ + + +class GenVarOrConst(object): + pass + +class GenVar(GenVarOrConst): + is_const = False + +class GenConst(GenVarOrConst): + is_const = True + + +class CodeGenBlock(object): + pass + +class CodeGenLink(object): + pass + + +class AbstractRGenOp(object): + pass From mwh at codespeak.net Wed Aug 23 16:30:20 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 16:30:20 +0200 (CEST) Subject: [pypy-svn] r31554 - in pypy/branch/no-zeroing-assumption/pypy: annotation rpython rpython/lltypesystem rpython/test Message-ID: <20060823143020.CDBD910050@code0.codespeak.net> Author: mwh Date: Wed Aug 23 16:30:16 2006 New Revision: 31554 Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/ll_str.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rlist.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py pypy/branch/no-zeroing-assumption/pypy/rpython/objectmodel.py pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rdict.py pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rstr.py pypy/branch/no-zeroing-assumption/pypy/rpython/test/tool.py Log: whacky-whack-whack until rpython/test all passes again. (cheat rather completely in rdict by inventing a new zero_malloc operation) Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py Wed Aug 23 16:30:16 2006 @@ -382,7 +382,7 @@ from pypy.annotation.model import SomePtr from pypy.rpython.lltypesystem import lltype -def malloc(s_T, s_n=None, s_flavor=None, s_extra_args=None): +def malloc(s_T, s_n=None, s_flavor=None, s_extra_args=None, s_zero=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, pypy.rpython.rarithmetic.base_int)) assert s_T.is_constant() @@ -390,6 +390,8 @@ n = 1 else: n = None + if s_zero: + assert s_zero.is_constant() if s_flavor is None: p = lltype.malloc(s_T.const, n) r = SomePtr(lltype.typeOf(p)) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py Wed Aug 23 16:30:16 2006 @@ -532,6 +532,10 @@ else: return self.heap.malloc(obj) + def op_zero_malloc(self, obj): + assert self.llinterpreter.gc is None + return self.heap.malloc(obj, zero=True) + def op_malloc_varsize(self, obj, size): if self.llinterpreter.gc is not None: args = self.llinterpreter.gc.get_arg_malloc(obj, size) @@ -544,6 +548,10 @@ except MemoryError: self.make_llexception() + def op_zero_malloc_varsize(self, obj, size): + assert self.llinterpreter.gc is None + return self.heap.malloc(obj, size, zero=True) + def op_flavored_malloc(self, flavor, obj): assert isinstance(flavor, str) if flavor == "stack": Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/ll_str.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/ll_str.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/ll_str.py Wed Aug 23 16:30:16 2006 @@ -26,6 +26,7 @@ len += 1 len += sign result = malloc(STR, len) + result.hash = 0 if sign: result.chars[0] = '-' j = 1 @@ -61,6 +62,7 @@ if addPrefix: len += 2 result = malloc(STR, len) + result.hash = 0 j = 0 if sign: result.chars[0] = '-' @@ -78,6 +80,7 @@ from pypy.rpython.lltypesystem.rstr import STR if i == 0: result = malloc(STR, 1) + result.hash = 0 result.chars[0] = '0' return result temp = malloc(CHAR_ARRAY, 25) @@ -94,6 +97,7 @@ if addPrefix: len += 1 result = malloc(STR, len) + result.hash = 0 j = 0 if sign: result.chars[0] = '-' Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py Wed Aug 23 16:30:16 2006 @@ -290,7 +290,9 @@ # __________ pointer operations __________ 'malloc': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'zero_malloc': LLOp(canraise=(MemoryError,), canunwindgc=True), 'malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), + 'zero_malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), 'flavored_malloc': LLOp(canraise=(MemoryError,)), 'flavored_free': LLOp(), 'getfield': LLOp(sideeffects=False), Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py Wed Aug 23 16:30:16 2006 @@ -1555,7 +1555,7 @@ return _ptr(Ptr(TYPE), o) def nullptr(T): - return Ptr(T)._defl() + return Ptr(T)._defl(example=True) def opaqueptr(TYPE, name, **attrs): if not isinstance(TYPE, OpaqueType): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py Wed Aug 23 16:30:16 2006 @@ -517,7 +517,10 @@ mangled_name, r = self.allinstancefields[fldname] if r.lowleveltype is Void: continue - value = self.classdef.classdesc.read_attribute(fldname, None) + if fldname == '_hash_cache_': + value = Constant(0, Signed) + else: + value = self.classdef.classdesc.read_attribute(fldname, None) if value is not None: cvalue = inputconst(r.lowleveltype, r.convert_desc_or_const(value)) @@ -568,6 +571,7 @@ from pypy.rpython.lltypesystem import rstr nameLen = len(instance.typeptr.name) nameString = malloc(rstr.STR, nameLen-1) + nameString.hash = 0 i = 0 while i < nameLen - 1: nameString.chars[i] = instance.typeptr.name[i] Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rdict.py Wed Aug 23 16:30:16 2006 @@ -458,7 +458,7 @@ new_size = old_size * 2 while new_size > DICT_INITSIZE and d.num_items < new_size / 4: new_size /= 2 - d.entries = lltype.malloc(lltype.typeOf(old_entries).TO, new_size) + d.entries = lltype.malloc(lltype.typeOf(old_entries).TO, new_size, zero=True) d.num_items = 0 d.num_pristine_entries = new_size i = 0 @@ -550,7 +550,7 @@ def ll_newdict(DICT): d = lltype.malloc(DICT) - d.entries = lltype.malloc(DICT.entries.TO, DICT_INITSIZE) + d.entries = lltype.malloc(DICT.entries.TO, DICT_INITSIZE, zero=True) d.num_items = 0 d.num_pristine_entries = DICT_INITSIZE return d @@ -561,7 +561,7 @@ while n < length_estimate: n *= 2 d = lltype.malloc(DICT) - d.entries = lltype.malloc(DICT.entries.TO, n) + d.entries = lltype.malloc(DICT.entries.TO, n, zero=True) d.num_items = 0 d.num_pristine_entries = DICT_INITSIZE return d @@ -654,7 +654,7 @@ DICT = lltype.typeOf(dict).TO dictsize = len(dict.entries) d = lltype.malloc(DICT) - d.entries = lltype.malloc(DICT.entries.TO, dictsize) + d.entries = lltype.malloc(DICT.entries.TO, dictsize, zero=True) d.num_items = dict.num_items d.num_pristine_entries = dict.num_pristine_entries if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq @@ -676,7 +676,7 @@ if len(d.entries) == d.num_pristine_entries == DICT_INITSIZE: return DICT = lltype.typeOf(d).TO - d.entries = lltype.malloc(DICT.entries.TO, DICT_INITSIZE) + d.entries = lltype.malloc(DICT.entries.TO, DICT_INITSIZE, zero=True) d.num_items = 0 d.num_pristine_entries = DICT_INITSIZE Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rlist.py Wed Aug 23 16:30:16 2006 @@ -277,8 +277,9 @@ # XXX consider to have a real realloc items = l.items newitems = malloc(typeOf(l).TO.items.TO, new_allocated) - if allocated < new_allocated: - p = allocated - 1 + before_len = l.length + if before_len < new_allocated: + p = before_len - 1 else: p = new_allocated - 1 while p >= 0: Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py Wed Aug 23 16:30:16 2006 @@ -73,7 +73,7 @@ return malloc(self.EMPTY, immortal=True) def null_instance(self): - return llmemory.Address._defl() + return llmemory.Address._defl(example=True) class __extend__(pairtype(MultipleUnrelatedFrozenPBCRepr, MultipleUnrelatedFrozenPBCRepr), Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py Wed Aug 23 16:30:16 2006 @@ -117,10 +117,15 @@ # get flowed and annotated, mostly with SomePtr. # +def mallocstr(size): + r = malloc(STR, size) + r.hash = 0 + return r + class LLHelpers(AbstractLLHelpers): def ll_char_mul(ch, times): - newstr = malloc(STR, times) + newstr = mallocstr(times) j = 0 while j < times: newstr.chars[j] = ch @@ -134,7 +139,7 @@ return s.chars[i] def ll_chr2str(ch): - s = malloc(STR, 1) + s = mallocstr(1) s.chars[0] = ch return s @@ -154,7 +159,7 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - newstr = malloc(STR, len1 + len2) + newstr = mallocstr(len1 + len2) j = 0 while j < len1: newstr.chars[j] = s1.chars[j] @@ -179,7 +184,7 @@ while lpos < rpos and s.chars[rpos] == ch: rpos -= 1 r_len = rpos - lpos + 1 - result = malloc(STR, r_len) + result = mallocstr(r_len) i = 0 j = lpos while i < r_len: @@ -194,7 +199,7 @@ if s_len == 0: return emptystr i = 0 - result = malloc(STR, s_len) + result = mallocstr(s_len) while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': @@ -209,7 +214,7 @@ if s_len == 0: return emptystr i = 0 - result = malloc(STR, s_len) + result = mallocstr(s_len) while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': @@ -229,7 +234,7 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = malloc(STR, itemslen + s_len * (num_items - 1)) + result = mallocstr(itemslen + s_len * (num_items - 1)) res_chars = result.chars res_index = 0 i = 0 @@ -449,7 +454,7 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = malloc(STR, itemslen) + result = mallocstr(itemslen) res_chars = result.chars res_index = 0 i = 0 @@ -466,7 +471,7 @@ def ll_join_chars(length, chars): num_chars = length - result = malloc(STR, num_chars) + result = mallocstr(num_chars) res_chars = result.chars i = 0 while i < num_chars: @@ -476,7 +481,7 @@ def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) - newstr = malloc(STR, len1 - start) + newstr = mallocstr(len1 - start) j = 0 while start < len1: newstr.chars[j] = s1.chars[start] @@ -491,7 +496,7 @@ if start == 0: return s1 stop = len(s1.chars) - newstr = malloc(STR, stop - start) + newstr = mallocstr(stop - start) j = 0 while start < stop: newstr.chars[j] = s1.chars[start] @@ -502,7 +507,7 @@ def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 assert newlen >= 0 - newstr = malloc(STR, newlen) + newstr = mallocstr(newlen) j = 0 while j < newlen: newstr.chars[j] = s1.chars[j] @@ -525,7 +530,7 @@ resindex = 0 while j < strlen: if chars[j] == c: - item = items[resindex] = malloc(STR, j - i) + item = items[resindex] = mallocstr(j - i) newchars = item.chars k = i while k < j: @@ -534,7 +539,7 @@ resindex += 1 i = j + 1 j += 1 - item = items[resindex] = malloc(STR, j - i) + item = items[resindex] = mallocstr(j - i) newchars = item.chars k = i while k < j: @@ -546,7 +551,7 @@ def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) - newstr = malloc(STR, length) + newstr = mallocstr(length) src = s.chars dst = newstr.chars j = 0 @@ -619,6 +624,7 @@ cTEMP = inputconst(Void, TEMP) vtemp = hop.genop("malloc_varsize", [cTEMP, size], resulttype=Ptr(TEMP)) + # XXX hash r_tuple = hop.args_r[1] v_tuple = hop.args_v[1] Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/objectmodel.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/objectmodel.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/objectmodel.py Wed Aug 23 16:30:16 2006 @@ -52,7 +52,7 @@ from pypy.rpython.lltypesystem import lltype return lltype.Signed -malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=1) +malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0) def instantiate(cls): "Create an empty instance of 'cls'." Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py Wed Aug 23 16:30:16 2006 @@ -321,15 +321,19 @@ BUILTIN_TYPER[object.__init__] = rtype_object__init__ # annotation of low-level types -def rtype_malloc(hop, i_flavor=None, i_extra_args=None): +def rtype_malloc(hop, i_flavor=None, i_extra_args=None, i_zero=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' - v_flavor, v_extra_args = parse_kwds(hop, (i_flavor, lltype.Void), - (i_extra_args, None)) + v_flavor, v_extra_args, v_zero = parse_kwds(hop, (i_flavor, lltype.Void), + (i_extra_args, None), + (i_zero, None)) if v_flavor is not None: vlist.insert(0, v_flavor) opname = 'flavored_' + opname + if i_zero is not None: + assert i_extra_args is i_flavor is None + opname = 'zero_' + opname if hop.nb_args == 2: vlist.append(hop.inputarg(lltype.Signed, arg=1)) opname += '_varsize' Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rdict.py Wed Aug 23 16:30:16 2006 @@ -513,6 +513,7 @@ p = lltype.malloc(rstr.STR, len(value)) for i in range(len(value)): p.chars[i] = value[i] + p.hash = 0 return rstr.LLHelpers.ll_strhash(p) def func(c1, c2): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/test/test_rstr.py Wed Aug 23 16:30:16 2006 @@ -600,6 +600,7 @@ p = malloc(STR, len(s)) for i in range(len(s)): p.chars[i] = s[i] + p.hash = 0 return p def test_ll_find_rfind(self): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/test/tool.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/test/tool.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/test/tool.py Wed Aug 23 16:30:16 2006 @@ -29,7 +29,11 @@ return LLSupport.to_rstr(s) def ll_to_list(self, l): - return map(None, l.ll_items())[:l.ll_length()] + r = [] + items = l.ll_items() + for i in range(l.ll_length()): + r.append(items[i]) + return r def get_callable(self, fnptr): return fnptr._obj._callable From mwh at codespeak.net Wed Aug 23 17:46:28 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 17:46:28 +0200 (CEST) Subject: [pypy-svn] r31560 - in pypy/branch/no-zeroing-assumption/pypy: rpython translator/c Message-ID: <20060823154628.EF55010060@code0.codespeak.net> Author: mwh Date: Wed Aug 23 17:46:27 2006 New Revision: 31560 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py pypy/branch/no-zeroing-assumption/pypy/translator/c/database.py Log: make a bundle more tests pass. Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py Wed Aug 23 17:46:27 2006 @@ -515,7 +515,7 @@ graph = obj.graph args = [] for arg in obj.graph.startblock.inputargs: - args.append(arg.concretetype._defl()) + args.append(arg.concretetype._defl(example=True)) frame = self.__class__(graph, args, self.llinterpreter, self) result = frame.eval() from pypy.translator.stackless.frame import storage_type Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/database.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/database.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/database.py Wed Aug 23 17:46:27 2006 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem.lltype import \ Primitive, Ptr, typeOf, RuntimeTypeInfo, \ Struct, Array, FuncType, PyObject, Void, \ - ContainerType, OpaqueType, FixedSizeArray + ContainerType, OpaqueType, FixedSizeArray, Uninitialized from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.llmemory import Address from pypy.rpython.memory.lladdress import NULL @@ -282,6 +282,8 @@ def add_dependencies(newdependencies): for value in newdependencies: + if value is Uninitialized: + continue if isinstance(typeOf(value), ContainerType): self.getcontainernode(value) else: From mwh at codespeak.net Wed Aug 23 18:01:27 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 18:01:27 +0200 (CEST) Subject: [pypy-svn] r31562 - in pypy/branch/no-zeroing-assumption/pypy: rpython/lltypesystem translator/c Message-ID: <20060823160127.956CC1006C@code0.codespeak.net> Author: mwh Date: Wed Aug 23 18:01:25 2006 New Revision: 31562 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py Log: couple of small hacks to get _most_ of the c tests running (they still zero memory though) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py Wed Aug 23 18:01:25 2006 @@ -1526,9 +1526,9 @@ def malloc(T, n=None, flavor='gc', immortal=False, extra_args=(), zero=False): if isinstance(T, Struct): - o = _struct(T, n, example=zero) + o = _struct(T, n, example=zero or immortal) elif isinstance(T, Array): - o = _array(T, n, example=zero) + o = _array(T, n, example=zero or immortal) else: raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py Wed Aug 23 18:01:25 2006 @@ -508,6 +508,8 @@ return self.gcpolicy.zero_malloc(TYPE, esize, eresult) + OP_ZERO_MALLOC = OP_MALLOC + def OP_MALLOC_VARSIZE(self, op): TYPE = self.lltypemap(op.result).TO typename = self.db.gettype(TYPE) @@ -546,6 +548,8 @@ result += '\n}' return result + OP_ZERO_MALLOC_VARSIZE = OP_MALLOC_VARSIZE + def OP_RAW_MALLOC(self, op): eresult = self.expr(op.result) esize = self.expr(op.args[0]) From mwh at codespeak.net Wed Aug 23 18:39:49 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 18:39:49 +0200 (CEST) Subject: [pypy-svn] r31565 - pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem Message-ID: <20060823163949.5BC6F10069@code0.codespeak.net> Author: mwh Date: Wed Aug 23 18:39:48 2006 New Revision: 31565 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py Log: un-nice some code to please the annotator :( Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rstr.py Wed Aug 23 18:39:48 2006 @@ -117,15 +117,11 @@ # get flowed and annotated, mostly with SomePtr. # -def mallocstr(size): - r = malloc(STR, size) - r.hash = 0 - return r - class LLHelpers(AbstractLLHelpers): def ll_char_mul(ch, times): - newstr = mallocstr(times) + newstr = malloc(STR, times) + newstr.hash = 0 j = 0 while j < times: newstr.chars[j] = ch @@ -139,7 +135,8 @@ return s.chars[i] def ll_chr2str(ch): - s = mallocstr(1) + s = malloc(STR, 1) + s.hash = 0 s.chars[0] = ch return s @@ -159,7 +156,8 @@ def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) - newstr = mallocstr(len1 + len2) + newstr = malloc(STR, len1 + len2) + newstr.hash = 0 j = 0 while j < len1: newstr.chars[j] = s1.chars[j] @@ -184,7 +182,8 @@ while lpos < rpos and s.chars[rpos] == ch: rpos -= 1 r_len = rpos - lpos + 1 - result = mallocstr(r_len) + result = malloc(STR, r_len) + result.hash = 0 i = 0 j = lpos while i < r_len: @@ -199,7 +198,8 @@ if s_len == 0: return emptystr i = 0 - result = mallocstr(s_len) + result = malloc(STR, s_len) + result.hash = 0 while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': @@ -214,7 +214,8 @@ if s_len == 0: return emptystr i = 0 - result = mallocstr(s_len) + result = malloc(STR, s_len) + result.hash = 0 while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': @@ -234,7 +235,8 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen + s_len * (num_items - 1)) + result = malloc(STR, itemslen + s_len * (num_items - 1)) + result.hash = 0 res_chars = result.chars res_index = 0 i = 0 @@ -454,7 +456,8 @@ while i < num_items: itemslen += len(items[i].chars) i += 1 - result = mallocstr(itemslen) + result = malloc(STR, itemslen) + result.hash = 0 res_chars = result.chars res_index = 0 i = 0 @@ -471,7 +474,8 @@ def ll_join_chars(length, chars): num_chars = length - result = mallocstr(num_chars) + result = malloc(STR, num_chars) + result.hash = 0 res_chars = result.chars i = 0 while i < num_chars: @@ -481,7 +485,8 @@ def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) - newstr = mallocstr(len1 - start) + newstr = malloc(STR, len1 - start) + newstr.hash = 0 j = 0 while start < len1: newstr.chars[j] = s1.chars[start] @@ -496,7 +501,8 @@ if start == 0: return s1 stop = len(s1.chars) - newstr = mallocstr(stop - start) + newstr = malloc(STR, stop - start) + newstr.hash = 0 j = 0 while start < stop: newstr.chars[j] = s1.chars[start] @@ -507,7 +513,8 @@ def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 assert newlen >= 0 - newstr = mallocstr(newlen) + newstr = malloc(STR, newlen) + newstr.hash = 0 j = 0 while j < newlen: newstr.chars[j] = s1.chars[j] @@ -530,7 +537,8 @@ resindex = 0 while j < strlen: if chars[j] == c: - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = malloc(STR, j - i) + item.hash = 0 newchars = item.chars k = i while k < j: @@ -539,7 +547,8 @@ resindex += 1 i = j + 1 j += 1 - item = items[resindex] = mallocstr(j - i) + item = items[resindex] = malloc(STR, j - i) + item.hash = 0 newchars = item.chars k = i while k < j: @@ -551,7 +560,8 @@ def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) - newstr = mallocstr(length) + newstr = malloc(STR, length) + newstr.hash = 0 src = s.chars dst = newstr.chars j = 0 From arigo at codespeak.net Wed Aug 23 18:39:55 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 18:39:55 +0200 (CEST) Subject: [pypy-svn] r31566 - pypy/dist/pypy/doc/js Message-ID: <20060823163955.3DB6E10079@code0.codespeak.net> Author: arigo Date: Wed Aug 23 18:39:52 2006 New Revision: 31566 Modified: pypy/dist/pypy/doc/js/ (props changed) Log: Ignore *.html in doc/js/. From mwh at codespeak.net Wed Aug 23 18:40:53 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 18:40:53 +0200 (CEST) Subject: [pypy-svn] r31567 - pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem Message-ID: <20060823164053.ADB9710069@code0.codespeak.net> Author: mwh Date: Wed Aug 23 18:40:52 2006 New Revision: 31567 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py Log: when a prebuilt instance doesn't have a value for an attribute, don't default to Uninitialized (which will explode later). bit nasty really, but this is easiest for now. Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py Wed Aug 23 18:40:52 2006 @@ -440,8 +440,9 @@ if attrvalue is None: warning("prebuilt instance %r has no attribute %r" % ( value, name)) - continue - llattrvalue = r.convert_desc_or_const(attrvalue) + llattrvalue = r.lowleveltype._defl(example=True) + else: + llattrvalue = r.convert_desc_or_const(attrvalue) else: llattrvalue = r.convert_const(attrvalue) setattr(result, mangled_name, llattrvalue) From arigo at codespeak.net Wed Aug 23 18:47:41 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Aug 2006 18:47:41 +0200 (CEST) Subject: [pypy-svn] r31568 - in pypy/dist/pypy: annotation jit/codegen/i386 jit/codegen/i386/test jit/codegen/llgraph jit/llabstractinterp jit/timeshifter jit/timeshifter/test rpython rpython/lltypesystem rpython/test Message-ID: <20060823164741.C8FE710078@code0.codespeak.net> Author: arigo Date: Wed Aug 23 18:47:27 2006 New Revision: 31568 Removed: pypy/dist/pypy/rpython/rgenop.py pypy/dist/pypy/rpython/test/test_rgenop.py Modified: pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/policy.py pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py pypy/dist/pypy/jit/codegen/llgraph/llimpl.py pypy/dist/pypy/jit/codegen/llgraph/rgenop.py pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py pypy/dist/pypy/jit/llabstractinterp/llcontainer.py pypy/dist/pypy/jit/llabstractinterp/llvalue.py pypy/dist/pypy/jit/timeshifter/oop.py pypy/dist/pypy/jit/timeshifter/rcontainer.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/rtyper.py pypy/dist/pypy/jit/timeshifter/rvalue.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/test/test_tl.py pypy/dist/pypy/jit/timeshifter/test/test_tlr.py pypy/dist/pypy/jit/timeshifter/test/test_vlist.py pypy/dist/pypy/jit/timeshifter/timeshift.py pypy/dist/pypy/jit/timeshifter/vlist.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/lltypesystem/rlist.py pypy/dist/pypy/rpython/rtyper.py pypy/dist/pypy/rpython/test/test_rpbc.py Log: (pedronis, arigo) Make the timeshifter parametrized on a RGenOp class, which is the interface to specific code generation back-ends. For this we did the following: slightly change the interface of rgenop, which is now located in pypy/jit/codegen/*/rgenop.py and is based on classes a bit more. the old rgenop with opaque objects is located in codegen/llgraph/llimpl and codegen/llgraph/rgenop gives a nicer OO interface over it. updated codegen/i386/ri386genop to match the same OO interface. fixed everything everywhere. _annspecialcase_ = 'specialize:ll' can be used with any policy now. added lltypesystem.rclass.fishllattr() to read an instance attribute when we only have a low-level GcStruct. (Could be used to replace a few existing 'inst_xyz' attribute fishing, too.) Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Wed Aug 23 18:47:27 2006 @@ -408,18 +408,16 @@ result.knowntype = tp # at least for types this needs to be correct else: result = SomePBC([self.getdesc(x)]) - elif hasattr(x, '__class__') \ - and x.__class__.__module__ != '__builtin__': + elif hasattr(x, '_freeze_') and x._freeze_(): # user-defined classes can define a method _freeze_(), which # is called when a prebuilt instance is found. If the method # returns True, the instance is considered immutable and becomes # a SomePBC(). Otherwise it's just SomeInstance(). - frozen = hasattr(x, '_freeze_') and x._freeze_() - if frozen: - result = SomePBC([self.getdesc(x)]) - else: - self.see_mutable(x) - result = SomeInstance(self.getuniqueclassdef(x.__class__)) + result = SomePBC([self.getdesc(x)]) + elif hasattr(x, '__class__') \ + and x.__class__.__module__ != '__builtin__': + self.see_mutable(x) + result = SomeInstance(self.getuniqueclassdef(x.__class__)) elif x is None: return s_None else: @@ -512,18 +510,16 @@ result.knowntype = tp # at least for types this needs to be correct else: result = SomePBC([self.getdesc(x)]) - elif hasattr(x, '__class__') \ - and x.__class__.__module__ != '__builtin__': + elif hasattr(x, '_freeze_') and x._freeze_(): # user-defined classes can define a method _freeze_(), which # is called when a prebuilt instance is found. If the method # returns True, the instance is considered immutable and becomes # a SomePBC(). Otherwise it's just SomeInstance(). - frozen = hasattr(x, '_freeze_') and x._freeze_() - if frozen: - result = SomePBC([self.getdesc(x)]) - else: - self.see_mutable(x) - result = SomeInstance(self.getuniqueclassdef(x.__class__)) + result = SomePBC([self.getdesc(x)]) + elif hasattr(x, '__class__') \ + and x.__class__.__module__ != '__builtin__': + self.see_mutable(x) + result = SomeInstance(self.getuniqueclassdef(x.__class__)) elif x is None: return s_None else: Modified: pypy/dist/pypy/annotation/policy.py ============================================================================== --- pypy/dist/pypy/annotation/policy.py (original) +++ pypy/dist/pypy/annotation/policy.py Wed Aug 23 18:47:27 2006 @@ -8,7 +8,7 @@ from pypy.annotation.bookkeeper import getbookkeeper -class BasicAnnotatorPolicy: +class BasicAnnotatorPolicy(object): allow_someobjects = True def event(pol, bookkeeper, what, *args): @@ -83,6 +83,10 @@ specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N) specialize__arglistitemtype = staticmethod(specialize_arglistitemtype) + def specialize__ll(pol, *args): + from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy + return LowLevelAnnotatorPolicy.default_specialize(*args) + def override__ignore(pol, *args): bk = getbookkeeper() return bk.immutablevalue(None) Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Wed Aug 23 18:47:27 2006 @@ -1,16 +1,14 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.codegen.i386.codebuf import MachineCodeBlock from pypy.jit.codegen.i386.ri386 import * +from pypy.jit.codegen.model import AbstractRGenOp, CodeGenBlock, CodeGenLink +from pypy.jit.codegen.model import GenVar, GenConst WORD = 4 -class VarOrConst(object): - pass - - -class Var(VarOrConst): +class Var(GenVar): def __init__(self, stackpos): # 'stackpos' is an index relative to the pushed arguments: @@ -28,13 +26,13 @@ return block.stack_access(self.stackpos) -class TypeConst(VarOrConst): +class TypeConst(GenConst): def __init__(self, kind): self.kind = kind -class IntConst(VarOrConst): +class IntConst(GenConst): def __init__(self, value): self.value = value @@ -42,6 +40,10 @@ def operand(self, block): return imm(self.value) + def revealconst(self, T): + return lltype.cast_int_to_ptr(T, self.value) + revealconst._annspecialcase_ = 'specialize:arg(1)' + class FnPtrConst(IntConst): def __init__(self, value, mc): @@ -49,8 +51,9 @@ self.mc = mc # to keep it alive -class Block(object): - def __init__(self, mc): +class Block(CodeGenBlock): + def __init__(self, rgenop, mc): + self.rgenop = rgenop self.argcount = 0 self.stackdepth = 0 self.mc = mc @@ -62,6 +65,21 @@ self.stackdepth += 1 return res + def genop(self, opname, args_gv, gv_RESTYPE=None): + genmethod = getattr(self, 'op_' + opname) + return genmethod(args_gv, gv_RESTYPE) + genop._annspecialcase_ = 'specialize:arg(1)' + + def close1(self): + return Link(self) + + def close2(self, gv_condition): + false_block = self.rgenop.newblock() + false_block.stackdepth = self.stackdepth + self.mc.CMP(gv_condition.operand(self), imm8(0)) + self.mc.JE(rel32(false_block.startaddr)) + return Link(false_block), Link(self) + def stack_access(self, stackpos): return mem(esp, WORD * (self.stackdepth-1 - stackpos)) @@ -89,51 +107,24 @@ return self.push(eax) -class RI386GenOp(object): - gv_IntWord = TypeConst('IntWord') - gv_Void = TypeConst('Void') +class Link(CodeGenLink): - def __init__(self): - self.mcs = [] # machine code blocks where no-one is currently writing - - def get_rgenop_for_testing(): - return RI386GenOp() - get_rgenop_for_testing = staticmethod(get_rgenop_for_testing) - - def open_mc(self): - if self.mcs: - # XXX think about inserting NOPS for alignment - return self.mcs.pop() - else: - return MachineCodeBlock(65536) # XXX supposed infinite for now - - def close_mc(self, mc): - self.mcs.append(mc) - - def newblock(self): - return Block(self.open_mc()) + def __init__(self, prevblock): + self.prevblock = prevblock - def closeblock1(self, block): - return block # NB. links and blocks are the same for us + def closereturn(self, gv_result): + block = self.prevblock + block.mc.MOV(eax, gv_result.operand(block)) + block.mc.ADD(esp, imm(WORD * block.stackdepth)) + block.mc.RET() + block.rgenop.close_mc(block.mc) - def closeblock2(self, block, gv_condition): - false_block = self.newblock() - false_block.stackdepth = block.stackdepth - block.mc.CMP(gv_condition.operand(block), imm8(0)) - block.mc.JE(rel32(false_block.startaddr)) - return false_block, block - - def closereturnlink(self, link, gv_result): - link.mc.MOV(eax, gv_result.operand(link)) - link.mc.ADD(esp, imm(WORD * link.stackdepth)) - link.mc.RET() - self.close_mc(link.mc) - - def closelink(self, link, outputargs_gv, targetblock): + def close(self, outputargs_gv, targetblock): + block = self.prevblock N = len(outputargs_gv) - if link.stackdepth < N: - link.mc.SUB(esp, imm(WORD * (N - link.stackdepth))) - link.stackdepth = N + if block.stackdepth < N: + block.mc.SUB(esp, imm(WORD * (N - block.stackdepth))) + block.stackdepth = N pending_dests = N srccount = [0] * N @@ -155,15 +146,15 @@ srccount[i] = -1 pending_dests -= 1 gv_src = outputargs_gv[i] - link.mc.MOV(eax, gv_src.operand(link)) - link.mc.MOV(link.stack_access(i), eax) + block.mc.MOV(eax, gv_src.operand(block)) + block.mc.MOV(block.stack_access(i), eax) progress = True if not progress: # we are left with only pure disjoint cycles; break them for i in range(N): if srccount[i] >= 0: dst = i - link.mc.MOV(edx, link.stack_access(dst)) + block.mc.MOV(edx, block.stack_access(dst)) while True: assert srccount[dst] == 1 srccount[dst] = -1 @@ -174,20 +165,42 @@ assert 0 <= src < N if src == i: break - link.mc.MOV(eax, link.stack_access(src)) - link.mc.MOV(link.stack_access(dst), eax) + block.mc.MOV(eax, block.stack_access(src)) + block.mc.MOV(block.stack_access(dst), eax) dst = src - link.mc.MOV(link.stack_access(dst), edx) + block.mc.MOV(block.stack_access(dst), edx) assert pending_dests == 0 - if link.stackdepth > N: - link.mc.ADD(esp, imm(WORD * (link.stackdepth - N))) - link.stackdepth = N - link.mc.JMP(rel32(targetblock.startaddr)) - self.close_mc(link.mc) + if block.stackdepth > N: + block.mc.ADD(esp, imm(WORD * (block.stackdepth - N))) + block.stackdepth = N + block.mc.JMP(rel32(targetblock.startaddr)) + block.rgenop.close_mc(block.mc) + + +class RI386GenOp(AbstractRGenOp): + gv_IntWord = TypeConst('IntWord') + gv_Void = TypeConst('Void') + + def __init__(self): + self.mcs = [] # machine code blocks where no-one is currently writing + + def get_rgenop_for_testing(): + return RI386GenOp() + get_rgenop_for_testing = staticmethod(get_rgenop_for_testing) + + def open_mc(self): + if self.mcs: + # XXX think about inserting NOPS for alignment + return self.mcs.pop() + else: + return MachineCodeBlock(65536) # XXX supposed infinite for now - def geninputarg(self, block, gv_TYPE): - return block.geninputarg(gv_TYPE) + def close_mc(self, mc): + self.mcs.append(mc) + + def newblock(self): + return Block(self, self.open_mc()) def genconst(llvalue): T = lltype.typeOf(llvalue) @@ -204,11 +217,6 @@ constTYPE._annspecialcase_ = 'specialize:memo' constTYPE = staticmethod(constTYPE) - def genop(self, block, opname, args_gv, gv_RESTYPE): - genmethod = getattr(block, 'op_' + opname) - return genmethod(args_gv, gv_RESTYPE) - genop._annspecialcase_ = 'specialize:arg(2)' - def gencallableconst(self, name, block, gv_FUNCTYPE): prologue = self.newblock() #prologue.mc.BREAKPOINT() @@ -220,9 +228,3 @@ prologue.mc.JMP(rel32(block.startaddr)) self.close_mc(prologue.mc) return FnPtrConst(prologue.startaddr, prologue.mc) - - def revealconst(T, gv_const): - assert isinstance(gv_const, IntConst) # for now - return lltype.cast_int_to_ptr(T, gv_const.value) - revealconst._annspecialcase_ = 'specialize:arg(0)' - revealconst = staticmethod(revealconst) Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Wed Aug 23 18:47:27 2006 @@ -14,12 +14,12 @@ # 'return x+n' gv_SIGNED = rgenop.constTYPE(lltype.Signed) block = rgenop.newblock() - gv_x = rgenop.geninputarg(block, gv_SIGNED) + gv_x = block.geninputarg(gv_SIGNED) args_gv = [gv_x, rgenop.genconst(n)] - gv_result = rgenop.genop(block, "int_add", args_gv, gv_SIGNED) - link = rgenop.closeblock1(block) - rgenop.closereturnlink(link, gv_result) + gv_result = block.genop("int_add", args_gv, gv_SIGNED) + link = block.close1() + link.closereturn(gv_result) gv_FUNC = rgenop.constTYPE(FUNC) gv_add_one = rgenop.gencallableconst("adder", block, gv_FUNC) @@ -28,15 +28,15 @@ def runner(x, y): rgenop = RI386GenOp() gv_add_x = make_adder(rgenop, x) - add_x = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_x) + add_x = gv_add_x.revealconst(lltype.Ptr(FUNC)) res = add_x(y) keepalive_until_here(rgenop) # to keep the code blocks alive return res def test_adder_interpret(): - from pypy.rpython import rgenop + from pypy.jit.codegen.llgraph.rgenop import rgenop gv_add_5 = make_adder(rgenop, 5) - add_5 = rgenop.revealconst(lltype.Ptr(FUNC), gv_add_5) + add_5 = gv_add_5.revealconst(lltype.Ptr(FUNC)) llinterp = LLInterpreter(None) res = llinterp.eval_graph(add_5._obj.graph, [12]) assert res == 17 @@ -62,25 +62,25 @@ # 'return x - (y - (x-1))' gv_SIGNED = rgenop.constTYPE(lltype.Signed) block = rgenop.newblock() - gv_x = rgenop.geninputarg(block, gv_SIGNED) - gv_y = rgenop.geninputarg(block, gv_SIGNED) + gv_x = block.geninputarg(gv_SIGNED) + gv_y = block.geninputarg(gv_SIGNED) args_gv = [gv_x, rgenop.genconst(1)] - gv_z = rgenop.genop(block, "int_sub", args_gv, gv_SIGNED) - link = rgenop.closeblock1(block) + gv_z = block.genop("int_sub", args_gv, gv_SIGNED) + link = block.close1() block2 = rgenop.newblock() - gv_y2 = rgenop.geninputarg(block2, gv_SIGNED) - gv_z2 = rgenop.geninputarg(block2, gv_SIGNED) - gv_x2 = rgenop.geninputarg(block2, gv_SIGNED) - rgenop.closelink(link, [gv_y, gv_z, gv_x], block2) + gv_y2 = block2.geninputarg(gv_SIGNED) + gv_z2 = block2.geninputarg(gv_SIGNED) + gv_x2 = block2.geninputarg(gv_SIGNED) + link.close([gv_y, gv_z, gv_x], block2) args_gv = [gv_y2, gv_z2] - gv_s2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) + gv_s2 = block2.genop("int_sub", args_gv, gv_SIGNED) args_gv = [gv_x2, gv_s2] - gv_t2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) - link2 = rgenop.closeblock1(block2) + gv_t2 = block2.genop("int_sub", args_gv, gv_SIGNED) + link2 = block2.close1() - rgenop.closereturnlink(link2, gv_t2) + link2.closereturn(gv_t2) gv_FUNC2 = rgenop.constTYPE(FUNC2) gv_dummyfn = rgenop.gencallableconst("dummy", block, gv_FUNC2) return gv_dummyfn @@ -88,15 +88,15 @@ def dummy_runner(x, y): rgenop = RI386GenOp() gv_dummyfn = make_dummy(rgenop) - dummyfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_dummyfn) + dummyfn = gv_dummyfn.revealconst(lltype.Ptr(FUNC2)) res = dummyfn(x, y) keepalive_until_here(rgenop) # to keep the code blocks alive return res def test_dummy_interpret(): - from pypy.rpython import rgenop + from pypy.jit.codegen.llgraph.rgenop import rgenop gv_dummyfn = make_dummy(rgenop) - dummyfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_dummyfn) + dummyfn = gv_dummyfn.revealconst(lltype.Ptr(FUNC2)) llinterp = LLInterpreter(None) res = llinterp.eval_graph(dummyfn._obj.graph, [30, 17]) assert res == 42 @@ -122,22 +122,22 @@ gv_SIGNED = rgenop.constTYPE(lltype.Signed) gv_BOOL = rgenop.constTYPE(lltype.Bool) block = rgenop.newblock() - gv_x = rgenop.geninputarg(block, gv_SIGNED) - gv_y = rgenop.geninputarg(block, gv_SIGNED) + gv_x = block.geninputarg(gv_SIGNED) + gv_y = block.geninputarg(gv_SIGNED) args_gv = [gv_x, rgenop.genconst(5)] - gv_cond = rgenop.genop(block, "int_gt", args_gv, gv_BOOL) - link_false, link_true = rgenop.closeblock2(block, gv_cond) + gv_cond = block.genop("int_gt", args_gv, gv_BOOL) + link_false, link_true = block.close2(gv_cond) block2 = rgenop.newblock() - gv_x2 = rgenop.geninputarg(block2, gv_SIGNED) - rgenop.closelink(link_true, [gv_x], block2) + gv_x2 = block2.geninputarg(gv_SIGNED) + link_true.close([gv_x], block2) args_gv = [gv_x2, rgenop.genconst(1)] - gv_s2 = rgenop.genop(block2, "int_sub", args_gv, gv_SIGNED) - link2 = rgenop.closeblock1(block2) - rgenop.closereturnlink(link2, gv_s2) + gv_s2 = block2.genop("int_sub", args_gv, gv_SIGNED) + link2 = block2.close1() + link2.closereturn(gv_s2) - rgenop.closereturnlink(link_false, gv_y) + link_false.closereturn(gv_y) gv_FUNC2 = rgenop.constTYPE(FUNC2) gv_branchingfn = rgenop.gencallableconst("branching", block, gv_FUNC2) @@ -146,15 +146,15 @@ def branching_runner(x, y): rgenop = RI386GenOp() gv_branchingfn = make_branching(rgenop) - branchingfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_branchingfn) + branchingfn = gv_branchingfn.revealconst(lltype.Ptr(FUNC2)) res = branchingfn(x, y) keepalive_until_here(rgenop) # to keep the code blocks alive return res def test_branching_interpret(): - from pypy.rpython import rgenop + from pypy.jit.codegen.llgraph.rgenop import rgenop gv_branchingfn = make_branching(rgenop) - branchingfn = rgenop.revealconst(lltype.Ptr(FUNC2), gv_branchingfn) + branchingfn = gv_branchingfn.revealconst(lltype.Ptr(FUNC2)) llinterp = LLInterpreter(None) res = llinterp.eval_graph(branchingfn._obj.graph, [30, 17]) assert res == 29 Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/llimpl.py Wed Aug 23 18:47:27 2006 @@ -12,6 +12,7 @@ from pypy.rpython.module.support import LLSupport from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.lltypesystem.rclass import fishllattr # for debugging, sanity checks in non-RPython code @@ -42,13 +43,13 @@ def _inputvars(vars): if not isinstance(vars, list): n = vars.ll_length() - vars = [getattr(llvar, 'inst_v', llvar) for llvar in vars.ll_items()] + vars = vars.ll_items() + vars = [fishllattr(vars[i], 'v', vars[i]) for i in range(n)] else: - n = len(vars) vars = [getattr(llvar, 'v', llvar) for llvar in vars] res = [] - for i in range(n): - v = from_opaque_object(vars[i]) + for v1 in vars: + v = from_opaque_object(v1) assert isinstance(v, (flowmodel.Constant, flowmodel.Variable)) res.append(v) return res Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Wed Aug 23 18:47:27 2006 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import lltype -from pypy.jit.codegen.model import AbstractRGenOp, CodeGenBlock +from pypy.jit.codegen.model import AbstractRGenOp, CodeGenBlock, CodeGenLink from pypy.jit.codegen.model import GenVar, GenConst from pypy.jit.codegen.llgraph import llimpl +from pypy.rpython.lltypesystem.rclass import fishllattr class LLVar(GenVar): @@ -18,6 +19,9 @@ revealconst._annspecialcase_ = 'specialize:arg(1)' +gv_Void = LLConst(llimpl.constTYPE(lltype.Void)) + + class LLBlock(CodeGenBlock): def __init__(self, b): self.b = b @@ -25,8 +29,9 @@ def geninputarg(self, gv_TYPE): return LLVar(llimpl.geninputarg(self.b, gv_TYPE.v)) - def genop(self, opname, vars_gv, gv_RESULT_TYPE): - return LLVar(llimpl.genop(self.b, opname, vars_gv, gv_RESULT_TYPE.v)) + def genop(self, opname, vars_gv, gv_RESULT_TYPE=None): + return LLVar(llimpl.genop(self.b, opname, vars_gv, + (gv_RESULT_TYPE or gv_Void).v)) genop._annspecialcase_ = 'specialize:arg(1)' def close1(self): @@ -49,6 +54,7 @@ class RGenOp(AbstractRGenOp): + gv_Void = gv_Void def newblock(self): return LLBlock(llimpl.newblock()) @@ -68,7 +74,7 @@ constTYPE = staticmethod(constTYPE) def placeholder(dummy): - return LLConst(llimpl.placerholder(dummy)) + return LLConst(llimpl.placeholder(dummy)) placeholder._annspecialcase_ = 'specialize:arg(0)' placeholder = staticmethod(placeholder) @@ -76,3 +82,31 @@ return LLConst(llimpl.constFieldName(name)) constFieldName._annspecialcase_ = 'specialize:memo' constFieldName = staticmethod(constFieldName) + + constPrebuiltGlobal = genconst + + # not RPython, just for debugging. Specific to llgraph. + def reveal(gv): + return llimpl.reveal(gv.v) + reveal = staticmethod(reveal) + + # Builds a real flow.model.FunctionGraph. Specific to llgraph. + def buildgraph(block): + if hasattr(block, 'b'): + b = block.b + else: + b = fishllattr(block, 'b') + return llimpl.buildgraph(b) + buildgraph = staticmethod(buildgraph) + + testgengraph = staticmethod(llimpl.testgengraph) + + def get_rgenop_for_testing(): + return rgenop + get_rgenop_for_testing = staticmethod(get_rgenop_for_testing) + + def _freeze_(self): + return True # no real point in using a full class in llgraph + + +rgenop = RGenOp() # no real point in using a full class in llgraph Modified: pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py ============================================================================== --- pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py (original) +++ pypy/dist/pypy/jit/llabstractinterp/llabstractinterp.py Wed Aug 23 18:47:27 2006 @@ -10,7 +10,7 @@ from pypy.jit.llabstractinterp.llcontainer import LLAbstractContainer from pypy.jit.llabstractinterp.llcontainer import virtualcontainervalue from pypy.jit.llabstractinterp.llcontainer import hasllcontent -from pypy.rpython import rgenop +from pypy.jit.codegen.llgraph.rgenop import rgenop # ____________________________________________________________ @@ -264,7 +264,7 @@ self.target_set = False def setreturn(self): - rgenop.closereturnlink(self.link, self.args_genv[0]) + self.link.closereturn(self.args_genv[0]) self.target_set = True def settarget(self, block, blockargs): @@ -277,7 +277,7 @@ assert v1.value == v2.value else: args.append(v1) - rgenop.closelink(self.link, args, block) + self.link.close(args, block) self.target_set = True class GraphState(object): @@ -524,14 +524,14 @@ def buildblock(self, newexitswitch, newlinkstates): b = self.newblock if newexitswitch is None: - link = rgenop.closeblock1(b) + link = b.close1() newlinkstates[0].link = link return b assert len(newlinkstates) == 2 v = newexitswitch.getgenvar(self) - false_link, true_link = rgenop.closeblock2(b, v) + false_link, true_link = b.close2(v) cases = {False: false_link, True: true_link} for ls in newlinkstates: @@ -539,14 +539,17 @@ return b def genop(self, opname, args, RESULT_TYPE=lltype.Void): - return rgenop.genop(self.newblock, opname, args, - rgenop.constTYPE(RESULT_TYPE)) + return self.newblock.genop(opname, args, + rgenop.constTYPE(RESULT_TYPE)) def genconst(self, llvalue): return rgenop.genconst(llvalue) def genvoidconst(self, placeholder): return rgenop.placeholder(placeholder) + + def constTYPE(self, T): + return T def binding(self, v): assert isinstance(v, (Constant, Variable)) Modified: pypy/dist/pypy/jit/llabstractinterp/llcontainer.py ============================================================================== --- pypy/dist/pypy/jit/llabstractinterp/llcontainer.py (original) +++ pypy/dist/pypy/jit/llabstractinterp/llcontainer.py Wed Aug 23 18:47:27 2006 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.llabstractinterp.llvalue import LLAbstractValue, AConstant, const -from pypy.rpython import rgenop +from pypy.jit.codegen.llgraph.rgenop import rgenop class LLAbstractContainer(object): """Abstract base class for abstract containers. @@ -51,7 +51,8 @@ def build_runtime_container(self, builder): RESULT_TYPE = lltype.Ptr(self.T) if self.a_parent is not None: - parentindex = rgenop.constFieldName(self.parentindex) + PARENT_TYPE = self.a_parent.getconcretetype().TO + parentindex = rgenop.constFieldName(PARENT_TYPE, self.parentindex) v_parent = self.a_parent.forcegenvarorconst(builder) v_result = builder.genop('getsubstruct', [v_parent, parentindex], RESULT_TYPE) @@ -75,8 +76,9 @@ T = self.fieldtype(name) if isinstance(T, lltype.ContainerType): # initialize the substructure/subarray + c_name = rgenop.constFieldName(self.T, name) v_subptr = builder.genop('getsubstruct', - [v_target, rgenop.constFieldName(name)], + [v_target, c_name], lltype.Ptr(T)) assert isinstance(a_value.content, LLVirtualContainer) a_value.content.buildcontent(builder, v_subptr) @@ -165,7 +167,8 @@ return getattr(self.T, name) def setop(self, builder, v_target, name, v_value): - builder.genop('setfield', [v_target, rgenop.constFieldName(name), v_value], + c_name = rgenop.constFieldName(self.T, name) + builder.genop('setfield', [v_target, c_name, v_value], lltype.Void) Modified: pypy/dist/pypy/jit/llabstractinterp/llvalue.py ============================================================================== --- pypy/dist/pypy/jit/llabstractinterp/llvalue.py (original) +++ pypy/dist/pypy/jit/llabstractinterp/llvalue.py Wed Aug 23 18:47:27 2006 @@ -1,6 +1,6 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype -from pypy.rpython import rgenop +from pypy.jit.codegen.llgraph.rgenop import rgenop from pypy.tool.uid import Hashable class AVariable(object): @@ -202,7 +202,7 @@ assert c.concretetype == self.concretetype result = LLAbstractValue(c) else: - gen_v = rgenop.geninputarg(block, rgenop.constTYPE(self.concretetype)) + gen_v = block.geninputarg(rgenop.constTYPE(self.concretetype)) result = LLAbstractValue(AVariable(self.concretetype, genvar=gen_v)) result.origin.append(self) return result Modified: pypy/dist/pypy/jit/timeshifter/oop.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/oop.py (original) +++ pypy/dist/pypy/jit/timeshifter/oop.py Wed Aug 23 18:47:27 2006 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype -from pypy.rpython import rgenop from pypy.jit.timeshifter.rcontainer import cachedtype from pypy.jit.timeshifter import rvalue @@ -12,7 +11,7 @@ class OopSpecDesc: __metaclass__ = cachedtype - def __init__(self, fnobj): + def __init__(self, RGenOp, fnobj): ll_func = fnobj._callable FUNCTYPE = lltype.typeOf(fnobj) nb_args = len(FUNCTYPE.ARGS) @@ -37,14 +36,14 @@ ARGTYPE = FUNCTYPE.ARGS[i] assert ((i+1) in self.argpositions) == (ARGTYPE is not lltype.Void) - self.args_gv = [rgenop.placeholder(None)] * nb_args - self.args_gv.insert(0, rgenop.genconst(fnobj._as_ptr())) - self.gv_result_type = rgenop.constTYPE(FUNCTYPE.RESULT) + self.args_gv = [RGenOp.placeholder(None)] * nb_args + self.args_gv.insert(0, RGenOp.constPrebuiltGlobal(fnobj._as_ptr())) + self.gv_result_type = RGenOp.constTYPE(FUNCTYPE.RESULT) self.redboxbuilder = rvalue.ll_redboxbuilder(FUNCTYPE.RESULT) if operation_name == 'newlist': from pypy.jit.timeshifter.vlist import ListTypeDesc, oop_newlist - self.typedesc = ListTypeDesc(FUNCTYPE.RESULT.TO) + self.typedesc = ListTypeDesc(RGenOp, FUNCTYPE.RESULT.TO) self.ll_handler = oop_newlist else: typename, method = operation_name.split('.') Modified: pypy/dist/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/dist/pypy/jit/timeshifter/rcontainer.py Wed Aug 23 18:47:27 2006 @@ -1,6 +1,5 @@ import operator from pypy.rpython.lltypesystem import lltype -from pypy.rpython import rgenop from pypy.jit.timeshifter import rvalue class AbstractContainer(object): @@ -30,28 +29,29 @@ __metaclass__ = cachedtype firstsubstructdesc = None - def __init__(self, TYPE): + def __init__(self, RGenOp, TYPE): self.TYPE = TYPE self.PTRTYPE = lltype.Ptr(TYPE) - self.gv_type = rgenop.constTYPE(self.TYPE) - self.gv_ptrtype = rgenop.constTYPE(self.PTRTYPE) + self.gv_type = RGenOp.constTYPE(self.TYPE) + self.gv_ptrtype = RGenOp.constTYPE(self.PTRTYPE) fielddescs = [] for name in self.TYPE._names: FIELDTYPE = getattr(self.TYPE, name) if isinstance(FIELDTYPE, lltype.ContainerType): - substructdesc = StructTypeDesc(FIELDTYPE) + substructdesc = StructTypeDesc(RGenOp, FIELDTYPE) assert name == self.TYPE._names[0], ( "unsupported: inlined substructures not as first field") self.firstsubstructdesc = substructdesc for subfielddesc in substructdesc.fielddescs: dottedname = '%s.%s' % (name, subfielddesc.fieldname) index = len(fielddescs) - fielddescs.append(StructFieldDesc(self.PTRTYPE, dottedname, - index)) + fielddescs.append(StructFieldDesc(RGenOp, self.PTRTYPE, + dottedname, index)) else: index = len(fielddescs) - fielddescs.append(StructFieldDesc(self.PTRTYPE, name, index)) + fielddescs.append(StructFieldDesc(RGenOp, self.PTRTYPE, + name, index)) self.fielddescs = fielddescs def getfielddesc(self, name): @@ -79,12 +79,12 @@ class FieldDesc(object): __metaclass__ = cachedtype - def __init__(self, PTRTYPE, RESTYPE): + def __init__(self, RGenOp, PTRTYPE, RESTYPE): self.PTRTYPE = PTRTYPE if isinstance(RESTYPE, lltype.ContainerType): RESTYPE = lltype.Ptr(RESTYPE) self.RESTYPE = RESTYPE - self.gv_resulttype = rgenop.constTYPE(RESTYPE) + self.gv_resulttype = RGenOp.constTYPE(RESTYPE) self.redboxcls = rvalue.ll_redboxcls(RESTYPE) self.immutable = PTRTYPE.TO._hints.get('immutable', False) @@ -93,8 +93,8 @@ class NamedFieldDesc(FieldDesc): - def __init__(self, PTRTYPE, name): - FieldDesc.__init__(self, PTRTYPE, getattr(PTRTYPE.TO, name)) + def __init__(self, RGenOp, PTRTYPE, name): + FieldDesc.__init__(self, RGenOp, PTRTYPE, getattr(PTRTYPE.TO, name)) self.structdepth = 0 T = self.PTRTYPE.TO while (T._names and @@ -102,33 +102,35 @@ self.structdepth += 1 T = getattr(T, T._names[0]) self.fieldname = name - self.gv_fieldname = rgenop.constFieldName(name) + self.gv_fieldname = RGenOp.constFieldName(T, name) class ArrayFieldDesc(FieldDesc): - def __init__(self, PTRTYPE): + def __init__(self, RGenOp, PTRTYPE): assert isinstance(PTRTYPE.TO, lltype.Array) - FieldDesc.__init__(self, PTRTYPE, PTRTYPE.TO.OF) + FieldDesc.__init__(self, RGenOp, PTRTYPE, PTRTYPE.TO.OF) class StructFieldDesc(object): __metaclass__ = cachedtype - def __init__(self, PTRTYPE, fieldname, index): + def __init__(self, RGenOp, PTRTYPE, fieldname, index): assert isinstance(PTRTYPE.TO, lltype.Struct) RES1 = PTRTYPE.TO accessptrtype_gv = self.accessptrtype_gv = [] + accessors = [] for component in fieldname.split('.'): LASTSTRUCT = RES1 - accessptrtype_gv.append(rgenop.constTYPE(lltype.Ptr(LASTSTRUCT))) + accessptrtype_gv.append(RGenOp.constTYPE(lltype.Ptr(LASTSTRUCT))) + accessors.append((RES1, component)) RES1 = getattr(RES1, component) assert not isinstance(RES1, lltype.ContainerType) self.PTRTYPE = PTRTYPE self.RESTYPE = RES1 - self.gv_resulttype = rgenop.constTYPE(RES1) + self.gv_resulttype = RGenOp.constTYPE(RES1) self.fieldname = fieldname - self.fieldname_gv = [rgenop.constFieldName(component) - for component in fieldname.split('.')] + self.fieldname_gv = [RGenOp.constFieldName(T, component) + for T, component in accessors] self.fieldindex = index - self.gv_default = rgenop.genconst(RES1._defl()) + self.gv_default = RGenOp.constPrebuiltGlobal(RES1._defl()) self.redboxcls = rvalue.ll_redboxcls(RES1) self.immutable = LASTSTRUCT._hints.get('immutable', False) @@ -149,7 +151,7 @@ op_args = [gv_sub, self.fieldname_gv[-1], box.getgenvar(builder)] - genop('setfield', op_args, rgenop.gv_Void) + genop('setfield', op_args, builder.rgenop.gv_Void) # ____________________________________________________________ Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Wed Aug 23 18:47:27 2006 @@ -1,6 +1,5 @@ import operator, weakref from pypy.rpython.lltypesystem import lltype, lloperation, llmemory -from pypy.rpython import rgenop from pypy.jit.timeshifter import rvalue FOLDABLE_OPS = dict.fromkeys(lloperation.enum_foldable_ops()) @@ -19,13 +18,14 @@ def _freeze_(self): return True - def __init__(self, opname, ARGS, RESULT): + def __init__(self, RGenOp, opname, ARGS, RESULT): + self.RGenOp = RGenOp self.opname = opname self.llop = lloperation.LL_OPERATIONS[opname] self.nb_args = len(ARGS) self.ARGS = ARGS self.RESULT = RESULT - self.gv_RESULT = rgenop.constTYPE(RESULT) + self.gv_RESULT = RGenOp.constTYPE(RESULT) self.redboxcls = rvalue.ll_redboxcls(RESULT) self.canfold = opname in FOLDABLE_OPS @@ -43,7 +43,7 @@ def make_opdesc(hop): hrtyper = hop.rtyper - op_key = (hop.spaceop.opname, + op_key = (hrtyper.RGenOp, hop.spaceop.opname, tuple([hrtyper.originalconcretetype(s_arg) for s_arg in hop.args_s]), hrtyper.originalconcretetype(hop.s_result)) try: @@ -60,7 +60,7 @@ if opdesc.canfold and argbox.is_constant(): arg = rvalue.ll_getvalue(argbox, ARG0) res = opdesc.llop(RESULT, arg) - return rvalue.ll_fromvalue(res) + return rvalue.ll_fromvalue(jitstate, res) op_args = [argbox.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop(opdesc.opname, op_args, opdesc.gv_RESULT) @@ -76,7 +76,7 @@ arg0 = rvalue.ll_getvalue(argbox0, ARG0) arg1 = rvalue.ll_getvalue(argbox1, ARG1) res = opdesc.llop(RESULT, arg0, arg1) - return rvalue.ll_fromvalue(res) + return rvalue.ll_fromvalue(jitstate, res) op_args = [argbox0.getgenvar(jitstate.curbuilder), argbox1.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop(opdesc.opname, op_args, @@ -87,7 +87,7 @@ if fielddesc.immutable and argbox.is_constant(): res = getattr(rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE), fielddesc.fieldname) - return rvalue.ll_fromvalue(res) + return rvalue.ll_fromvalue(jitstate, res) assert isinstance(argbox, rvalue.PtrRedBox) if argbox.content is None: op_args = [argbox.getgenvar(jitstate.curbuilder), @@ -104,8 +104,8 @@ op_args = [destbox.getgenvar(jitstate.curbuilder), fielddesc.fieldname_gv[-1], valuebox.getgenvar(jitstate.curbuilder)] - jitstate.curbuilder.genop('setfield', op_args, - rgenop.gv_Void) + builder = jitstate.curbuilder + builder.genop('setfield', op_args, builder.rgenop.gv_Void) else: destbox.content.op_setfield(jitstate, fielddesc, valuebox) @@ -113,7 +113,7 @@ if argbox.is_constant(): res = getattr(rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE), fielddesc.fieldname) - return rvalue.ll_fromvalue(res) + return rvalue.ll_fromvalue(jitstate, res) assert isinstance(argbox, rvalue.PtrRedBox) if argbox.content is None: op_args = [argbox.getgenvar(jitstate.curbuilder), @@ -129,7 +129,7 @@ if fielddesc.immutable and argbox.is_constant() and indexbox.is_constant(): array = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) res = array[rvalue.ll_getvalue(indexbox, lltype.Signed)] - return rvalue.ll_fromvalue(res) + return rvalue.ll_fromvalue(jitstate, res) op_args = [argbox.getgenvar(jitstate.curbuilder), indexbox.getgenvar(jitstate.curbuilder)] genvar = jitstate.curbuilder.genop('getarrayitem', op_args, @@ -143,6 +143,7 @@ return builder.build_jitstate(backstate) def retrieve_jitstate_for_merge(states_dic, jitstate, key, redboxes): + rgenop = jitstate.rgenop mylocalredboxes = redboxes redboxes = list(redboxes) jitstate.extend_with_parent_locals(redboxes) @@ -158,7 +159,7 @@ linkargs = [] for box in outgoingvarboxes: linkargs.append(box.getgenvar(None)) - box.genvar = rgenop.geninputarg(newblock, box.gv_type) + box.genvar = newblock.geninputarg(box.gv_type) jitstate.curbuilder.enter_block(linkargs, newblock) states_dic[key] = frozens, newblock return jitstate @@ -191,7 +192,7 @@ for box in outgoingvarboxes: if box.is_constant(): # constant boxes considered immutable: box = box.copy(replace_memo) # copy to avoid patching the original - box.genvar = rgenop.geninputarg(newblock, box.gv_type) + box.genvar = newblock.geninputarg(box.gv_type) if replace_memo.boxes: for i in range(len(mylocalredboxes)): newbox = redboxes[i].replace(replace_memo) @@ -206,6 +207,7 @@ def enter_block(jitstate, redboxes): # 'redboxes' is a fixed-size list (s_box_list) of the current red boxes + rgenop = jitstate.rgenop newblock = rgenop.newblock() incoming = [] memo = rvalue.enter_block_memo() @@ -224,6 +226,7 @@ def dyn_enter_block(jitstate, redboxes): # 'redboxes' is a var-sized list (s_box_accum) of *all* the boxes # including the ones from the callers' locals + rgenop = jitstate.rgenop newblock = rgenop.newblock() incoming = [] memo = rvalue.enter_block_memo() @@ -269,15 +272,16 @@ retrieve_jitstate_for_merge(cache, jitstate, (), [retbox]) frozens, block = cache[()] _, returnbox = jitstate.return_queue[-1] - builder = ResidualGraphBuilder(block) + rgenop = jitstate.rgenop + builder = ResidualGraphBuilder(rgenop, block) builder.valuebox = returnbox return builder def ll_gvar_from_redbox(jitstate, redbox): return redbox.getgenvar(jitstate.curbuilder) -def ll_gvar_from_constant(ll_value): - return rgenop.genconst(ll_value) +def ll_gvar_from_constant(jitstate, ll_value): + return jitstate.rgenop.genconst(ll_value) def save_locals(jitstate, redboxes): jitstate.localredboxes = redboxes @@ -292,43 +296,57 @@ # ____________________________________________________________ -class ResidualGraphBuilder(rgenop.LowLevelOpBuilder): - def __init__(self, block=rgenop.nullblock, link=rgenop.nulllink): - rgenop.LowLevelOpBuilder.__init__(self, block) +class ResidualGraphBuilder(object): + def __init__(self, rgenop, block=None, link=None): + self.rgenop = rgenop + self.block = block self.outgoinglink = link self.valuebox = None + def genconst(self, llvalue): + return self.rgenop.genconst(llvalue) + genconst._annspecialcase_ = 'specialize:ll' + + def genvoidconst(self, dummy): + return self.rgenop.placeholder(dummy) + genvoidconst._annspecialcase_ = 'specialize:arg(1)' + + def genop(self, opname, args_gv, gv_resulttype=None): + return self.block.genop(opname, args_gv, gv_resulttype) + genop._annspecialcase_ = 'specialize:arg(1)' + + def constTYPE(self, T): + return self.rgenop.constTYPE(T) + constTYPE._annspecialcase_ = 'specialize:arg(1)' + def build_jitstate(self, backstate=None): return JITState(self, backstate) def enter_block(self, linkargs, newblock): - rgenop.closelink(self.outgoinglink, linkargs, newblock) + self.outgoinglink.close(linkargs, newblock) self.block = newblock - self.outgoinglink = rgenop.nulllink + self.outgoinglink = None def leave_block(self): - self.outgoinglink = rgenop.closeblock1(self.block) + self.outgoinglink = self.block.close1() def leave_block_split(self, exitgvar): - false_link, true_link = rgenop.closeblock2(self.block, exitgvar) - later_builder = ResidualGraphBuilder(link=false_link) + false_link, true_link = self.block.close2(exitgvar) + later_builder = ResidualGraphBuilder(self.rgenop, link=false_link) self.outgoinglink = true_link return later_builder def finish_and_goto(self, linkargs, targetblock): - rgenop.closelink(self.outgoinglink, linkargs, targetblock) - self.outgoinglink = rgenop.nulllink + self.outgoinglink.close(linkargs, targetblock) + self.outgoinglink = None def finish_and_return(self): gv_retval = self.valuebox.getgenvar(self) - returnlink = rgenop.closeblock1(self.block) - rgenop.closereturnlink(returnlink, gv_retval) - - def clone(self): - XXX + returnlink = self.block.close1() + returnlink.closereturn(gv_retval) -def ll_make_builder(): - return ResidualGraphBuilder(rgenop.newblock()) +def make_builder(rgenop): + return ResidualGraphBuilder(rgenop, rgenop.newblock()) def ll_int_box(gv_type, gv): return rvalue.IntRedBox(gv_type, gv) @@ -340,7 +358,7 @@ return rvalue.PtrRedBox(gv_type, gv) def ll_geninputarg(builder, gv_TYPE): - return rgenop.geninputarg(builder.block, gv_TYPE) + return builder.block.geninputarg(gv_TYPE) def ll_end_setup_builder(builder): builder.leave_block() @@ -357,6 +375,7 @@ self.split_queue = [] self.return_queue = [] self.curbuilder = builder + self.rgenop = builder.rgenop self.backstate = backstate def extend_with_parent_locals(self, redboxes): Modified: pypy/dist/pypy/jit/timeshifter/rtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtyper.py Wed Aug 23 18:47:27 2006 @@ -8,7 +8,6 @@ from pypy.rpython.lltypesystem.rstr import string_repr from pypy.rpython.typesystem import LowLevelTypeSystem from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython import rgenop from pypy.jit.hintannotator import model as hintmodel from pypy.jit.hintannotator import container as hintcontainer from pypy.jit.timeshifter import rtimeshift, rvalue, rcontainer @@ -40,6 +39,7 @@ self.green_reprs = PRECOMPUTED_GREEN_REPRS.copy() self.red_reprs = {} self.timeshifter = timeshifter + self.RGenOp = timeshifter.RGenOp originalconcretetype = staticmethod(originalconcretetype) @@ -127,11 +127,11 @@ ts = self.timeshifter v_argbox, c_fieldname = hop.inputargs(self.getredrepr(PTRTYPE), green_void_repr) - fielddesc = rcontainer.StructTypeDesc(PTRTYPE.TO).getfielddesc(c_fieldname.value) + structdesc = rcontainer.StructTypeDesc(self.RGenOp, PTRTYPE.TO) + fielddesc = structdesc.getfielddesc(c_fieldname.value) c_fielddesc = inputconst(lltype.Void, fielddesc) s_fielddesc = ts.rtyper.annotator.bookkeeper.immutablevalue(fielddesc) v_jitstate = hop.llops.getjitstate() - s_CONSTORVAR = annmodel.SomePtr(rgenop.CONSTORVAR) return hop.llops.genmixlevelhelpercall(rtimeshift.ll_generate_getfield, [ts.s_JITState, s_fielddesc, ts.s_RedBox], [v_jitstate, c_fielddesc, v_argbox ], @@ -147,11 +147,10 @@ ts = self.timeshifter v_argbox, v_index = hop.inputargs(self.getredrepr(PTRTYPE), self.getredrepr(lltype.Signed)) - fielddesc = rcontainer.ArrayFieldDesc(PTRTYPE) + fielddesc = rcontainer.ArrayFieldDesc(self.RGenOp, PTRTYPE) c_fielddesc = inputconst(lltype.Void, fielddesc) s_fielddesc = ts.rtyper.annotator.bookkeeper.immutablevalue(fielddesc) v_jitstate = hop.llops.getjitstate() - s_CONSTORVAR = annmodel.SomePtr(rgenop.CONSTORVAR) return hop.llops.genmixlevelhelpercall(rtimeshift.ll_generate_getarrayitem, [ts.s_JITState, s_fielddesc, ts.s_RedBox, ts.s_RedBox], [v_jitstate, c_fielddesc, v_argbox, v_index ], @@ -168,11 +167,11 @@ green_void_repr, self.getredrepr(VALUETYPE) ) - fielddesc = rcontainer.StructTypeDesc(PTRTYPE.TO).getfielddesc(c_fieldname.value) + structdesc = rcontainer.StructTypeDesc(self.RGenOp, PTRTYPE.TO) + fielddesc = structdesc.getfielddesc(c_fieldname.value) c_fielddesc = inputconst(lltype.Void, fielddesc) s_fielddesc = ts.rtyper.annotator.bookkeeper.immutablevalue(fielddesc) v_jitstate = hop.llops.getjitstate() - s_CONSTORVAR = annmodel.SomePtr(rgenop.CONSTORVAR) return hop.llops.genmixlevelhelpercall(rtimeshift.ll_generate_setfield, [ts.s_JITState, s_fielddesc, ts.s_RedBox, ts.s_RedBox], [v_jitstate, c_fielddesc, v_destbox, v_valuebox], @@ -186,11 +185,11 @@ PTRTYPE = originalconcretetype(hop.args_s[0]) v_argbox, c_fieldname = hop.inputargs(self.getredrepr(PTRTYPE), green_void_repr) - fielddesc = rcontainer.NamedFieldDesc(PTRTYPE, c_fieldname.value) # XXX + fielddesc = rcontainer.NamedFieldDesc(self.RGenOp, PTRTYPE, + c_fieldname.value) # XXX c_fielddesc = inputconst(lltype.Void, fielddesc) s_fielddesc = ts.rtyper.annotator.bookkeeper.immutablevalue(fielddesc) v_jitstate = hop.llops.getjitstate() - s_CONSTORVAR = annmodel.SomePtr(rgenop.CONSTORVAR) return hop.llops.genmixlevelhelpercall(rtimeshift.ll_generate_getsubstruct, [ts.s_JITState, s_fielddesc, ts.s_RedBox], [v_jitstate, c_fielddesc, v_argbox ], @@ -257,7 +256,7 @@ def handle_highlevel_operation(self, fnobj, hop): from pypy.jit.timeshifter.oop import OopSpecDesc, Index - oopspecdesc = OopSpecDesc(fnobj) + oopspecdesc = OopSpecDesc(self.RGenOp, fnobj) args_v = [] for obj in oopspecdesc.argtuple: @@ -424,10 +423,11 @@ return llops.genmixlevelhelpercall(rtimeshift.ll_gvar_from_redbox, [ts.s_JITState, llops.timeshifter.s_RedBox], [v_jitstate, v], - annmodel.SomePtr(rgenop.CONSTORVAR)) + ts.s_ConstOrVar) def convert_const(self, ll_value): - redbox = rvalue.ll_fromvalue(ll_value) + RGenOp = self.timeshifter.RGenOp + redbox = rvalue.redbox_from_prebuilt_value(RGenOp, ll_value) timeshifter = self.timeshifter return timeshifter.annhelper.delayedconst(timeshifter.r_RedBox, redbox) @@ -439,10 +439,10 @@ typedesc = None def create(self, hop): + ts = self.timeshifter if self.typedesc is None: T = self.original_concretetype.TO - self.typedesc = rcontainer.StructTypeDesc(T) - ts = self.timeshifter + self.typedesc = rcontainer.StructTypeDesc(ts.RGenOp, T) return hop.llops.genmixlevelhelpercall(self.typedesc.ll_factory, [], [], ts.s_RedBox) @@ -597,9 +597,12 @@ return annmodel.SomeInteger() def get_genop_var(self, v, llops): + ts = self.timeshifter + v_jitstate = hop.llops.getjitstate() return llops.genmixlevelhelpercall(rtimeshift.ll_gvar_from_constant, - [self.annotation()], [v], - annmodel.SomePtr(rgenop.CONSTORVAR)) + [ts.s_JITState, self.annotation()], + [v_jitstate, v], + ts.s_ConstOrVar) def convert_const(self, ll_value): return ll_value @@ -624,9 +627,12 @@ def convert_from_to((r_from, r_to), v, llops): assert r_from.lowleveltype == r_to.original_concretetype + ts = llops.timeshifter + v_jitstate = llops.getjitstate() return llops.genmixlevelhelpercall(rvalue.ll_fromvalue, - [r_from.annotation()], [v], - llops.timeshifter.s_RedBox) + [ts.s_JITState, r_from.annotation()], + [v_jitstate, v], + llops.timeshifter.s_RedBox) # ____________________________________________________________ Modified: pypy/dist/pypy/jit/timeshifter/rvalue.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rvalue.py (original) +++ pypy/dist/pypy/jit/timeshifter/rvalue.py Wed Aug 23 18:47:27 2006 @@ -1,4 +1,3 @@ -from pypy.rpython import rgenop from pypy.rpython.lltypesystem import lltype, llmemory class Memo(object): @@ -14,8 +13,8 @@ def freeze_memo(): return Memo() -def unfreeze_memo(): - return {} # Memo() +##def unfreeze_memo(): +## return {} # Memo() def exactmatch_memo(): return Memo() @@ -26,19 +25,20 @@ class RedBox(object): - def __init__(self, gv_type, genvar=rgenop.nullvar): - assert rgenop.isconst(gv_type) # temporary? + def __init__(self, gv_type, genvar=None): + assert gv_type.is_const self.gv_type = gv_type - self.genvar = genvar # nullvar or a genvar + self.genvar = genvar # None or a genvar def __repr__(self): if not self.genvar: return '' else: + from pypy.jit.codegen.llgraph.rgenop import rgenop return '<%r>' % (rgenop.reveal(self.genvar),) def is_constant(self): - return bool(self.genvar) and rgenop.isconst(self.genvar) + return bool(self.genvar) and self.genvar.is_const def getgenvar(self, builder): return self.genvar @@ -48,7 +48,7 @@ if not self.is_constant() and self not in memo: incoming.append(self.genvar) memo[self] = None - self.genvar = rgenop.geninputarg(newblock, self.gv_type) + self.genvar = newblock.geninputarg(self.gv_type) def replace(self, memo): memo = memo.boxes @@ -83,17 +83,25 @@ # XXX what about long longs? return redboxbuilder_int -def ll_fromvalue(value): +def ll_fromvalue(jitstate, value): "Make a constant RedBox from a low-level value." + rgenop = jitstate.rgenop T = lltype.typeOf(value) gv_type = rgenop.constTYPE(T) gv = rgenop.genconst(value) cls = ll_redboxcls(T) return cls(gv_type, gv) +def redbox_from_prebuilt_value(RGenOp, value): + T = lltype.typeOf(value) + gv_type = RGenOp.constTYPE(T) + gv = RGenOp.constPrebuiltGlobal(value) + cls = ll_redboxcls(T) + return cls(gv_type, gv) + def ll_getvalue(box, T): "Return the content of a known-to-be-constant RedBox." - return rgenop.revealconst(T, box.genvar) + return box.genvar.revealconst(T) class IntRedBox(RedBox): @@ -216,17 +224,17 @@ def __init__(self, gv_const): self.gv_const = gv_const - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - box = memo[self] = IntRedBox(gv_type, self.gv_const) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## box = memo[self] = IntRedBox(gv_type, self.gv_const) +## return box def exactmatch(self, box, outgoingvarboxes, memo): if (box.is_constant() and - rgenop.revealconst(lltype.Signed, self.gv_const) == - rgenop.revealconst(lltype.Signed, box.genvar)): + self.gv_const.revealconst(lltype.Signed) == + box.genvar.revealconst(lltype.Signed)): return True else: outgoingvarboxes.append(box) @@ -235,13 +243,13 @@ class FrozenIntVar(FrozenValue): - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - gv_value = rgenop.geninputarg(block, gv_type) - box = memo[self] = IntRedBox(gv_type, gv_value) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## gv_value = rgenop.geninputarg(block, gv_type) +## box = memo[self] = IntRedBox(gv_type, gv_value) +## return box def exactmatch(self, box, outgoingvarboxes, memo): memo = memo.boxes @@ -261,17 +269,17 @@ def __init__(self, gv_const): self.gv_const = gv_const - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - box = memo[self] = DoubleRedBox(gv_type, self.gv_const) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## box = memo[self] = DoubleRedBox(gv_type, self.gv_const) +## return box def exactmatch(self, box, outgoingvarboxes, memo): if (box.is_constant() and - rgenop.revealconst(lltype.Float, self.gv_const) == - rgenop.revealconst(lltype.Float, box.genvar)): + self.gv_const.revealconst(lltype.Float) == + box.genvar.revealconst(lltype.Float)): return True else: outgoingvarboxes.append(box) @@ -280,13 +288,13 @@ class FrozenDoubleVar(FrozenValue): - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - gv_value = rgenop.geninputarg(block, gv_type) - box = memo[self] = DoubleRedBox(gv_type, gv_value) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## gv_value = rgenop.geninputarg(block, gv_type) +## box = memo[self] = DoubleRedBox(gv_type, gv_value) +## return box def exactmatch(self, box, outgoingvarboxes, memo): memo = memo.boxes @@ -306,17 +314,17 @@ def __init__(self, gv_const): self.gv_const = gv_const - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - box = memo[self] = PtrRedBox(gv_type, self.gv_const) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## box = memo[self] = PtrRedBox(gv_type, self.gv_const) +## return box def exactmatch(self, box, outgoingvarboxes, memo): if (box.is_constant() and - rgenop.revealconst(llmemory.Address, self.gv_const) == - rgenop.revealconst(llmemory.Address, box.genvar)): + self.gv_const.revealconst(llmemory.Address) == + box.genvar.revealconst(llmemory.Address)): return True else: outgoingvarboxes.append(box) @@ -325,13 +333,13 @@ class FrozenPtrVar(FrozenValue): - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - gv_value = rgenop.geninputarg(block, gv_type) - box = memo[self] = PtrRedBox(gv_type, gv_value) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## gv_value = rgenop.geninputarg(block, gv_type) +## box = memo[self] = PtrRedBox(gv_type, gv_value) +## return box def exactmatch(self, box, outgoingvarboxes, memo): memo = memo.boxes @@ -348,13 +356,13 @@ class FrozenPtrVirtual(FrozenValue): - def unfreeze(self, memo, block, gv_type): - try: - return memo[self] - except KeyError: - box = memo[self] = PtrRedBox(gv_type) - box.content = self.fz_content.unfreeze(memo, block) - return box +## def unfreeze(self, memo, block, gv_type): +## try: +## return memo[self] +## except KeyError: +## box = memo[self] = PtrRedBox(gv_type) +## box.content = self.fz_content.unfreeze(memo, block) +## return box def exactmatch(self, box, outgoingvarboxes, memo): assert isinstance(box, PtrRedBox) Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Wed Aug 23 18:47:27 2006 @@ -9,7 +9,6 @@ from pypy.jit.llabstractinterp.test.test_llabstractinterp import summary from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.objectmodel import hint, keepalive_until_here -from pypy.rpython import rgenop from pypy.rpython.lltypesystem import rstr from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter @@ -45,547 +44,533 @@ hannotator.translator.view() return hs, hannotator, rtyper -_cache = {} -_cache_order = [] -def timeshift_cached(ll_function, values, inline, policy): - key = ll_function, inline, policy - try: - result, argtypes = _cache[key] - except KeyError: - if len(_cache_order) >= 3: - del _cache[_cache_order.pop(0)] - hs, ha, rtyper = hannotate(ll_function, values, - inline=inline, policy=policy) - htshift = HintTimeshift(ha, rtyper) - htshift.timeshift() - t = rtyper.annotator.translator - for graph in ha.translator.graphs: - checkgraph(graph) - t.graphs.append(graph) - if conftest.option.view: - from pypy.translator.tool.graphpage import FlowGraphPage - FlowGraphPage(t, ha.translator.graphs).display() - result = hs, ha, rtyper, htshift - _cache[key] = result, getargtypes(rtyper.annotator, values) - _cache_order.append(key) - else: - hs, ha, rtyper, htshift = result - assert argtypes == getargtypes(rtyper.annotator, values) - return result - -def timeshift(ll_function, values, opt_consts=[], inline=None, policy=None): - hs, ha, rtyper, htshift = timeshift_cached(ll_function, values, - inline, policy) - # run the time-shifted graph-producing graphs - graph1 = ha.translator.graphs[0] - llinterp = LLInterpreter(rtyper) - builder = llinterp.eval_graph(htshift.ll_make_builder_graph, []) - graph1args = [builder, lltype.nullptr(htshift.r_JITState.lowleveltype.TO)] - residual_graph_args = [] - assert len(graph1.getargs()) == 2 + len(values) - for i, (v, llvalue) in enumerate(zip(graph1.getargs()[2:], values)): - r = htshift.hrtyper.bindingrepr(v) - residual_v = r.residual_values(llvalue) - if len(residual_v) == 0: - # green - graph1args.append(llvalue) +class TimeshiftingTests(object): + from pypy.jit.codegen.llgraph.rgenop import RGenOp + + def setup_class(cls): + cls._cache = {} + cls._cache_order = [] + + def teardown_class(cls): + del cls._cache + del cls._cache_order + + def timeshift_cached(self, ll_function, values, inline, policy): + key = ll_function, inline, policy + try: + result, argtypes = self._cache[key] + except KeyError: + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] + hs, ha, rtyper = hannotate(ll_function, values, + inline=inline, policy=policy) + htshift = HintTimeshift(ha, rtyper, self.RGenOp) + htshift.timeshift() + t = rtyper.annotator.translator + for graph in ha.translator.graphs: + checkgraph(graph) + t.graphs.append(graph) + if conftest.option.view: + from pypy.translator.tool.graphpage import FlowGraphPage + FlowGraphPage(t, ha.translator.graphs).display() + result = hs, ha, rtyper, htshift + self._cache[key] = result, getargtypes(rtyper.annotator, values) + self._cache_order.append(key) else: - # red - assert residual_v == [llvalue], "XXX for now" - TYPE = htshift.originalconcretetype(v) - gv_type = rgenop.constTYPE(TYPE) - gvar = llinterp.eval_graph(htshift.ll_geninputarg_graph, [builder, - gv_type]) - if i in opt_consts: # XXX what should happen here interface wise is unclear - gvar = rgenop.genconst(llvalue) - if isinstance(lltype.typeOf(llvalue), lltype.Ptr): - ll_box_graph = htshift.ll_addr_box_graph - elif isinstance(llvalue, float): - ll_box_graph = htshift.ll_double_box_graph + hs, ha, rtyper, htshift = result + assert argtypes == getargtypes(rtyper.annotator, values) + return result + + def timeshift(self, ll_function, values, opt_consts=[], inline=None, + policy=None): + hs, ha, rtyper, htshift = self.timeshift_cached(ll_function, values, + inline, policy) + # run the time-shifted graph-producing graphs + graph1 = ha.translator.graphs[0] + llinterp = LLInterpreter(rtyper) + builder = llinterp.eval_graph(htshift.ll_make_builder_graph, []) + graph1args = [builder, lltype.nullptr(htshift.r_JITState.lowleveltype.TO)] + residual_graph_args = [] + assert len(graph1.getargs()) == 2 + len(values) + for i, (v, llvalue) in enumerate(zip(graph1.getargs()[2:], values)): + r = htshift.hrtyper.bindingrepr(v) + residual_v = r.residual_values(llvalue) + if len(residual_v) == 0: + # green + graph1args.append(llvalue) else: - ll_box_graph = htshift.ll_int_box_graph - box = llinterp.eval_graph(ll_box_graph, [gv_type, gvar]) - graph1args.append(box) - residual_graph_args.append(llvalue) - startblock = llinterp.eval_graph(htshift.ll_end_setup_builder_graph, [builder]) - - builder = llinterp.eval_graph(graph1, graph1args) - r = htshift.hrtyper.getrepr(hs) - llinterp.eval_graph(htshift.ll_close_builder_graph, [builder]) - - # now try to run the blocks produced by the builder - residual_graph = rgenop.buildgraph(startblock) - insns = summary(residual_graph) - res = rgenop.testgengraph(residual_graph, residual_graph_args, - viewbefore = conftest.option.view) - return insns, res - -##def test_ll_get_return_queue(): -## t = TranslationContext() -## a = t.buildannotator() -## rtyper = t.buildrtyper() -## rtyper.specialize() # XXX - -## htshift = HintTimeshift(None, rtyper) - -## questate = htshift.QUESTATE_PTR.TO.ll_newstate() - -## def llf(questate): -## return questate.ll_get_return_queue() - -## from pypy.rpython import annlowlevel - -## graph = annlowlevel.annotate_mixlevel_helper(rtyper, llf, [ -## annmodel.SomePtr(htshift.QUESTATE_PTR)]) - -## s = a.binding(graph.getreturnvar()) - -## assert s == htshift.s_return_queue - -## rtyper.specialize_more_blocks() - -## llinterp = LLInterpreter(rtyper) -## rq = llinterp.eval_graph(graph, [questate]) -## assert lltype.typeOf(rq) == rtyper.getrepr(s).lowleveltype - - -def test_simple_fixed(): - py.test.skip("green return not working") - def ll_function(x, y): - return hint(x + y, concrete=True) - insns, res = timeshift(ll_function, [5, 7]) - assert res == 12 - assert insns == {} - -def test_very_simple(): - def ll_function(x, y): - return x + y - insns, res = timeshift(ll_function, [5, 7]) - assert res == 12 - assert insns == {'int_add': 1} - -def test_convert_const_to_redbox(): - def ll_function(x, y): - x = hint(x, concrete=True) - tot = 0 - while x: # conversion from green '0' to red 'tot' - tot += y - x -= 1 - return tot - insns, res = timeshift(ll_function, [7, 2]) - assert res == 14 - assert insns == {'int_add': 7} - -def test_simple_opt_const_propagation2(): - def ll_function(x, y): - return x + y - insns, res = timeshift(ll_function, [5, 7], [0, 1]) - assert res == 12 - assert insns == {} - -def test_simple_opt_const_propagation1(): - def ll_function(x): - return -x - insns, res = timeshift(ll_function, [5], [0]) - assert res == -5 - assert insns == {} - -def test_loop_folding(): - def ll_function(x, y): - tot = 0 - x = hint(x, concrete=True) - while x: - tot += y - x -= 1 - return tot - insns, res = timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - assert insns == {} - -def test_loop_merging(): - def ll_function(x, y): - tot = 0 - while x: - tot += y + # red + assert residual_v == [llvalue], "XXX for now" + TYPE = htshift.originalconcretetype(v) + gv_type = self.RGenOp.constTYPE(TYPE) + ll_type = htshift.r_ConstOrVar.convert_const(gv_type) + ll_gvar = llinterp.eval_graph(htshift.ll_geninputarg_graph, + [builder, ll_type]) + if i in opt_consts: # XXX what should happen here interface wise is unclear + gvar = self.RGenOp.constPrebuiltGlobal(llvalue) + ll_gvar = htshift.r_ConstOrVar.convert_const(gvar) + if isinstance(lltype.typeOf(llvalue), lltype.Ptr): + ll_box_graph = htshift.ll_addr_box_graph + elif isinstance(llvalue, float): + ll_box_graph = htshift.ll_double_box_graph + else: + ll_box_graph = htshift.ll_int_box_graph + box = llinterp.eval_graph(ll_box_graph, [ll_type, ll_gvar]) + graph1args.append(box) + residual_graph_args.append(llvalue) + startblock = llinterp.eval_graph(htshift.ll_end_setup_builder_graph, [builder]) + + builder = llinterp.eval_graph(graph1, graph1args) + r = htshift.hrtyper.getrepr(hs) + llinterp.eval_graph(htshift.ll_close_builder_graph, [builder]) + + # now try to run the blocks produced by the builder + residual_graph = self.RGenOp.buildgraph(startblock) + insns = summary(residual_graph) + res = self.RGenOp.testgengraph(residual_graph, residual_graph_args, + viewbefore = conftest.option.view) + return insns, res + + +class TestTimeshift(TimeshiftingTests): + + def test_simple_fixed(self): + py.test.skip("green return not working") + def ll_function(x, y): + return hint(x + y, concrete=True) + insns, res = self.timeshift(ll_function, [5, 7]) + assert res == 12 + assert insns == {} + + def test_very_simple(self): + def ll_function(x, y): + return x + y + insns, res = self.timeshift(ll_function, [5, 7]) + assert res == 12 + assert insns == {'int_add': 1} + + def test_convert_const_to_redbox(self): + def ll_function(x, y): + x = hint(x, concrete=True) + tot = 0 + while x: # conversion from green '0' to red 'tot' + tot += y + x -= 1 + return tot + insns, res = self.timeshift(ll_function, [7, 2]) + assert res == 14 + assert insns == {'int_add': 7} + + def test_simple_opt_const_propagation2(self): + def ll_function(x, y): + return x + y + insns, res = self.timeshift(ll_function, [5, 7], [0, 1]) + assert res == 12 + assert insns == {} + + def test_simple_opt_const_propagation1(self): + def ll_function(x): + return -x + insns, res = self.timeshift(ll_function, [5], [0]) + assert res == -5 + assert insns == {} + + def test_loop_folding(self): + def ll_function(x, y): + tot = 0 + x = hint(x, concrete=True) + while x: + tot += y + x -= 1 + return tot + insns, res = self.timeshift(ll_function, [7, 2], [0, 1]) + assert res == 14 + assert insns == {} + + def test_loop_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + return tot + insns, res = self.timeshift(ll_function, [7, 2], []) + assert res == 14 + assert insns['int_add'] == 2 + assert insns['int_is_true'] == 2 + + insns, res = self.timeshift(ll_function, [7, 2], [0]) + assert res == 14 + assert insns['int_add'] == 2 + assert insns['int_is_true'] == 1 + + insns, res = self.timeshift(ll_function, [7, 2], [1]) + assert res == 14 + assert insns['int_add'] == 1 + assert insns['int_is_true'] == 2 + + insns, res = self.timeshift(ll_function, [7, 2], [0, 1]) + assert res == 14 + assert insns['int_add'] == 1 + assert insns['int_is_true'] == 1 + + def test_two_loops_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + while y: + tot += y + y -= 1 + return tot + insns, res = self.timeshift(ll_function, [7, 3], []) + assert res == 27 + assert insns['int_add'] == 3 + assert insns['int_is_true'] == 3 + + def test_convert_greenvar_to_redvar(self): + def ll_function(x, y): + hint(x, concrete=True) + return x - y + insns, res = self.timeshift(ll_function, [70, 4], [0]) + assert res == 66 + assert insns['int_sub'] == 1 + insns, res = self.timeshift(ll_function, [70, 4], [0, 1]) + assert res == 66 + assert insns == {} + + def test_green_across_split(self): + def ll_function(x, y): + hint(x, concrete=True) + if y > 2: + z = x - y + else: + z = x + y + return z + insns, res = self.timeshift(ll_function, [70, 4], [0]) + assert res == 66 + assert insns['int_add'] == 1 + assert insns['int_sub'] == 1 + + def test_merge_const_before_return(self): + def ll_function(x): + if x > 0: + y = 17 + else: + y = 22 x -= 1 - return tot - insns, res = timeshift(ll_function, [7, 2], []) - assert res == 14 - assert insns['int_add'] == 2 - assert insns['int_is_true'] == 2 - - insns, res = timeshift(ll_function, [7, 2], [0]) - assert res == 14 - assert insns['int_add'] == 2 - assert insns['int_is_true'] == 1 - - insns, res = timeshift(ll_function, [7, 2], [1]) - assert res == 14 - assert insns['int_add'] == 1 - assert insns['int_is_true'] == 2 - - insns, res = timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - assert insns['int_add'] == 1 - assert insns['int_is_true'] == 1 - -def test_two_loops_merging(): - def ll_function(x, y): - tot = 0 - while x: - tot += y + y += 1 + return y+x + insns, res = self.timeshift(ll_function, [-70], []) + assert res == 23-71 + assert insns == {'int_gt': 1, 'int_add': 2, 'int_sub': 2} + + def test_merge_3_redconsts_before_return(self): + def ll_function(x): + if x > 2: + y = hint(54, variable=True) + elif x > 0: + y = hint(17, variable=True) + else: + y = hint(22, variable=True) x -= 1 - while y: - tot += y - y -= 1 - return tot - insns, res = timeshift(ll_function, [7, 3], []) - assert res == 27 - assert insns['int_add'] == 3 - assert insns['int_is_true'] == 3 - -def test_convert_greenvar_to_redvar(): - def ll_function(x, y): - hint(x, concrete=True) - return x - y - insns, res = timeshift(ll_function, [70, 4], [0]) - assert res == 66 - assert insns['int_sub'] == 1 - insns, res = timeshift(ll_function, [70, 4], [0, 1]) - assert res == 66 - assert insns == {} - -def test_green_across_split(): - def ll_function(x, y): - hint(x, concrete=True) - if y > 2: - z = x - y - else: - z = x + y - return z - insns, res = timeshift(ll_function, [70, 4], [0]) - assert res == 66 - assert insns['int_add'] == 1 - assert insns['int_sub'] == 1 - -def test_merge_const_before_return(): - def ll_function(x): - if x > 0: - y = 17 - else: - y = 22 - x -= 1 - y += 1 - return y+x - insns, res = timeshift(ll_function, [-70], []) - assert res == 23-71 - assert insns == {'int_gt': 1, 'int_add': 2, 'int_sub': 2} - -def test_merge_3_redconsts_before_return(): - def ll_function(x): - if x > 2: - y = hint(54, variable=True) - elif x > 0: - y = hint(17, variable=True) - else: - y = hint(22, variable=True) - x -= 1 - y += 1 - return y+x - insns, res = timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - insns, res = timeshift(ll_function, [1], []) - assert res == ll_function(1) - insns, res = timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - -def test_merge_const_at_return(): - py.test.skip("green return") - def ll_function(x): - if x > 0: - return 17 - else: - return 22 - insns, res = timeshift(ll_function, [-70], []) - assert res == 22 - assert insns == {'int_gt': 1} - -def test_arith_plus_minus(): - def ll_plus_minus(encoded_insn, nb_insn, x, y): - acc = x - pc = 0 - while pc < nb_insn: - op = (encoded_insn >> (pc*4)) & 0xF - op = hint(op, concrete=True) - if op == 0xA: - acc += y - elif op == 0x5: - acc -= y - pc += 1 - return acc - assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - insns, res = timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) - assert res == 42 - assert insns == {'int_add': 2, - 'int_sub': 1} - -def test_simple_struct(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed), - hints={'immutable': True}) - def ll_function(s): - return s.hello * s.world - s1 = lltype.malloc(S) - s1.hello = 6 - s1.world = 7 - insns, res = timeshift(ll_function, [s1], []) - assert res == 42 - assert insns == {'getfield': 2, - 'int_mul': 1} - insns, res = timeshift(ll_function, [s1], [0]) - assert res == 42 - assert insns == {} - -def test_simple_array(): - A = lltype.GcArray(lltype.Signed, - hints={'immutable': True}) - def ll_function(a): - return a[0] * a[1] - a1 = lltype.malloc(A, 2) - a1[0] = 6 - a1[1] = 7 - insns, res = timeshift(ll_function, [a1], []) - assert res == 42 - assert insns == {'getarrayitem': 2, - 'int_mul': 1} - insns, res = timeshift(ll_function, [a1], [0]) - assert res == 42 - assert insns == {} - -def test_simple_struct_malloc(): - py.test.skip("blue containers: to be reimplemented") - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(x): - s = lltype.malloc(S) - s.hello = x - return s.hello + s.world - - insns, res = timeshift(ll_function, [3], []) - assert res == 3 - assert insns == {'int_add': 1} - - insns, res = timeshift(ll_function, [3], [0]) - assert res == 3 - assert insns == {} - -def test_inlined_substructure(): - py.test.skip("blue containers: to be reimplemented") - S = lltype.Struct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - def ll_function(k): - t = lltype.malloc(T) - t.s.n = k - l = t.s.n - return l - insns, res = timeshift(ll_function, [7], []) - assert res == 7 - assert insns == {} - - insns, res = timeshift(ll_function, [7], [0]) - assert res == 7 - assert insns == {} - -def test_degenerated_before_return(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - s.n += 1 - return s.n * t.s.n - insns, res = timeshift(ll_function, [0], []) - assert res == 5 * 3 - insns, res = timeshift(ll_function, [1], []) - assert res == 4 * 4 - -def test_degenerated_before_return_2(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - pass - else: - s = t.s - s.n += 1 - return s.n * t.s.n - insns, res = timeshift(ll_function, [1], []) - assert res == 5 * 3 - insns, res = timeshift(ll_function, [0], []) - assert res == 4 * 4 - -def test_degenerated_at_return(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.n = 3.25 - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - return s - insns, res = timeshift(ll_function, [0], []) - assert res.n == 4 - assert lltype.parentlink(res._obj) == (None, None) - insns, res = timeshift(ll_function, [1], []) - assert res.n == 3 - parent, parentindex = lltype.parentlink(res._obj) - assert parentindex == 's' - assert parent.n == 3.25 - -def test_degenerated_via_substructure(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 7 - if flag: - pass - else: - s = t.s - t.s.n += 1 - return s.n * t.s.n - insns, res = timeshift(ll_function, [1], []) - assert res == 7 * 4 - insns, res = timeshift(ll_function, [0], []) - assert res == 4 * 4 - -def test_plus_minus_all_inlined(): - def ll_plus_minus(s, x, y): - acc = x - n = len(s) - pc = 0 - while pc < n: - op = s[pc] - op = hint(op, concrete=True) - if op == '+': - acc += y - elif op == '-': - acc -= y - pc += 1 - return acc - s = rstr.string_repr.convert_const("+-+") - insns, res = timeshift(ll_plus_minus, [s, 0, 2], [0], inline=999) - assert res == ll_plus_minus("+-+", 0, 2) - assert insns == {'int_add': 2, 'int_sub': 1} - -def test_red_virtual_container(): - # this checks that red boxes are able to be virtualized dynamically by - # the compiler (the P_NOVIRTUAL policy prevents the hint-annotator from - # marking variables in blue) - S = lltype.GcStruct('S', ('n', lltype.Signed)) - def ll_function(n): - s = lltype.malloc(S) - s.n = n - return s.n - insns, res = timeshift(ll_function, [42], [], policy=P_NOVIRTUAL) - assert res == 42 - assert insns == {} - -def test_red_propagate(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - def ll_function(n, k): - s = lltype.malloc(S) - s.n = n - if k < 0: - return -123 - return s.n * k - insns, res = timeshift(ll_function, [3, 8], [], policy=P_NOVIRTUAL) - assert res == 24 - assert insns == {'int_lt': 1, 'int_mul': 1} - -def test_red_subcontainer(): - S = lltype.Struct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - def ll_function(k): - t = lltype.malloc(T) - s = t.s - s.n = k - if k < 0: - return -123 - result = s.n * (k-1) - keepalive_until_here(t) - return result - insns, res = timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) - assert res == 42 - assert insns == {'int_lt': 1, 'int_mul': 1, 'int_sub': 1} - -def test_merge_structures(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', lltype.Ptr(S)), ('n', lltype.Signed)) + y += 1 + return y+x + insns, res = self.timeshift(ll_function, [-70], []) + assert res == ll_function(-70) + insns, res = self.timeshift(ll_function, [1], []) + assert res == ll_function(1) + insns, res = self.timeshift(ll_function, [-70], []) + assert res == ll_function(-70) + + def test_merge_const_at_return(self): + py.test.skip("green return") + def ll_function(x): + if x > 0: + return 17 + else: + return 22 + insns, res = self.timeshift(ll_function, [-70], []) + assert res == 22 + assert insns == {'int_gt': 1} + + def test_arith_plus_minus(self): + def ll_plus_minus(encoded_insn, nb_insn, x, y): + acc = x + pc = 0 + while pc < nb_insn: + op = (encoded_insn >> (pc*4)) & 0xF + op = hint(op, concrete=True) + if op == 0xA: + acc += y + elif op == 0x5: + acc -= y + pc += 1 + return acc + assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 + insns, res = self.timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) + assert res == 42 + assert insns == {'int_add': 2, + 'int_sub': 1} + + def test_simple_struct(self): + S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed), + hints={'immutable': True}) + def ll_function(s): + return s.hello * s.world + s1 = lltype.malloc(S) + s1.hello = 6 + s1.world = 7 + insns, res = self.timeshift(ll_function, [s1], []) + assert res == 42 + assert insns == {'getfield': 2, + 'int_mul': 1} + insns, res = self.timeshift(ll_function, [s1], [0]) + assert res == 42 + assert insns == {} + + def test_simple_array(self): + A = lltype.GcArray(lltype.Signed, + hints={'immutable': True}) + def ll_function(a): + return a[0] * a[1] + a1 = lltype.malloc(A, 2) + a1[0] = 6 + a1[1] = 7 + insns, res = self.timeshift(ll_function, [a1], []) + assert res == 42 + assert insns == {'getarrayitem': 2, + 'int_mul': 1} + insns, res = self.timeshift(ll_function, [a1], [0]) + assert res == 42 + assert insns == {} + + def test_simple_struct_malloc(self): + py.test.skip("blue containers: to be reimplemented") + S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + def ll_function(x): + s = lltype.malloc(S) + s.hello = x + return s.hello + s.world - def ll_function(flag): - if flag: + insns, res = self.timeshift(ll_function, [3], []) + assert res == 3 + assert insns == {'int_add': 1} + + insns, res = self.timeshift(ll_function, [3], [0]) + assert res == 3 + assert insns == {} + + def test_inlined_substructure(self): + py.test.skip("blue containers: to be reimplemented") + S = lltype.Struct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + def ll_function(k): + t = lltype.malloc(T) + t.s.n = k + l = t.s.n + return l + insns, res = self.timeshift(ll_function, [7], []) + assert res == 7 + assert insns == {} + + insns, res = self.timeshift(ll_function, [7], [0]) + assert res == 7 + assert insns == {} + + def test_degenerated_before_return(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 s = lltype.malloc(S) - s.n = 1 + s.n = 4 + if flag: + s = t.s + s.n += 1 + return s.n * t.s.n + insns, res = self.timeshift(ll_function, [0], []) + assert res == 5 * 3 + insns, res = self.timeshift(ll_function, [1], []) + assert res == 4 * 4 + + def test_degenerated_before_return_2(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): t = lltype.malloc(T) - t.s = s - t.n = 2 - else: + t.s.n = 3 s = lltype.malloc(S) - s.n = 5 + s.n = 4 + if flag: + pass + else: + s = t.s + s.n += 1 + return s.n * t.s.n + insns, res = self.timeshift(ll_function, [1], []) + assert res == 5 * 3 + insns, res = self.timeshift(ll_function, [0], []) + assert res == 4 * 4 + + def test_degenerated_at_return(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): t = lltype.malloc(T) - t.s = s - t.n = 6 - return t.n + t.s.n - insns, res = timeshift(ll_function, [0], [], policy=P_NOVIRTUAL) - assert res == 5 + 6 - assert insns == {'int_is_true': 1, 'int_add': 1} - insns, res = timeshift(ll_function, [1], [], policy=P_NOVIRTUAL) - assert res == 1 + 2 - assert insns == {'int_is_true': 1, 'int_add': 1} - -def test_call_simple(): - def ll_add_one(x): - return x + 1 - def ll_function(y): - return ll_add_one(y) - insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 6 - assert insns == {'int_add': 1} - -def test_call_2(): - def ll_add_one(x): - return x + 1 - def ll_function(y): - return ll_add_one(y) + y - insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 11 - assert insns == {'int_add': 2} - -def test_call_3(): - def ll_add_one(x): - return x + 1 - def ll_two(x): - return ll_add_one(ll_add_one(x)) - x - def ll_function(y): - return ll_two(y) * y - insns, res = timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 10 - assert insns == {'int_add': 2, 'int_sub': 1, 'int_mul': 1} + t.n = 3.25 + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + s = t.s + return s + insns, res = self.timeshift(ll_function, [0], []) + assert res.n == 4 + assert lltype.parentlink(res._obj) == (None, None) + insns, res = self.timeshift(ll_function, [1], []) + assert res.n == 3 + parent, parentindex = lltype.parentlink(res._obj) + assert parentindex == 's' + assert parent.n == 3.25 + + def test_degenerated_via_substructure(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 7 + if flag: + pass + else: + s = t.s + t.s.n += 1 + return s.n * t.s.n + insns, res = self.timeshift(ll_function, [1], []) + assert res == 7 * 4 + insns, res = self.timeshift(ll_function, [0], []) + assert res == 4 * 4 + + def test_plus_minus_all_inlined(self): + def ll_plus_minus(s, x, y): + acc = x + n = len(s) + pc = 0 + while pc < n: + op = s[pc] + op = hint(op, concrete=True) + if op == '+': + acc += y + elif op == '-': + acc -= y + pc += 1 + return acc + s = rstr.string_repr.convert_const("+-+") + insns, res = self.timeshift(ll_plus_minus, [s, 0, 2], [0], inline=999) + assert res == ll_plus_minus("+-+", 0, 2) + assert insns == {'int_add': 2, 'int_sub': 1} + + def test_red_virtual_container(self): + # this checks that red boxes are able to be virtualized dynamically by + # the compiler (the P_NOVIRTUAL policy prevents the hint-annotator from + # marking variables in blue) + S = lltype.GcStruct('S', ('n', lltype.Signed)) + def ll_function(n): + s = lltype.malloc(S) + s.n = n + return s.n + insns, res = self.timeshift(ll_function, [42], [], policy=P_NOVIRTUAL) + assert res == 42 + assert insns == {} + + def test_red_propagate(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + def ll_function(n, k): + s = lltype.malloc(S) + s.n = n + if k < 0: + return -123 + return s.n * k + insns, res = self.timeshift(ll_function, [3, 8], [], policy=P_NOVIRTUAL) + assert res == 24 + assert insns == {'int_lt': 1, 'int_mul': 1} + + def test_red_subcontainer(self): + S = lltype.Struct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + def ll_function(k): + t = lltype.malloc(T) + s = t.s + s.n = k + if k < 0: + return -123 + result = s.n * (k-1) + keepalive_until_here(t) + return result + insns, res = self.timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) + assert res == 42 + assert insns == {'int_lt': 1, 'int_mul': 1, 'int_sub': 1} + + def test_merge_structures(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', lltype.Ptr(S)), ('n', lltype.Signed)) + + def ll_function(flag): + if flag: + s = lltype.malloc(S) + s.n = 1 + t = lltype.malloc(T) + t.s = s + t.n = 2 + else: + s = lltype.malloc(S) + s.n = 5 + t = lltype.malloc(T) + t.s = s + t.n = 6 + return t.n + t.s.n + insns, res = self.timeshift(ll_function, [0], [], policy=P_NOVIRTUAL) + assert res == 5 + 6 + assert insns == {'int_is_true': 1, 'int_add': 1} + insns, res = self.timeshift(ll_function, [1], [], policy=P_NOVIRTUAL) + assert res == 1 + 2 + assert insns == {'int_is_true': 1, 'int_add': 1} + + def test_call_simple(self): + def ll_add_one(x): + return x + 1 + def ll_function(y): + return ll_add_one(y) + insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + assert res == 6 + assert insns == {'int_add': 1} + + def test_call_2(self): + def ll_add_one(x): + return x + 1 + def ll_function(y): + return ll_add_one(y) + y + insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + assert res == 11 + assert insns == {'int_add': 2} + + def test_call_3(self): + def ll_add_one(x): + return x + 1 + def ll_two(x): + return ll_add_one(ll_add_one(x)) - x + def ll_function(y): + return ll_two(y) * y + insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + assert res == 10 + assert insns == {'int_add': 2, 'int_sub': 1, 'int_mul': 1} Modified: pypy/dist/pypy/jit/timeshifter/test/test_tl.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_tl.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_tl.py Wed Aug 23 18:47:27 2006 @@ -1,15 +1,18 @@ from pypy.rpython.lltypesystem.rstr import string_repr -from pypy.jit.timeshifter.test.test_timeshift import timeshift +from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests from pypy.jit.timeshifter.test.test_vlist import P_OOPSPEC from pypy.jit.tl import tl from pypy.jit.tl.test.test_tl import FACTORIAL_SOURCE -def test_tl(): - import py; py.test.skip("in-progress") - code = tl.compile(FACTORIAL_SOURCE) - ll_code = string_repr.convert_const(code) - insns, res = timeshift(tl.interp_without_call, [ll_code, 0, 5], [0, 1], - policy=P_OOPSPEC) - assert res == 120 +class TestTL(TimeshiftingTests): + + def test_tl(self): + import py; py.test.skip("in-progress") + code = tl.compile(FACTORIAL_SOURCE) + ll_code = string_repr.convert_const(code) + insns, res = self.timeshift(tl.interp_without_call, + [ll_code, 0, 5], [0, 1], + policy=P_OOPSPEC) + assert res == 120 Modified: pypy/dist/pypy/jit/timeshifter/test/test_tlr.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_tlr.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_tlr.py Wed Aug 23 18:47:27 2006 @@ -1,11 +1,14 @@ from pypy.rpython.lltypesystem.rstr import string_repr -from pypy.jit.timeshifter.test.test_timeshift import timeshift +from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests from pypy.jit.timeshifter.test.test_vlist import P_OOPSPEC from pypy.jit.tl import tlr -def test_tlr(): - bytecode = string_repr.convert_const(tlr.SQUARE) - insns, res = timeshift(tlr.interpret, [bytecode, 9], [0], policy=P_OOPSPEC) - assert res == 81 +class TestTLR(TimeshiftingTests): + + def test_tlr(self): + bytecode = string_repr.convert_const(tlr.SQUARE) + insns, res = self.timeshift(tlr.interpret, [bytecode, 9], [0], + policy=P_OOPSPEC) + assert res == 81 Modified: pypy/dist/pypy/jit/timeshifter/test/test_vlist.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_vlist.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_vlist.py Wed Aug 23 18:47:27 2006 @@ -1,74 +1,76 @@ from pypy.annotation.policy import AnnotatorPolicy -from pypy.jit.timeshifter.test.test_timeshift import timeshift +from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests P_OOPSPEC = AnnotatorPolicy() P_OOPSPEC.novirtualcontainer = True P_OOPSPEC.oopspec = True -def test_vlist(): - def ll_function(): - lst = [] - lst.append(12) - return lst[0] - insns, res = timeshift(ll_function, [], [], policy=P_OOPSPEC) - assert res == 12 - assert insns == {} - -def test_enter_block(): - def ll_function(flag): - lst = [] - lst.append(flag) - lst.append(131) - if flag: +class TestVList(TimeshiftingTests): + + def test_vlist(self): + def ll_function(): + lst = [] + lst.append(12) return lst[0] - else: - return lst[1] - insns, res = timeshift(ll_function, [6], [], policy=P_OOPSPEC) - assert res == 6 - assert insns == {'int_is_true': 1} - insns, res = timeshift(ll_function, [0], [], policy=P_OOPSPEC) - assert res == 131 - assert insns == {'int_is_true': 1} - -def test_merge(): - def ll_function(flag): - lst = [] - if flag: + insns, res = self.timeshift(ll_function, [], [], policy=P_OOPSPEC) + assert res == 12 + assert insns == {} + + def test_enter_block(self): + def ll_function(flag): + lst = [] lst.append(flag) - else: lst.append(131) - return lst[-1] - insns, res = timeshift(ll_function, [6], [], policy=P_OOPSPEC) - assert res == 6 - assert insns == {'int_is_true': 1} - insns, res = timeshift(ll_function, [0], [], policy=P_OOPSPEC) - assert res == 131 - assert insns == {'int_is_true': 1} - -def test_replace(): - def ll_function(flag): - lst = [] - if flag: - lst.append(12) - else: - lst.append(131) - return lst[-1] - insns, res = timeshift(ll_function, [6], [], policy=P_OOPSPEC) - assert res == 12 - assert insns == {'int_is_true': 1} - insns, res = timeshift(ll_function, [0], [], policy=P_OOPSPEC) - assert res == 131 - assert insns == {'int_is_true': 1} - -def test_force(): - def ll_function(n): - lst = [] - lst.append(n) - if n: - lst.append(12) - return lst[-1] - insns, res = timeshift(ll_function, [6], [], policy=P_OOPSPEC) - assert res == 12 - insns, res = timeshift(ll_function, [0], [], policy=P_OOPSPEC) - assert res == 0 + if flag: + return lst[0] + else: + return lst[1] + insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + assert res == 6 + assert insns == {'int_is_true': 1} + insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + assert res == 131 + assert insns == {'int_is_true': 1} + + def test_merge(self): + def ll_function(flag): + lst = [] + if flag: + lst.append(flag) + else: + lst.append(131) + return lst[-1] + insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + assert res == 6 + assert insns == {'int_is_true': 1} + insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + assert res == 131 + assert insns == {'int_is_true': 1} + + def test_replace(self): + def ll_function(flag): + lst = [] + if flag: + lst.append(12) + else: + lst.append(131) + return lst[-1] + insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + assert res == 12 + assert insns == {'int_is_true': 1} + insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + assert res == 131 + assert insns == {'int_is_true': 1} + + def test_force(self): + def ll_function(n): + lst = [] + lst.append(n) + if n: + lst.append(12) + return lst[-1] + insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + assert res == 12 + insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + assert res == 0 Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Wed Aug 23 18:47:27 2006 @@ -1,25 +1,27 @@ -import py +import py, types from pypy.rpython.lltypesystem import lltype from pypy.objspace.flow import model as flowmodel from pypy.annotation import model as annmodel from pypy.annotation import listdef, dictdef from pypy.jit.timeshifter import rvalue, oop from pypy.jit.timeshifter.rtimeshift import JITState, ResidualGraphBuilder -from pypy.rpython import rmodel, rgenop, annlowlevel +from pypy.rpython import rmodel, annlowlevel from pypy.rpython.lltypesystem import rtuple, rlist, rdict from pypy.jit.timeshifter import rtimeshift from pypy.jit.timeshifter.rtyper import HintRTyper, originalconcretetype from pypy.jit.timeshifter.rtyper import GreenRepr, RedRepr, HintLowLevelOpList from pypy.translator.unsimplify import varoftype, copyvar from pypy.translator.backendopt import support +from pypy.jit.codegen import model as cgmodel # ___________________________________________________________ class HintTimeshift(object): - def __init__(self, hannotator, rtyper): + def __init__(self, hannotator, rtyper, RGenOp): self.hannotator = hannotator self.rtyper = rtyper + self.RGenOp = RGenOp self.hrtyper = HintRTyper(hannotator, self) self.latestexitindex = -1 self.annhelper = annlowlevel.MixLevelHelperAnnotator(rtyper) @@ -29,6 +31,9 @@ self.s_RedBox, self.r_RedBox = self.s_r_instanceof(rvalue.RedBox) self.s_OopSpecDesc, self.r_OopSpecDesc = self.s_r_instanceof( oop.OopSpecDesc) + self.s_ConstOrVar, self.r_ConstOrVar = self.s_r_instanceof( + cgmodel.GenVarOrConst) + self.s_Block, self.r_Block = self.s_r_instanceof(cgmodel.CodeGenBlock) getrepr = self.rtyper.getrepr @@ -46,29 +51,33 @@ self.r_box_accum = getrepr(self.s_box_accum) self.r_box_accum.setup() + def ll_make_builder(): + rgenop = RGenOp.get_rgenop_for_testing() + return rtimeshift.make_builder(rgenop) + self.ll_make_builder_graph = self.annhelper.getgraph( - rtimeshift.ll_make_builder, + ll_make_builder, [], self.s_ResidualGraphBuilder) self.ll_int_box_graph = self.annhelper.getgraph( rtimeshift.ll_int_box, - [rgenop.s_ConstOrVar, rgenop.s_ConstOrVar], + [self.s_ConstOrVar, self.s_ConstOrVar], self.s_RedBox) self.ll_addr_box_graph = self.annhelper.getgraph( rtimeshift.ll_addr_box, - [rgenop.s_ConstOrVar, rgenop.s_ConstOrVar], + [self.s_ConstOrVar, self.s_ConstOrVar], self.s_RedBox) self.ll_double_box_graph = self.annhelper.getgraph( rtimeshift.ll_int_box, - [rgenop.s_ConstOrVar, rgenop.s_ConstOrVar], + [self.s_ConstOrVar, self.s_ConstOrVar], self.s_RedBox) self.ll_geninputarg_graph = self.annhelper.getgraph( rtimeshift.ll_geninputarg, - [self.s_ResidualGraphBuilder, annmodel.SomePtr(rgenop.CONSTORVAR)], - rgenop.s_ConstOrVar) + [self.s_ResidualGraphBuilder, self.s_ConstOrVar], + self.s_ConstOrVar) self.ll_end_setup_builder_graph = self.annhelper.getgraph( rtimeshift.ll_end_setup_builder, [self.s_ResidualGraphBuilder], - annmodel.SomePtr(rgenop.BLOCK)) + self.s_Block) ## self.ll_close_jitstate_graph = self.annhelper.getgraph( ## rtimeshift.ll_close_jitstate, @@ -266,10 +275,12 @@ newblock.closeblock(bridge) return newblock - def make_const_box(self, llops, r_green, v_value): + def make_const_box(self, llops, r_green, v_value, v_jitstate): v_box = llops.genmixlevelhelpercall( rvalue.ll_fromvalue, - [r_green.annotation()], [v_value], self.s_RedBox) + [self.s_JITState, r_green.annotation()], + [v_jitstate, v_value], + self.s_RedBox) return v_box @@ -566,6 +577,7 @@ [v_res], v_newjitstate)) else: + v_jitstate = rename(orig_v_jitstate) args_r = [] boxes_v = [] for var in inputargs[1:]: @@ -574,7 +586,7 @@ if isinstance(r, RedRepr): boxes_v.append(var) elif isinstance(r, GreenRepr): - v_box = self.make_const_box(llops, r, var) + v_box = self.make_const_box(llops, r, var, v_jitstate) boxes_v.append(v_box) else: raise RuntimeError('Unsupported boxtype') @@ -583,7 +595,6 @@ false_exit = [exit for exit in newblock.exits if exit.exitcase is False][0] exitindex = self.getexitindex(false_exit, inputargs[1:], args_r, entering_links) c_exitindex = rmodel.inputconst(lltype.Signed, exitindex) - v_jitstate = rename(orig_v_jitstate) v_res = llops.genmixlevelhelpercall(rtimeshift.leave_block_split, [self.s_JITState, self.s_RedBox, @@ -665,9 +676,10 @@ return_cache = self.return_cache assert return_cache is not None RETURN_TYPE = self.r_returnvalue.original_concretetype + gv_RETURN_TYPE = self.RGenOp.constTYPE(RETURN_TYPE) def prepare_return(jitstate): return rtimeshift.prepare_return(jitstate, return_cache, - rgenop.constTYPE(RETURN_TYPE)) + gv_RETURN_TYPE) llops = HintLowLevelOpList(self, None) v_return_builder = llops.genmixlevelhelpercall(prepare_return, [self.s_JITState], [v_jitstate2], Modified: pypy/dist/pypy/jit/timeshifter/vlist.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/vlist.py (original) +++ pypy/dist/pypy/jit/timeshifter/vlist.py Wed Aug 23 18:47:27 2006 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype -from pypy.rpython import rgenop from pypy.jit.timeshifter.rcontainer import AbstractContainer, cachedtype from pypy.jit.timeshifter import rvalue @@ -7,11 +6,11 @@ class ListTypeDesc(object): __metaclass__ = cachedtype - def __init__(self, LIST): + def __init__(self, RGenOp, LIST): self.LIST = LIST self.LISTPTR = lltype.Ptr(LIST) - self.gv_type = rgenop.constTYPE(self.LIST) - self.gv_ptrtype = rgenop.constTYPE(self.LISTPTR) + self.gv_type = RGenOp.constTYPE(self.LIST) + self.gv_ptrtype = RGenOp.constTYPE(self.LISTPTR) self.build_newlist = LIST.list_builder.build_newlist self.build_setitem = LIST.list_builder.build_setitem Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Wed Aug 23 18:47:27 2006 @@ -50,7 +50,7 @@ def __init__(pol, rtyper=None): pol.rtyper = rtyper - def default_specialize(pol, funcdesc, args_s): + def default_specialize(funcdesc, args_s): key = [] new_args_s = [] for s_obj in args_s: @@ -69,6 +69,7 @@ flowgraph = funcdesc.cachedgraph(tuple(key)) args_s[:] = new_args_s return flowgraph + default_specialize = staticmethod(default_specialize) def override__init_opaque_object(pol, s_opaqueptr, s_value): assert isinstance(s_opaqueptr, annmodel.SomePtr) @@ -117,7 +118,8 @@ def default_specialize(pol, funcdesc, args_s): name = funcdesc.name if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better? - return LowLevelAnnotatorPolicy.default_specialize(pol, funcdesc, args_s) + return super(MixLevelAnnotatorPolicy, pol).default_specialize( + funcdesc, args_s) else: return funcdesc.cachedgraph(None) Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Aug 23 18:47:27 2006 @@ -969,6 +969,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): start, stop = self._obj.getbounds() if not (start <= i < stop): + if isinstance(i, slice): + raise TypeError("array slicing not supported") raise IndexError("array index out of bounds") o = self._obj.getitem(i) return _expose(o, self._solid) @@ -986,6 +988,8 @@ " got %r" % (self._T, T1, T2)) start, stop = self._obj.getbounds() if not (start <= i < stop): + if isinstance(i, slice): + raise TypeError("array slicing not supported") raise IndexError("array index out of bounds") self._obj.setitem(i, val) return Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Wed Aug 23 18:47:27 2006 @@ -779,11 +779,6 @@ return outof_cobject(v, r_to, llops) # ____________________________________________________________ - -def ll_both_none(ins1, ins2): - return not ins1 and not ins2 - -# ____________________________________________________________ # # Low-level implementation of operations on classes and instances @@ -836,3 +831,26 @@ else: # type(None) -> NULL (for now) return nullptr(typeOf(obj).TO.typeptr.TO) + +def ll_both_none(ins1, ins2): + return not ins1 and not ins2 + +# ____________________________________________________________ + +_missing = object() + +def fishllattr(inst, name, default=_missing): + p = widest = lltype.normalizeptr(inst) + while True: + try: + return getattr(p, 'inst_' + name) + except AttributeError: + pass + try: + p = p.super + except AttributeError: + break + if default is _missing: + raise AttributeError("%s has no field %s" % (lltype.typeOf(widest), + name)) + return default Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Wed Aug 23 18:47:27 2006 @@ -122,7 +122,7 @@ c_LIST = llops.genvoidconst(LIST) return llops.genop('direct_call', [c_newlist, c_LIST, c_len], - LISTPTR) + llops.constTYPE(LISTPTR)) def build_setitem(llops, v_list, index, v_item): c_setitem_nonneg = llops.genconst(setitem_nonneg_ptr) Modified: pypy/dist/pypy/rpython/rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/rtyper.py (original) +++ pypy/dist/pypy/rpython/rtyper.py Wed Aug 23 18:47:27 2006 @@ -897,6 +897,9 @@ def genvoidconst(self, placeholder): return inputconst(Void, placeholder) + def constTYPE(self, T): + return T + # _______________________________________________________________________ # this has the side-effect of registering the unary and binary operations # and the rtyper_chooserepr() methods Modified: pypy/dist/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rpbc.py (original) +++ pypy/dist/pypy/rpython/test/test_rpbc.py Wed Aug 23 18:47:27 2006 @@ -428,6 +428,21 @@ res = self.interpret(f1, [2]) assert res == 6 + def test_call_memo_with_string(self): + def memofn(s): + return eval(s) + memofn._annspecialcase_ = "specialize:memo" + + def f1(i): + if i == 1: + return memofn("6*7") + else: + return memofn("1+2+3+4") + res = self.interpret(f1, [1]) + assert res == 42 + res = self.interpret(f1, [2]) + assert res == 10 + def test_rpbc_bound_method_static_call(self): class R: def meth(self): From mwh at codespeak.net Wed Aug 23 18:49:56 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 18:49:56 +0200 (CEST) Subject: [pypy-svn] r31570 - pypy/branch/no-zeroing-assumption/pypy/translator/c/test Message-ID: <20060823164956.03CBF1006C@code0.codespeak.net> Author: mwh Date: Wed Aug 23 18:49:54 2006 New Revision: 31570 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_lltyped.py Log: "fix" a couple of tests Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_lltyped.py Wed Aug 23 18:49:54 2006 @@ -97,7 +97,7 @@ def test_more_prebuilt_arrays(self): A = FixedSizeArray(Struct('s1', ('x', Signed)), 5) S = GcStruct('s', ('a1', Ptr(A)), ('a2', A)) - s = malloc(S) + s = malloc(S, zero=True) s.a1 = malloc(A, immortal=True) s.a1[2].x = 50 s.a2[2].x = 60 @@ -173,9 +173,9 @@ assert res == 34 def test_prebuilt_subarrays(self): - a1 = malloc(GcArray(Signed), 5) + a1 = malloc(GcArray(Signed), 5, zero=True) a2 = malloc(FixedSizeArray(Signed, 5), immortal=True) - s = malloc(GcStruct('S', ('x', Signed), ('y', Signed))) + s = malloc(GcStruct('S', ('x', Signed), ('y', Signed)), zero=True) a1[3] = 7000 a2[1] = 600 s.x = 50 From mwh at codespeak.net Wed Aug 23 19:06:47 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 23 Aug 2006 19:06:47 +0200 (CEST) Subject: [pypy-svn] r31576 - pypy/branch/no-zeroing-assumption/pypy/translator/stackless Message-ID: <20060823170647.1BFB710036@code0.codespeak.net> Author: mwh Date: Wed Aug 23 19:06:46 2006 New Revision: 31576 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/stackless/transform.py Log: fix resume point tests (grunk). Modified: pypy/branch/no-zeroing-assumption/pypy/translator/stackless/transform.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/stackless/transform.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/stackless/transform.py Wed Aug 23 19:06:46 2006 @@ -716,7 +716,7 @@ realrettype = op.result.concretetype for i, a in enumerate(noexclink.args): if a is op.result: - noexclink.args[i] = model.Constant(realrettype._defl(), realrettype) + noexclink.args[i] = model.Constant(realrettype._defl(example=True), realrettype) block.recloseblock(*((noexclink,) + block.exits[1:])) def insert_unwind_handling(self, block, i): From bea at codespeak.net Thu Aug 24 11:08:22 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:08:22 +0200 (CEST) Subject: [pypy-svn] r31581 - pypy/extradoc/sprintinfo Message-ID: <20060824090822.6DA9810036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:08:20 2006 New Revision: 31581 Modified: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt Log: inserted jacobs comments Modified: pypy/extradoc/sprintinfo/ep2005-sprintreport.txt ============================================================================== --- pypy/extradoc/sprintinfo/ep2005-sprintreport.txt (original) +++ pypy/extradoc/sprintinfo/ep2005-sprintreport.txt Thu Aug 24 11:08:20 2006 @@ -133,4 +133,19 @@ a talk for that conference we hear is happening quite soon. Cheers, -mwh \ No newline at end of file +mwh + +Apart from the presprint, which is already reported, there was an open +post-Europython sprint. At the open sprint there were 24 people on the first +day and activities happened on may fronts. A large group of people focused on +porting PyPy to support Python2.4. This work was 3/4 finished by the end of +the sprint. Christian and Jacob worked on support for floats and Jacob did a +bunch of work on binascii and related modules. I think there was also a bunch +of work going into annotation, translation and the C and llvm backends. Nick +worked on his sre Summer of Code project. There was also work being done on +the AST compiler. + +People I remember being there: All the regulars plus Anders Qvist and Marius +Gedminas. + +Jacob From bea at codespeak.net Thu Aug 24 11:09:44 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:09:44 +0200 (CEST) Subject: [pypy-svn] r31582 - pypy/extradoc/sprintinfo Message-ID: <20060824090944.2A1E310036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:09:42 2006 New Revision: 31582 Added: pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt (contents, props changed) Log: created a file with jacobs comments - please feel free to add more Added: pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt Thu Aug 24 11:09:42 2006 @@ -0,0 +1,20 @@ +Gothenburg (May 2003) +=================== +The major focus of the sprint was to make the interpreter chain complete. A +first goal was to make "Hello world" run and this was accomplished about half +way into the sprint. We then proceded to making a more complex program that +had operations involving string operations and dictionaries work. This was +accomplished before the end of the sprint. + +http://www.strakt.com/~jacob/pypy/img_0062.jpg + +There was a big discussion, mostly involving Armin and Samuele over how to +handle operator dispatching. This ended in the Multimethod dispatcher that +mostly still survives in the Standard Object Space. + +http://www.strakt.com/~jacob/pypy/img_0064.jpg +http://www.strakt.com/~jacob/pypy/img_0065.jpg +http://www.strakt.com/~jacob/pypy/img_0066.jpg + +People I remember being there: Armin, Christian, Samuele, Michael, Alex +Martelli, Anna Ravenscroft, Laura, Tomek Meka \ No newline at end of file From bea at codespeak.net Thu Aug 24 11:10:49 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:10:49 +0200 (CEST) Subject: [pypy-svn] r31583 - pypy/extradoc/sprintinfo Message-ID: <20060824091049.F309010036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:10:47 2006 New Revision: 31583 Added: pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt (contents, props changed) Log: created txt file with Jacob and Marius comments Added: pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt Thu Aug 24 11:10:47 2006 @@ -0,0 +1,15 @@ +Vilnius (Nov 2004) +=============== +I don't really remember what others did at this sprint, but this is where +Laura and I wrote the file I/O handling. + +People I remember being there: Marius Gedminas, Bob Ippolito, Christian, +Armin. + +Jacob + +IIRC everyone was overjoyed when PyPy managed to completely translate +into C and compute the Ultimate Answer to Life, the Universe, and +Everything (def somefunc(): return -6 * -7). + +Marius Gedminas \ No newline at end of file From bea at codespeak.net Thu Aug 24 11:12:03 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:12:03 +0200 (CEST) Subject: [pypy-svn] r31584 - pypy/extradoc/sprintinfo Message-ID: <20060824091203.5B5E410036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:11:59 2006 New Revision: 31584 Added: pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt (contents, props changed) Log: created txt file with jacobs comments Added: pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt Thu Aug 24 11:11:59 2006 @@ -0,0 +1,11 @@ +Leysin (April 2006) +=============== +Armin, Arre and Samuele started with 2 days working on the JIT. When others +joined the sprint focus changed and there was a bunch of work on fixing +translation to work with stackless features and other stuff. There was also +some work done on the constraints package. + +People I remember being there: +Armin, Arre, Samuele, Holger, Michael, Alexandre, Aurelien + +Jacob \ No newline at end of file From bea at codespeak.net Thu Aug 24 11:31:08 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:31:08 +0200 (CEST) Subject: [pypy-svn] r31585 - pypy/dist/pypy/doc Message-ID: <20060824093108.6295510036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:31:06 2006 New Revision: 31585 Modified: pypy/dist/pypy/doc/sprint-reports.txt Log: update with jacob and marius feedback Modified: pypy/dist/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/dist/pypy/doc/sprint-reports.txt (original) +++ pypy/dist/pypy/doc/sprint-reports.txt Thu Aug 24 11:31:06 2006 @@ -6,12 +6,12 @@ A good summary of the progress over the years.....enjoy! * `Hildesheim (Feb 2003)`_ - * Gothenburg (May 2003) + * `Gothenburg (May 2003)`_ * `LovainLaNeuve (June 2003)`_ * `Berlin (Sept 2003)`_ * `Amsterdam (Dec 2003)`_ * Europython/Gothenburg (June 2004) - * Vilnius (Nov 2004) + * `Vilnius (Nov 2004)`_ * `Leysin (Jan 2005)`_ * `PyCon/Washington (March 2005)`_ * `Europython/Gothenburg (June 2005)`_ @@ -22,16 +22,18 @@ * `Mallorca (Jan 2006)`_ * PyCon/Dallas (Feb 2006) * `LouvainLaNeuve (March 2006)`_ - * Leysin (April 2006) + * `Leysin (April 2006)`_ * `Tokyo (April 2006)`_ * `D?sseldorf (June 2006)`_ * `Europython/Geneva (July 2006)`_ * Limerick (Aug 2006) .. _Hildesheim (Feb 2003): https://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html + .. _Gothenburg (May 2003):https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt .. _LovainLaNeuve (June 2003): https://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt .. _Berlin (Sept 2003): https://codespeak.net/pypy/extradoc/sprintinfo/BerlinReport.txt .. _Amsterdam (Dec 2003): https://codespeak.net/pypy/extradoc/sprintinfo/AmsterdamReport.txt + .. _Vilnius (Nov 2004): https://codespeak.net/pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt .. _Leysin (Jan 2005): https://codespeak.net/pypy/extradoc/sprintinfo/LeysinReport.txt .. _PyCon/Washington (March 2005): https://codespeak.net/pypy/extradoc/sprintinfo/pycon_sprint_report.txt .. _Europython/Gothenburg (June 2005): https://codespeak.net/pypy/extradoc/sprintinfo/ep2005-sprintreport.txt @@ -41,6 +43,7 @@ .. _Gothenburg (Dec 2005): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt .. _Mallorca (Jan 2006): https://codespeak.net/pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt .. _LouvainLaNeuve (March 2006): https://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt + .. _Leysin (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt .. _Tokyo (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/tokyo/sprint-report.txt .. _D?sseldorf (June 2006): https://codespeak.net/pypy/extradoc/sprintinfo/ddorf2006/report1.txt .. _Europython/Geneva (July 2006): https://codespeak.net/pypy/extradoc/sprintinfo/post-ep2006/report.txt From bea at codespeak.net Thu Aug 24 11:31:38 2006 From: bea at codespeak.net (bea at codespeak.net) Date: Thu, 24 Aug 2006 11:31:38 +0200 (CEST) Subject: [pypy-svn] r31586 - pypy/dist/pypy/doc Message-ID: <20060824093138.000EA10036@code0.codespeak.net> Author: bea Date: Thu Aug 24 11:31:36 2006 New Revision: 31586 Modified: pypy/dist/pypy/doc/dev_method.txt Log: update with jacob and marius feedback Modified: pypy/dist/pypy/doc/dev_method.txt ============================================================================== --- pypy/dist/pypy/doc/dev_method.txt (original) +++ pypy/dist/pypy/doc/dev_method.txt Thu Aug 24 11:31:36 2006 @@ -94,7 +94,7 @@ As always with methodologies you have to adapt them to fit your project (and not the other way around which is much too common). The PyPy team have been -sprinting since early 2003 and have done 21 sprints so far, 18 in Europe, 2 +sprinting since early 2003 and have done 22 sprints so far, 19 in Europe, 2 in the USA and 1 in Asia. Certain practices have proven to be more successful within this team and those are the one we are summarizing here. @@ -245,35 +245,33 @@ The PyPy team have been sprinting on the following occasions:: - Hildesheim Feb 2003 - Gothenburg May 2003 - LovainLaNeuve June 2003 - Amsterdam Dec 2003 - Europython - /Gothenburg June 2004 - Vilnius Nov 2004 - Leysin Jan 2005 - PyCon - /Washington March 2005 - Europython - /Gothenburg June 2005 - Hildesheim July 2005 - Heidelberg Aug 2005 - Paris Oct 2005 - Gothenburg Dec 2005 - Mallorca Jan 2006 - PyCon/Dallas Feb 2006 - LouvainLaNeuve March 2006 - Leysin April 2006 - Tokyo April 2006 - D?sseldorf June 2006 - Europython/ - Geneva July 2006 - Limerick Aug 2006 + * Hildesheim Feb 2003 + * Gothenburg May 2003 + * LovainLaNeuve June 2003 + * Berlin Sept 2003 + * Amsterdam Dec 2003 + * Europython /Gothenburg June 2004 + * Vilnius Nov 2004 + * Leysin Jan 2005 + * PyCon/Washington March 2005 + * Europython/Gothenburg June 2005 + * Hildesheim July 2005 + * Heidelberg Aug 2005 + * Paris Oct 2005 + * Gothenburg Dec 2005 + * Mallorca Jan 2006 + * PyCon/Dallas Feb 2006 + * LouvainLaNeuve March 2006 + * Leysin April 2006 + * Tokyo April 2006 + * D?sseldorf June 2006 + * Europython/Geneva July 2006 + * Limerick Aug 2006 People who have participated and contributed during our sprints and thus -contributing to PyPy:: +contributing to PyPy (if we have missed someone here - please contact us +so we can correct it):: Armin Rigo Holger Krekel From mwh at codespeak.net Thu Aug 24 12:40:23 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 12:40:23 +0200 (CEST) Subject: [pypy-svn] r31590 - in pypy/branch/no-zeroing-assumption/pypy/translator/c: . src Message-ID: <20060824104023.916B110069@code0.codespeak.net> Author: mwh Date: Thu Aug 24 12:40:21 2006 New Revision: 31590 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Log: make it more possible to allocate non-zeroed memory. don't actually allocate any non-zeroed memory, though. Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py Thu Aug 24 12:40:21 2006 @@ -500,7 +500,7 @@ self.expr(op.args[0]), self.expr(op.args[1])) - def OP_MALLOC(self, op): + def OP_ZERO_MALLOC(self, op): TYPE = self.lltypemap(op.result).TO typename = self.db.gettype(TYPE) eresult = self.expr(op.result) @@ -508,6 +508,14 @@ return self.gcpolicy.zero_malloc(TYPE, esize, eresult) + def OP_MALLOC(self, op): + TYPE = self.lltypemap(op.result).TO + typename = self.db.gettype(TYPE) + eresult = self.expr(op.result) + esize = 'sizeof(%s)' % cdecl(typename, '') + + return self.gcpolicy.malloc(TYPE, esize, eresult) + OP_ZERO_MALLOC = OP_MALLOC def OP_MALLOC_VARSIZE(self, op): @@ -553,7 +561,8 @@ def OP_RAW_MALLOC(self, op): eresult = self.expr(op.result) esize = self.expr(op.args[0]) - return "OP_RAW_MALLOC(%s, %s, void *);" % (esize, eresult) + return ("OP_RAW_MALLOC(%s, %s, void *);\n\t" % (esize, eresult) + + "if (%s) OP_RAW_MEM_ZERO(%s, %s);" % (eresult, eresult, esize)) def OP_FLAVORED_MALLOC(self, op): TYPE = self.lltypemap(op.result).TO Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py Thu Aug 24 12:40:21 2006 @@ -103,6 +103,8 @@ erestype = cdecl(typename, '*') return 'OP_ZERO_MALLOC(%s, %s, %s);' % (esize, eresult, erestype) + malloc = zero_malloc + def OP_GC_CALL_RTTI_DESTRUCTOR(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] line = '%s(%s);' % (args[0], ', '.join(args[1:])) @@ -194,6 +196,8 @@ % (eresult, gcinfo.finalizer)) return result + malloc = zero_malloc + def gc_libraries(self): if sys.platform == 'win32': return ['gc_pypy'] @@ -339,6 +343,8 @@ TYPE, esize, eresult) return result + malloc = zero_malloc + # to get an idea how it looks like with no refcount/gc at all class NoneGcPolicy(BoehmGcPolicy): @@ -401,6 +407,8 @@ erestype = cdecl(typename, '*') return 'OP_ZERO_MALLOC(%s, %s, %s);' % (esize, eresult, erestype) + malloc = zero_malloc + class StacklessFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = gctransform.StacklessFrameworkGCTransformer requires_stackless = True Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Thu Aug 24 12:40:21 2006 @@ -2,7 +2,20 @@ /************************************************************/ /*** C header subsection: operations on LowLevelTypes ***/ -#define OP_RAW_MALLOC(size,r,restype) OP_ZERO_MALLOC(size, r, restype) +#define OP_RAW_MALLOC(size, r, restype) { \ + r = (restype) PyObject_Malloc(size); \ + if (r == NULL) { \ + FAIL_EXCEPTION(PyExc_MemoryError, \ + "out of memory"); \ + } \ + else { \ + COUNT_MALLOC; \ + } \ + } + +#define OP_RAW_FREE(p, r) PyObject_Free(p); COUNT_FREE; + +#define OP_RAW_MEM_ZERO(p, size) memset((void*)p, 0, size) #define OP_RAW_MALLOC_USAGE(size, r) r = size @@ -17,7 +30,6 @@ if (r == NULL) FAIL_EXCEPTION(PyExc_MemoryError, "out of memory"); \ memset((void*) r, 0, size); -#define OP_RAW_FREE(x,r) OP_FREE(x) #define OP_RAW_MEMCOPY(x,y,size,r) memcpy(y,x,size); /************************************************************/ @@ -41,16 +53,12 @@ other globals, plus one. This upper bound "approximation" will do... */ #define REFCOUNT_IMMORTAL (INT_MAX/2) -#define OP_ZERO_MALLOC(size, r, restype) { \ - r = (restype) PyObject_Malloc(size); \ - if (r == NULL) {FAIL_EXCEPTION(PyExc_MemoryError, "out of memory"); } \ - else { \ - memset((void*) r, 0, size); \ - COUNT_MALLOC; \ - } \ - } +#define OP_ZERO_MALLOC(size, r, restype) { \ + OP_RAW_MALLOC(size, r, restype); \ + if (r != NULL) OP_RAW_MEM_ZERO(r, size); \ + } -#define OP_FREE(p) { PyObject_Free(p); COUNT_FREE; } +#define OP_FREE(p) OP_RAW_FREE(p, do_not_use) /*------------------------------------------------------------*/ #ifndef COUNT_OP_MALLOCS From arigo at codespeak.net Thu Aug 24 12:42:02 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 Aug 2006 12:42:02 +0200 (CEST) Subject: [pypy-svn] r31591 - in pypy/dist/pypy: objspace/flow objspace/flow/test rpython Message-ID: <20060824104202.0CA6810069@code0.codespeak.net> Author: arigo Date: Thu Aug 24 12:41:57 2006 New Revision: 31591 Added: pypy/dist/pypy/objspace/flow/test/test_unroll.py (contents, props changed) pypy/dist/pypy/rpython/unroll.py (contents, props changed) Modified: pypy/dist/pypy/objspace/flow/flowcontext.py pypy/dist/pypy/objspace/flow/framestate.py pypy/dist/pypy/objspace/flow/objspace.py pypy/dist/pypy/objspace/flow/test/test_objspace.py Log: (arigo, pedronis) 'for' iteration over iterables wrapped in an instance of unrolling_iterable will be unrolled by the flow space, like in: names = unrolling_iterable(['a', 'b', 'c']) def f(x): for name in names: setattr(x, name, 0) nice new feature but a bit hackish around the corners, use with care Modified: pypy/dist/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/dist/pypy/objspace/flow/flowcontext.py (original) +++ pypy/dist/pypy/objspace/flow/flowcontext.py Thu Aug 24 12:41:57 2006 @@ -374,3 +374,13 @@ # re-raising an implicit operation makes it an explicit one operr = OperationError(operr.w_type, operr.w_value) return operr + + # hack for unrolling iterables, don't use this + def replace_in_stack(self, oldvalue, newvalue): + w_new = Constant(newvalue) + stack_items_w = self.crnt_frame.valuestack.items + for i in range(len(stack_items_w)): + w_v = stack_items_w[i] + if isinstance(w_v, Constant): + if w_v.value is oldvalue: + stack_items_w[i] = w_new Modified: pypy/dist/pypy/objspace/flow/framestate.py ============================================================================== --- pypy/dist/pypy/objspace/flow/framestate.py (original) +++ pypy/dist/pypy/objspace/flow/framestate.py Thu Aug 24 12:41:57 2006 @@ -1,8 +1,8 @@ from pypy.interpreter.pyframe import PyFrame, SuspendedUnroller from pypy.interpreter.error import OperationError from pypy.rpython.objectmodel import instantiate +from pypy.rpython.unroll import SpecTag from pypy.objspace.flow.model import * -from pypy.tool.uid import uid class FrameState: # XXX this class depends on the internal state of PyFrame objects @@ -135,18 +135,6 @@ # ____________________________________________________________ # -# Support for explicit specialization: in code using global constants -# that are instances of SpecTag, code paths are not merged when -# the same variable holds a different SpecTag instance. - -class SpecTag(object): - def __repr__(self): - return 'SpecTag(0x%x)' % uid(self) - def _freeze_(self): - return True - -# ____________________________________________________________ -# # We have to flatten out the state of the frame into a list of # Variables and Constants. This is done above by collecting the # locals and the items on the value stack, but the latter may contain Modified: pypy/dist/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/objspace.py (original) +++ pypy/dist/pypy/objspace/flow/objspace.py Thu Aug 24 12:41:57 2006 @@ -7,6 +7,7 @@ from pypy.objspace.flow.model import * from pypy.objspace.flow import flowcontext from pypy.objspace.flow.operation import FunctionByName +from pypy.rpython.unroll import unrolling_iterable, _unroller debug = 0 @@ -343,9 +344,33 @@ context = self.getexecutioncontext() return context.guessbool(w_truthvalue) + def iter(self, w_iterable): + try: + iterable = self.unwrap(w_iterable) + except UnwrapException: + pass + else: + if isinstance(iterable, unrolling_iterable): + return self.wrap(iterable.get_unroller()) + w_iter = self.do_operation("iter", w_iterable) + return w_iter + def next(self, w_iter): - w_item = self.do_operation("next", w_iter) context = self.getexecutioncontext() + try: + it = self.unwrap(w_iter) + except UnwrapException: + pass + else: + if isinstance(it, _unroller): + try: + v, next_unroller = it.step() + except IndexError: + raise OperationError(self.w_StopIteration, self.w_None) + else: + context.replace_in_stack(it, next_unroller) + return self.wrap(v) + w_item = self.do_operation("next", w_iter) outcome, w_exc_cls, w_exc_value = context.guessexception(StopIteration) if outcome is StopIteration: raise OperationError(self.w_StopIteration, w_exc_value) Modified: pypy/dist/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/dist/pypy/objspace/flow/test/test_objspace.py Thu Aug 24 12:41:57 2006 @@ -29,8 +29,6 @@ if conftest.option.view: graph.show() - -class TestFlowObjSpace(Base): def setup_class(cls): cls.space = FlowObjSpace() @@ -44,7 +42,9 @@ traverse(visit, graph) return result - #__________________________________________________________ + +class TestFlowObjSpace(Base): + def nothing(): pass Added: pypy/dist/pypy/objspace/flow/test/test_unroll.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/flow/test/test_unroll.py Thu Aug 24 12:41:57 2006 @@ -0,0 +1,40 @@ +from pypy.objspace.flow.test.test_objspace import Base +from pypy.rpython.unroll import unrolling_zero, unrolling_iterable + +class TestUnroll(Base): + + def test_unrolling_int(self): + l = range(10) + def f(tot): + i = unrolling_zero + while i < len(l): + tot += l[i] + i = i + 1 + return tot*2 + assert f(0) == sum(l)*2 + + graph = self.codetest(f) + ops = self.all_operations(graph) + assert ops == {'inplace_add': 10, 'mul': 1} + + def test_unroller(self): + l = unrolling_iterable(range(10)) + def f(tot): + for v in l: + tot += v + return tot*3 + assert f(0) == sum(l)*3 + + graph = self.codetest(f) + ops = self.all_operations(graph) + assert ops == {'inplace_add': 10, 'mul': 1} + + def test_unroll_setattrs(self): + values_names = unrolling_iterable(enumerate(['a', 'b', 'c'])) + def f(x): + for v, name in values_names: + setattr(x, name, v) + + graph = self.codetest(f) + ops = self.all_operations(graph) + assert ops == {'setattr': 3} Added: pypy/dist/pypy/rpython/unroll.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/unroll.py Thu Aug 24 12:41:57 2006 @@ -0,0 +1,63 @@ +from pypy.tool.uid import uid + +# Support for explicit specialization: in code using global constants +# that are instances of SpecTag, code paths are not merged when +# the same variable holds a different SpecTag instance. + +class SpecTag(object): + __slots__ = () + + def __repr__(self): + return '%s(0x%x)' % (self.__class__.__name__, uid(self)) + def _freeze_(self): + return True + + +class unrolling_int(int, SpecTag): + + def __add__(self, other): + return unrolling_int(int.__add__(self, other)) + + __radd__ = __add__ + + def __sub__(self, other): + return unrolling_int(int.__sub__(self, other)) + + def __rsub__(self, other): + return unrolling_int(int.__rsub__(self, other)) + + +unrolling_zero = unrolling_int(0) + +# ____________________________________________________________ + +# 'for' iteration over iterables wrapped in an instance +# of unrolling_iterable will be unrolled by the flow space, +# like in: +# names = unrolling_iterable(['a', 'b', 'c']) +# def f(x): +# for name in names: +# setattr(x, name, 0) + +class unrolling_iterable(SpecTag): + + def __init__(self, iterable): + self._items = list(iterable) + + def __iter__(self): + return iter(self._items) + + def get_unroller(self): + return _unroller(self._items) + + +class _unroller(SpecTag): + + def __init__(self, items, i=0): + self._items = items + self._i = i + + def step(self): + v = self._items[self._i] + next = _unroller(self._items, self._i+1) + return v, next From arigo at codespeak.net Thu Aug 24 13:15:28 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 Aug 2006 13:15:28 +0200 (CEST) Subject: [pypy-svn] r31594 - pypy/dist/pypy/rpython Message-ID: <20060824111528.2E97910068@code0.codespeak.net> Author: arigo Date: Thu Aug 24 13:15:26 2006 New Revision: 31594 Modified: pypy/dist/pypy/rpython/rtuple.py Log: Experimentally simplify rtuple's ll_hash and ll_eq function construction by using an unrolling_iterable instead of an exec. Looks quite good, I'd say. Modified: pypy/dist/pypy/rpython/rtuple.py ============================================================================== --- pypy/dist/pypy/rpython/rtuple.py (original) +++ pypy/dist/pypy/rpython/rtuple.py Thu Aug 24 13:15:26 2006 @@ -9,6 +9,7 @@ from pypy.rpython.rslice import AbstractSliceRepr from pypy.rpython.lltypesystem.lltype import Void, Signed from pypy.rpython.rarithmetic import intmask +from pypy.rpython.unroll import unrolling_iterable class __extend__(annmodel.SomeTuple): def rtyper_makerepr(self, rtyper): @@ -29,20 +30,19 @@ try: return _gen_eq_function_cache[key] except KeyError: - miniglobals = {} - source = """ -def ll_eq(t1, t2): - %s - return True -""" - body = [] - for i, eq_func in enumerate(eq_funcs): - miniglobals['eq%d' % i] = eq_func - body.append("if not eq%d(t1.item%d, t2.item%d): return False" % (i, i, i)) - body = ('\n'+' '*4).join(body) - source = source % body - exec source in miniglobals - ll_eq = miniglobals['ll_eq'] + autounrolling_funclist = unrolling_iterable(enumerate(eq_funcs)) + + def ll_eq(t1, t2): + equal_so_far = True + for i, eqfn in autounrolling_funclist: + if not equal_so_far: + return False + attrname = 'item%d' % i + item1 = getattr(t1, attrname) + item2 = getattr(t2, attrname) + equal_so_far = eqfn(item1, item2) + return equal_so_far + _gen_eq_function_cache[key] = ll_eq return ll_eq @@ -53,24 +53,18 @@ try: return _gen_hash_function_cache[key] except KeyError: - miniglobals = {} - source = """ -def ll_hash(t): - retval = 0x345678 - %s - return retval -""" - body = [] - mult = 1000003 - for i, hash_func in enumerate(hash_funcs): - miniglobals['hash%d' % i] = hash_func - body.append("retval = (retval ^ hash%d(t.item%d)) * %d" % - (i, i, mult)) - mult = intmask(mult + 82520 + 2*len(items_r)) - body = ('\n'+' '*4).join(body) - source = source % body - exec source in miniglobals - ll_hash = miniglobals['ll_hash'] + autounrolling_funclist = unrolling_iterable(enumerate(hash_funcs)) + + def ll_hash(t): + retval = 0x345678 + mult = 1000003 + for i, hash_func in autounrolling_funclist: + attrname = 'item%d' % i + item = getattr(t, attrname) + retval = intmask((retval ^ hash_func(item)) * intmask(mult)) + mult = mult + 82520 + 2*len(items_r) + return retval + _gen_hash_function_cache[key] = ll_hash return ll_hash From mwh at codespeak.net Thu Aug 24 13:19:23 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 13:19:23 +0200 (CEST) Subject: [pypy-svn] r31595 - pypy/branch/no-zeroing-assumption/pypy/translator/c/src Message-ID: <20060824111923.5792410068@code0.codespeak.net> Author: mwh Date: Thu Aug 24 13:19:22 2006 New Revision: 31595 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Log: zero cpy allocations too Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Thu Aug 24 13:19:22 2006 @@ -140,6 +140,7 @@ /* XXX add tp_itemsize later */ \ OP_RAW_MALLOC(((PyTypeObject *)cpytype)->tp_basicsize, r, restype); \ if (r) { \ + OP_RAW_MEM_ZERO(r, ((PyTypeObject *)cpytype)->tp_basicsize); \ PyObject_Init((PyObject *)r, (PyTypeObject *)cpytype); \ } \ } From mwh at codespeak.net Thu Aug 24 14:02:27 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 14:02:27 +0200 (CEST) Subject: [pypy-svn] r31598 - in pypy/branch/no-zeroing-assumption/pypy: annotation rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory Message-ID: <20060824120227.406A210068@code0.codespeak.net> Author: mwh Date: Thu Aug 24 14:02:24 2006 New Revision: 31598 Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llheap.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/branch/no-zeroing-assumption/pypy/rpython/memory/lladdress.py pypy/branch/no-zeroing-assumption/pypy/rpython/memory/simulator.py pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py Log: (ac, mwh) make raw_malloc-on-lltype return nearly uninitialized memory more often. add and test a raw_memclear function and llop. Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py Thu Aug 24 14:02:24 2006 @@ -562,6 +562,11 @@ assert isinstance(s_addr, SomeAddress) assert not s_addr.is_null +def raw_memclear(s_addr, s_int): + assert isinstance(s_addr, SomeAddress) + assert not s_addr.is_null + assert isinstance(s_int, SomeInteger) + def raw_memcopy(s_addr1, s_addr2, s_int): assert isinstance(s_addr1, SomeAddress) assert not s_addr1.is_null @@ -572,6 +577,7 @@ BUILTIN_ANALYZERS[lladdress.raw_malloc] = raw_malloc BUILTIN_ANALYZERS[lladdress.raw_malloc_usage] = raw_malloc_usage BUILTIN_ANALYZERS[lladdress.raw_free] = raw_free +BUILTIN_ANALYZERS[lladdress.raw_memclear] = raw_memclear BUILTIN_ANALYZERS[lladdress.raw_memcopy] = raw_memcopy #_________________________________ Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/llinterp.py Thu Aug 24 14:02:24 2006 @@ -661,6 +661,10 @@ checkadr(addr) self.heap.raw_free(addr) + def op_raw_memclear(self, addr, size): + checkadr(addr) + self.heap.raw_memclear(addr, size) + def op_raw_memcopy(self, fromaddr, toaddr, size): checkadr(fromaddr) checkadr(toaddr) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llheap.py Thu Aug 24 14:02:24 2006 @@ -2,5 +2,5 @@ from pypy.rpython.lltypesystem.lltype import pyobjectptr, malloc, free from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free -from pypy.rpython.lltypesystem.llmemory import raw_memcopy +from pypy.rpython.lltypesystem.llmemory import raw_memclear, raw_memcopy from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py Thu Aug 24 14:02:24 2006 @@ -60,7 +60,7 @@ return cast_ptr_to_adr(p) else: T = lltype.FixedSizeArray(self.TYPE, self.repeat) - p = lltype.malloc(T, immortal=True) + p = lltype.malloc(T, flavor='raw') array_adr = cast_ptr_to_adr(p) return array_adr + ArrayItemsOffset(T) @@ -555,6 +555,14 @@ size = convert_offset_to_int(size) return size +def raw_memclear(adr, size): + # hack hack hack + obj = adr.get() + TYPE = lltype.typeOf(obj) + fresh = lltype.malloc(TYPE.TO, zero=True, flavor='raw') + from pypy.rpython.rctypes.rmodel import reccopy + reccopy(fresh, obj) + def raw_memcopy(source, dest, size): source = source.get() dest = dest.get() Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lloperation.py Thu Aug 24 14:02:24 2006 @@ -324,6 +324,7 @@ 'raw_malloc': LLOp(canraise=(MemoryError,)), 'raw_malloc_usage': LLOp(sideeffects=False), 'raw_free': LLOp(), + 'raw_memclear': LLOp(), 'raw_memcopy': LLOp(), 'raw_load': LLOp(sideeffects=False), 'raw_store': LLOp(), Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py Thu Aug 24 14:02:24 2006 @@ -252,6 +252,9 @@ S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Ptr(T))) adr = raw_malloc(sizeof(S)) s = cast_adr_to_ptr(adr, lltype.Ptr(S)) + py.test.raises(lltype.UninitializedMemoryAccess, "s.x") + raw_memclear(adr, sizeof(S)) + assert s.x == 0 assert lltype.typeOf(s) == lltype.Ptr(S) s.x = 123 x_adr = adr + offsetof(S, 'x') @@ -259,6 +262,28 @@ x_adr.signed[0] = 124 assert s.x == 124 +def test_llinterp_raw_malloc_struct(): + T = lltype.GcStruct('T', ('z', lltype.Signed)) + S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Ptr(T))) + + from pypy.rpython.memory import lladdress # GRUMBLE! + + size = sizeof(S) + + def test_read_uninit(): + adr = lladdress.raw_malloc(size) + s = cast_adr_to_ptr(adr, lltype.Ptr(S)) + return s.x + py.test.raises(lltype.UninitializedMemoryAccess, "interpret(test_read_uninit, [])") + def test_read_init(): + adr = lladdress.raw_malloc(size) + lladdress.raw_memclear(adr, size) + s = cast_adr_to_ptr(adr, lltype.Ptr(S)) + return s.x + res = interpret(test_read_init, []) + assert res == 0 + + def test_raw_malloc_signed(): adr = raw_malloc(sizeof(lltype.Signed)) p = cast_adr_to_ptr(adr, Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/lladdress.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/memory/lladdress.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/memory/lladdress.py Thu Aug 24 14:02:24 2006 @@ -128,6 +128,9 @@ def raw_free(addr): simulator.free(addr.intaddress) +def raw_memclear(addr, size): + simulator.memclear(addr.intaddress, size) + def raw_memcopy(addr1, addr2, size): simulator.memcopy(addr1.intaddress, addr2.intaddress, size) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/simulator.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/memory/simulator.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/memory/simulator.py Thu Aug 24 14:02:24 2006 @@ -58,6 +58,9 @@ other.memory[offset2:offset2+size] = self.memory[offset1:offset1+size] other.status[offset2:offset2+size] = self.status[offset1:offset1+size] + def memclear(self, offset, size): + self.setbytes(offset, "\x00" * size) + # block which stores functions and PyObects class ObjectBlock(object): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/rbuiltin.py Thu Aug 24 14:02:24 2006 @@ -558,9 +558,14 @@ v_list = hop.inputargs(llmemory.Address, llmemory.Address, lltype.Signed) return hop.genop('raw_memcopy', v_list) +def rtype_raw_memclear(hop): + v_list = hop.inputargs(llmemory.Address, lltype.Signed) + return hop.genop('raw_memclear', v_list) + BUILTIN_TYPER[lladdress.raw_malloc] = rtype_raw_malloc BUILTIN_TYPER[lladdress.raw_malloc_usage] = rtype_raw_malloc_usage BUILTIN_TYPER[lladdress.raw_free] = rtype_raw_free +BUILTIN_TYPER[lladdress.raw_memclear] = rtype_raw_memclear BUILTIN_TYPER[lladdress.raw_memcopy] = rtype_raw_memcopy def rtype_offsetof(hop): From ac at codespeak.net Thu Aug 24 14:59:01 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Thu, 24 Aug 2006 14:59:01 +0200 (CEST) Subject: [pypy-svn] r31603 - in pypy/dist/pypy: doc/discussion rpython/memory Message-ID: <20060824125901.06D201005A@code0.codespeak.net> Author: ac Date: Thu Aug 24 14:59:00 2006 New Revision: 31603 Added: pypy/dist/pypy/doc/discussion/GC-performance.txt (contents, props changed) Modified: pypy/dist/pypy/rpython/memory/gc.py Log: (mwh, arre) A new heuristic for adjusting the bytes_malloced_threshold. The rationale behind it is that since collection is costly do less of it in some clever way. Further tweeking of the magic constants are sure to be needed. See doc/discussion/GC-performance.txt for numbers. Added: pypy/dist/pypy/doc/discussion/GC-performance.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/GC-performance.txt Thu Aug 24 14:59:00 2006 @@ -0,0 +1,116 @@ +StartHeapsize# is the framework GC as of revision 31586 with initial +bytes_malloced_threshold of 2-512 MB + +NewHeuristics is the framework GC with a new heuristics for adjusting +the bytes_malloced_threshold + +Pystone +StartHeapsize2: +This machine benchmarks at 5426.92 pystones/second +This machine benchmarks at 5193.91 pystones/second +This machine benchmarks at 5403.46 pystones/second +StartHeapsize8: +This machine benchmarks at 6075.33 pystones/second +This machine benchmarks at 6007.21 pystones/second +This machine benchmarks at 6122.45 pystones/second +StartHeapsize32: +This machine benchmarks at 6643.05 pystones/second +This machine benchmarks at 6590.51 pystones/second +This machine benchmarks at 6593.41 pystones/second +StartHeapsize128: +This machine benchmarks at 7065.47 pystones/second +This machine benchmarks at 7102.27 pystones/second +This machine benchmarks at 7082.15 pystones/second +StartHeapsize512: +This machine benchmarks at 7208.07 pystones/second +This machine benchmarks at 7197.7 pystones/second +This machine benchmarks at 7246.38 pystones/second +NewHeuristics: +This machine benchmarks at 6821.28 pystones/second +This machine benchmarks at 6858.71 pystones/second +This machine benchmarks at 6902.9 pystones/second + + +Richards +StartHeapSize2: +Average time per iteration: 5456.21 ms +Average time per iteration: 5529.31 ms +Average time per iteration: 5398.82 ms +StartHeapsize8: +Average time per iteration: 4775.43 ms +Average time per iteration: 4753.25 ms +Average time per iteration: 4781.37 ms +StartHeapsize32: +Average time per iteration: 4554.84 ms +Average time per iteration: 4501.86 ms +Average time per iteration: 4531.59 ms +StartHeapsize128: +Average time per iteration: 4329.42 ms +Average time per iteration: 4360.87 ms +Average time per iteration: 4392.81 ms +StartHeapsize512: +Average time per iteration: 4371.72 ms +Average time per iteration: 4399.70 ms +Average time per iteration: 4354.66 ms +NewHeuristics: +Average time per iteration: 4763.56 ms +Average time per iteration: 4803.49 ms +Average time per iteration: 4840.68 ms + + +translate rpystone + time pypy-c translate --text --batch --backendopt --no-compile targetrpystonedalone.py +StartHeapSize2: +real 1m38.459s +user 1m35.582s +sys 0m0.440s +StartHeapsize8: +real 1m35.398s +user 1m33.878s +sys 0m0.376s +StartHeapsize32: +real 1m5.475s +user 1m5.108s +sys 0m0.180s +StartHeapsize128: +real 0m52.941s +user 0m52.395s +sys 0m0.328s +StartHeapsize512: +real 1m3.727s +user 0m50.031s +sys 0m1.240s +NewHeuristics: +real 0m53.449s +user 0m52.771s +sys 0m0.356s + + +docutils + time pypy-c rst2html doc/coding-guide.txt +StartHeapSize2: +real 0m36.125s +user 0m35.562s +sys 0m0.088s +StartHeapsize8: +real 0m32.678s +user 0m31.106s +sys 0m0.084s +StartHeapsize32: +real 0m22.041s +user 0m21.085s +sys 0m0.132s +StartHeapsize128: +real 0m19.350s +user 0m18.653s +sys 0m0.324s +StartHeapsize512: +real 0m19.116s +user 0m17.517s +sys 0m0.620s +NewHeuristics: +real 0m20.990s +user 0m20.109s +sys 0m0.196s + + Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Thu Aug 24 14:59:00 2006 @@ -154,6 +154,7 @@ # found via 'poolnodes'. self.poolnodes = lltype.nullptr(self.POOLNODE) self.collect_in_progress = False + self.prev_collect_end_time = 0.0 def malloc(self, typeid, length=0): size = self.fixed_size(typeid) @@ -345,10 +346,33 @@ lltype.free(firstpoolnode, flavor='raw') #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) + end_time = time.time() + compute_time = start_time - self.prev_collect_end_time + collect_time = end_time - start_time + + + + + collect_time_fraction = collect_time / compute_time + old_heapsize = self.heap_usage + garbage_generated = old_malloced - (curr_heap_size - old_heapsize) + garbage_fraction = float(garbage_generated) / float(curr_heap_size) + + + if collect_time_fraction > 0.02 * garbage_fraction: + self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 + if collect_time_fraction < 0.005 * garbage_fraction: + self.bytes_malloced_threshold /= 2 + + # Use atleast as much memory as current live objects. if curr_heap_size > self.bytes_malloced_threshold: self.bytes_malloced_threshold = curr_heap_size - end_time = time.time() - self.total_collection_time += end_time - start_time + + # Cap at 1/4 GB + self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, + 256 * 1024 * 1024) + self.total_collection_time += collect_time + self.prev_collect_end_time = end_time if DEBUG_PRINT: llop.debug_print(lltype.Void, " malloced since previous collection:", @@ -365,6 +389,15 @@ llop.debug_print(lltype.Void, " total time spent collecting: ", self.total_collection_time, "seconds") + llop.debug_print(lltype.Void, + " collecting time fraction: ", + collect_time_fraction) + llop.debug_print(lltype.Void, + " garbage fraction: ", + garbage_fraction) + llop.debug_print(lltype.Void, + " new threshold: ", + self.bytes_malloced_threshold) ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size From mwh at codespeak.net Thu Aug 24 15:03:02 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 15:03:02 +0200 (CEST) Subject: [pypy-svn] r31604 - pypy/dist/pypy/doc Message-ID: <20060824130302.0331610060@code0.codespeak.net> Author: mwh Date: Thu Aug 24 15:03:01 2006 New Revision: 31604 Modified: pypy/dist/pypy/doc/sprint-reports.txt Log: tiny rest fix Modified: pypy/dist/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/dist/pypy/doc/sprint-reports.txt (original) +++ pypy/dist/pypy/doc/sprint-reports.txt Thu Aug 24 15:03:01 2006 @@ -29,7 +29,7 @@ * Limerick (Aug 2006) .. _Hildesheim (Feb 2003): https://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html - .. _Gothenburg (May 2003):https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt + .. _Gothenburg (May 2003): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt .. _LovainLaNeuve (June 2003): https://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt .. _Berlin (Sept 2003): https://codespeak.net/pypy/extradoc/sprintinfo/BerlinReport.txt .. _Amsterdam (Dec 2003): https://codespeak.net/pypy/extradoc/sprintinfo/AmsterdamReport.txt From mwh at codespeak.net Thu Aug 24 15:03:51 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 15:03:51 +0200 (CEST) Subject: [pypy-svn] r31605 - pypy/dist/pypy/doc Message-ID: <20060824130351.B322F10068@code0.codespeak.net> Author: mwh Date: Thu Aug 24 15:03:50 2006 New Revision: 31605 Modified: pypy/dist/pypy/doc/sprint-reports.txt Log: change https to http links Modified: pypy/dist/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/dist/pypy/doc/sprint-reports.txt (original) +++ pypy/dist/pypy/doc/sprint-reports.txt Thu Aug 24 15:03:50 2006 @@ -28,24 +28,24 @@ * `Europython/Geneva (July 2006)`_ * Limerick (Aug 2006) - .. _Hildesheim (Feb 2003): https://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html - .. _Gothenburg (May 2003): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt - .. _LovainLaNeuve (June 2003): https://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt - .. _Berlin (Sept 2003): https://codespeak.net/pypy/extradoc/sprintinfo/BerlinReport.txt - .. _Amsterdam (Dec 2003): https://codespeak.net/pypy/extradoc/sprintinfo/AmsterdamReport.txt - .. _Vilnius (Nov 2004): https://codespeak.net/pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt - .. _Leysin (Jan 2005): https://codespeak.net/pypy/extradoc/sprintinfo/LeysinReport.txt - .. _PyCon/Washington (March 2005): https://codespeak.net/pypy/extradoc/sprintinfo/pycon_sprint_report.txt - .. _Europython/Gothenburg (June 2005): https://codespeak.net/pypy/extradoc/sprintinfo/ep2005-sprintreport.txt - .. _Hildesheim (July 2005): https://codespeak.net/pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt - .. _Heidelberg (Aug 2005): https://codespeak.net/pypy/extradoc/sprintinfo/Heidelberg-report.txt - .. _Paris (Oct 2005): https://codespeak.net/pypy/extradoc/sprintinfo/paris/paris-report.txt - .. _Gothenburg (Dec 2005): https://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt - .. _Mallorca (Jan 2006): https://codespeak.net/pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt - .. _LouvainLaNeuve (March 2006): https://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt - .. _Leysin (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt - .. _Tokyo (April 2006): https://codespeak.net/pypy/extradoc/sprintinfo/tokyo/sprint-report.txt - .. _D?sseldorf (June 2006): https://codespeak.net/pypy/extradoc/sprintinfo/ddorf2006/report1.txt - .. _Europython/Geneva (July 2006): https://codespeak.net/pypy/extradoc/sprintinfo/post-ep2006/report.txt + .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html + .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt + .. _LovainLaNeuve (June 2003): http://codespeak.net/pypy/extradoc/sprintinfo/LouvainLaNeuveReport.txt + .. _Berlin (Sept 2003): http://codespeak.net/pypy/extradoc/sprintinfo/BerlinReport.txt + .. _Amsterdam (Dec 2003): http://codespeak.net/pypy/extradoc/sprintinfo/AmsterdamReport.txt + .. _Vilnius (Nov 2004): http://codespeak.net/pypy/extradoc/sprintinfo/vilnius-2004-sprintreport.txt + .. _Leysin (Jan 2005): http://codespeak.net/pypy/extradoc/sprintinfo/LeysinReport.txt + .. _PyCon/Washington (March 2005): http://codespeak.net/pypy/extradoc/sprintinfo/pycon_sprint_report.txt + .. _Europython/Gothenburg (June 2005): http://codespeak.net/pypy/extradoc/sprintinfo/ep2005-sprintreport.txt + .. _Hildesheim (July 2005): http://codespeak.net/pypy/extradoc/sprintinfo/hildesheim2005-sprintreport.txt + .. _Heidelberg (Aug 2005): http://codespeak.net/pypy/extradoc/sprintinfo/Heidelberg-report.txt + .. _Paris (Oct 2005): http://codespeak.net/pypy/extradoc/sprintinfo/paris/paris-report.txt + .. _Gothenburg (Dec 2005): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2005/gothenburg-dec2005-sprintreport.txt + .. _Mallorca (Jan 2006): http://codespeak.net/pypy/extradoc/sprintinfo/mallorca/mallorca-sprintreport.txt + .. _LouvainLaNeuve (March 2006): http://codespeak.net/pypy/extradoc/sprintinfo/louvain-la-neuve-2006/report.txt + .. _Leysin (April 2006): http://codespeak.net/pypy/extradoc/sprintinfo/leysin-winter-2006-sprintreport.txt + .. _Tokyo (April 2006): http://codespeak.net/pypy/extradoc/sprintinfo/tokyo/sprint-report.txt + .. _D?sseldorf (June 2006): http://codespeak.net/pypy/extradoc/sprintinfo/ddorf2006/report1.txt + .. _Europython/Geneva (July 2006): http://codespeak.net/pypy/extradoc/sprintinfo/post-ep2006/report.txt From mwh at codespeak.net Thu Aug 24 15:22:50 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 24 Aug 2006 15:22:50 +0200 (CEST) Subject: [pypy-svn] r31606 - pypy/branch/no-zeroing-assumption/pypy/translator/c/test Message-ID: <20060824132250.0D07F10071@code0.codespeak.net> Author: mwh Date: Thu Aug 24 15:22:49 2006 New Revision: 31606 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py Log: (mwh, ac) add a test that fails when we fill raw_malloced memory with garbage (none of the others do!) Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py Thu Aug 24 15:22:49 2006 @@ -849,6 +849,19 @@ res = fn() assert res == 10 + def test_framework_late_filling_pointers(self): + A = lltype.GcStruct('A', ('x', lltype.Signed)) + B = lltype.GcStruct('B', ('a', lltype.Ptr(A))) + + def f(): + p = lltype.malloc(B) + llop.gc__collect(lltype.Void) + p.a = lltype.malloc(A) + return p.a.x + fn = self.getcompiled(f) + res = fn() + assert res == 123 + class TestUsingStacklessFramework(TestUsingFramework): from pypy.translator.c.gc import StacklessFrameworkGcPolicy as gcpolicy From arigo at codespeak.net Thu Aug 24 15:36:08 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 Aug 2006 15:36:08 +0200 (CEST) Subject: [pypy-svn] r31607 - in pypy/dist/pypy/rpython: . test Message-ID: <20060824133608.4BD6210060@code0.codespeak.net> Author: arigo Date: Thu Aug 24 15:36:06 2006 New Revision: 31607 Modified: pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/test/test_llann.py Log: (pedronis, arigo) pypy.rpython.annlowlevel.PseudoHighLevelCallable: a gateway to a low-level function pointer. To high-level RPython code it looks like a normal function, taking high-level arguments and returning a high-level result. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Thu Aug 24 15:36:06 2006 @@ -7,7 +7,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.lltypesystem import lltype -from pypy.rpython import extfunctable +from pypy.rpython import extfunctable, extregistry from pypy.objspace.flow.model import Constant def not_const(s_obj): # xxx move it somewhere else @@ -186,6 +186,13 @@ self.delayedreprs.append(r) return r + def s_r_instanceof(self, cls, can_be_None=True): + classdesc = self.rtyper.annotator.bookkeeper.getdesc(cls) + classdef = classdesc.getuniqueclassdef() + s_instance = annmodel.SomeInstance(classdef, can_be_None) + r_instance = self.getdelayedrepr(s_instance) + return s_instance, r_instance + def delayedconst(self, repr, obj): if repr.is_setup_delayed(): # record the existence of this 'obj' for the bookkeeper - e.g. @@ -250,3 +257,34 @@ newgraphs = translator.graphs[self.original_graph_count:] self.original_graph_count = len(translator.graphs) backend_optimizations(translator, newgraphs, **flags) + +# ____________________________________________________________ + +class PseudoHighLevelCallable(object): + """A gateway to a low-level function pointer. To high-level RPython + code it looks like a normal function, taking high-level arguments + and returning a high-level result. + """ + def __init__(self, llfnptr, args_s, s_result): + self.llfnptr = llfnptr + self.args_s = args_s + self.s_result = s_result + + def __call__(self, *args): + raise Exception("PseudoHighLevelCallable objects are not really " + "callable directly") + +class Entry(extregistry.ExtRegistryEntry): + _type_ = PseudoHighLevelCallable + + def compute_result_annotation(self, *args_s): + return self.instance.s_result + + def specialize_call(self, hop): + args_r = [hop.rtyper.getrepr(s) for s in self.instance.args_s] + r_res = hop.rtyper.getrepr(self.instance.s_result) + vlist = hop.inputargs(*args_r) + p = self.instance.llfnptr + c_func = Constant(p, lltype.typeOf(p)) + hop.exception_is_here() + return hop.genop('direct_call', [c_func] + vlist, resulttype = r_res) Modified: pypy/dist/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llann.py (original) +++ pypy/dist/pypy/rpython/test/test_llann.py Thu Aug 24 15:36:06 2006 @@ -1,6 +1,10 @@ from pypy.rpython.lltypesystem.lltype import * +from pypy.translator.translator import TranslationContext from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import annotate_lowlevel_helper +from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.rpython.annlowlevel import PseudoHighLevelCallable +from pypy.rpython.llinterp import LLInterpreter from pypy.objspace.flow import FlowObjSpace # helpers @@ -308,3 +312,36 @@ assert s.unsigned == True +def test_pseudohighlevelcallable(): + t = TranslationContext() + t.buildannotator() + rtyper = t.buildrtyper() + rtyper.specialize() + a = MixLevelHelperAnnotator(rtyper) + + class A: + value = 5 + def double(self): + return self.value * 2 + + def fn1(a): + a2 = A() + a2.value = a.double() + return a2 + + s_A, r_A = a.s_r_instanceof(A) + fn1ptr = a.delayedfunction(fn1, [s_A], s_A) + pseudo = PseudoHighLevelCallable(fn1ptr, [s_A], s_A) + + def fn2(n): + a = A() + a.value = n + a2 = pseudo(a) + return a2.value + + graph = a.getgraph(fn2, [annmodel.SomeInteger()], annmodel.SomeInteger()) + a.finish() + + llinterp = LLInterpreter(rtyper) + res = llinterp.eval_graph(graph, [21]) + assert res == 42 From arigo at codespeak.net Thu Aug 24 18:19:58 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 Aug 2006 18:19:58 +0200 (CEST) Subject: [pypy-svn] r31614 - in pypy/dist/pypy: annotation jit/codegen/llgraph jit/timeshifter jit/timeshifter/test objspace/flow rpython Message-ID: <20060824161958.8CF5610074@code0.codespeak.net> Author: arigo Date: Thu Aug 24 18:19:50 2006 New Revision: 31614 Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/annotation/unaryop.py pypy/dist/pypy/jit/codegen/llgraph/rgenop.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/rvalue.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/timeshift.py pypy/dist/pypy/objspace/flow/model.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/rtyper.py Log: (pedronis, arigo) Refactor the timeshifter test runner to be parametrizable on the RGenOp. Caused a lot of pain and obscure crashes. Added sanity checks and XXXs a bit everywhere. Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Thu Aug 24 18:19:50 2006 @@ -169,15 +169,24 @@ def addpendingblock(self, graph, block, cells, called_from_graph=None): """Register an entry point into block with the given input cells.""" - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells, called_from_graph) + if graph in self.fixed_graphs: + # special case for annotating/rtyping in several phases: calling + # a graph that has already been rtyped. Safety-check the new + # annotations that are passed in, and don't annotate the old + # graph -- it's already low-level operations! + for a, s_newarg in zip(graph.getargs(), cells): + s_oldarg = self.binding(a) + assert s_oldarg.contains(s_newarg) else: - self.mergeinputargs(graph, block, cells, called_from_graph) - if not self.annotated[block]: - self.pendingblocks[block] = graph + assert not self.frozen + for a in cells: + assert isinstance(a, annmodel.SomeObject) + if block not in self.annotated: + self.bindinputargs(graph, block, cells, called_from_graph) + else: + self.mergeinputargs(graph, block, cells, called_from_graph) + if not self.annotated[block]: + self.pendingblocks[block] = graph def complete(self): """Process pending blocks until none is left.""" @@ -348,17 +357,8 @@ callpositions[callback] = True # generalize the function's input arguments - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), inputcells): - s_oldarg = self.binding(a) - assert s_oldarg.contains(s_newarg) - else: - self.addpendingblock(graph, graph.startblock, inputcells, - position_key) + self.addpendingblock(graph, graph.startblock, inputcells, + position_key) # get the (current) return value v = graph.getreturnvar() @@ -456,6 +456,7 @@ def reflowpendingblock(self, graph, block): assert not self.frozen + assert graph not in self.fixed_graphs self.pendingblocks[block] = graph assert block in self.annotated self.annotated[block] = False # must re-flow Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Thu Aug 24 18:19:50 2006 @@ -77,6 +77,7 @@ def finish(self): from pypy.annotation.model import unionof + assert self.graph is None, "MemoTable already finished" # list of which argument positions can take more than one value example_args, example_value = self.table.iteritems().next() nbargs = len(example_args) @@ -236,15 +237,12 @@ def compute_one_result(args): value = func(*args) - return MemoTable(funcdesc, args, value) - - def finish(): - for memotable in memotables.infos(): - memotable.finish() + memotable = MemoTable(funcdesc, args, value) + bookkeeper.pending_specializations.append(memotable.finish) + return memotable memotables = UnionFind(compute_one_result) bookkeeper.all_specializations[funcdesc] = memotables - bookkeeper.pending_specializations.append(finish) # merge the MemoTables for the individual argument combinations firstvalues = possiblevalues.next() Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Thu Aug 24 18:19:50 2006 @@ -607,8 +607,11 @@ v_lltype = annotation_to_lltype(s_value) setattr(example, s_attr.const, v_lltype._defl()) - def simple_call(p, *args_s): - llargs = [annotation_to_lltype(arg_s)._defl() for arg_s in args_s] + def call(p, args): + args_s, kwds_s = args.unpack() + if kwds_s: + raise Exception("keyword arguments to call to a low-level fn ptr") + llargs = [annotation_to_lltype(s_arg)._defl() for s_arg in args_s] v = p.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Thu Aug 24 18:19:50 2006 @@ -87,7 +87,11 @@ # not RPython, just for debugging. Specific to llgraph. def reveal(gv): - return llimpl.reveal(gv.v) + if hasattr(gv, 'v'): + v = gv.v + else: + v = fishllattr(gv, 'v') + return llimpl.reveal(v) reveal = staticmethod(reveal) # Builds a real flow.model.FunctionGraph. Specific to llgraph. @@ -99,8 +103,6 @@ return llimpl.buildgraph(b) buildgraph = staticmethod(buildgraph) - testgengraph = staticmethod(llimpl.testgengraph) - def get_rgenop_for_testing(): return rgenop get_rgenop_for_testing = staticmethod(get_rgenop_for_testing) Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Thu Aug 24 18:19:50 2006 @@ -366,7 +366,10 @@ def ll_close_builder(builder): builder.finish_and_return() - + +def ll_gencallableconst(builder, name, startblock, gv_functype): + return builder.rgenop.gencallableconst(name, startblock, gv_functype) + class JITState(object): # XXX obscure interface localredboxes = None Modified: pypy/dist/pypy/jit/timeshifter/rvalue.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rvalue.py (original) +++ pypy/dist/pypy/jit/timeshifter/rvalue.py Thu Aug 24 18:19:50 2006 @@ -67,9 +67,9 @@ return IntRedBox def redboxbuilder_void(gv_type, gv_value):return None -def redboxbuilder_int(gv_ptr, gv_value): return IntRedBox(gv_ptr, gv_value) -def redboxbuilder_dbl(gv_ptr, gv_value): return DoubleRedBox(gv_ptr, gv_value) -def redboxbuilder_ptr(gv_ptr, gv_value): return PtrRedBox(gv_ptr, gv_value) +def redboxbuilder_int(gv_type, gv_value): return IntRedBox(gv_type, gv_value) +def redboxbuilder_dbl(gv_type, gv_value): return DoubleRedBox(gv_type,gv_value) +def redboxbuilder_ptr(gv_type, gv_value): return PtrRedBox(gv_type, gv_value) def ll_redboxbuilder(TYPE): if TYPE is lltype.Void: Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Thu Aug 24 18:19:50 2006 @@ -4,12 +4,14 @@ from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.model import * from pypy.jit.timeshifter.timeshift import HintTimeshift -from pypy.jit.timeshifter import rtimeshift, rtyper as hintrtyper +from pypy.jit.timeshifter import rtimeshift, rvalue, rtyper as hintrtyper from pypy.jit.llabstractinterp.test.test_llabstractinterp import annotation from pypy.jit.llabstractinterp.test.test_llabstractinterp import summary from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.objectmodel import hint, keepalive_until_here +from pypy.rpython.unroll import unrolling_iterable from pypy.rpython.lltypesystem import rstr +from pypy.rpython.annlowlevel import PseudoHighLevelCallable from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter from pypy.objspace.flow.model import checkgraph @@ -65,6 +67,8 @@ hs, ha, rtyper = hannotate(ll_function, values, inline=inline, policy=policy) htshift = HintTimeshift(ha, rtyper, self.RGenOp) + RESTYPE = htshift.originalconcretetype( + ha.translator.graphs[0].getreturnvar()) htshift.timeshift() t = rtyper.annotator.translator for graph in ha.translator.graphs: @@ -73,62 +77,101 @@ if conftest.option.view: from pypy.translator.tool.graphpage import FlowGraphPage FlowGraphPage(t, ha.translator.graphs).display() - result = hs, ha, rtyper, htshift + result = hs, ha, rtyper, htshift, RESTYPE self._cache[key] = result, getargtypes(rtyper.annotator, values) self._cache_order.append(key) else: - hs, ha, rtyper, htshift = result + hs, ha, rtyper, htshift, RESTYPE = result assert argtypes == getargtypes(rtyper.annotator, values) return result def timeshift(self, ll_function, values, opt_consts=[], inline=None, policy=None): - hs, ha, rtyper, htshift = self.timeshift_cached(ll_function, values, - inline, policy) - # run the time-shifted graph-producing graphs + hs, ha, rtyper, htshift, RESTYPE = self.timeshift_cached( + ll_function, values, inline, policy) + # build a runner function + original_entrypoint_graph = rtyper.annotator.translator.graphs[0] graph1 = ha.translator.graphs[0] - llinterp = LLInterpreter(rtyper) - builder = llinterp.eval_graph(htshift.ll_make_builder_graph, []) - graph1args = [builder, lltype.nullptr(htshift.r_JITState.lowleveltype.TO)] - residual_graph_args = [] + timeshifted_entrypoint_fnptr = rtyper.type_system.getcallable(graph1) assert len(graph1.getargs()) == 2 + len(values) - for i, (v, llvalue) in enumerate(zip(graph1.getargs()[2:], values)): + graph1varargs = graph1.getargs()[2:] + + argdescription = [] # list of: ("green"/"redconst"/"redvar", value) + residual_args = [] + residual_argtypes = [] + timeshifted_entrypoint_args_s = [] + + for i, (v, llvalue) in enumerate(zip(graph1varargs, values)): r = htshift.hrtyper.bindingrepr(v) residual_v = r.residual_values(llvalue) if len(residual_v) == 0: # green - graph1args.append(llvalue) + color = "green" + s_var = annmodel.ll_to_annotation(llvalue) + timeshifted_entrypoint_args_s.append(s_var) else: # red assert residual_v == [llvalue], "XXX for now" - TYPE = htshift.originalconcretetype(v) - gv_type = self.RGenOp.constTYPE(TYPE) - ll_type = htshift.r_ConstOrVar.convert_const(gv_type) - ll_gvar = llinterp.eval_graph(htshift.ll_geninputarg_graph, - [builder, ll_type]) - if i in opt_consts: # XXX what should happen here interface wise is unclear - gvar = self.RGenOp.constPrebuiltGlobal(llvalue) - ll_gvar = htshift.r_ConstOrVar.convert_const(gvar) - if isinstance(lltype.typeOf(llvalue), lltype.Ptr): - ll_box_graph = htshift.ll_addr_box_graph - elif isinstance(llvalue, float): - ll_box_graph = htshift.ll_double_box_graph + if i in opt_consts: + color = "redconst" else: - ll_box_graph = htshift.ll_int_box_graph - box = llinterp.eval_graph(ll_box_graph, [ll_type, ll_gvar]) - graph1args.append(box) - residual_graph_args.append(llvalue) - startblock = llinterp.eval_graph(htshift.ll_end_setup_builder_graph, [builder]) - - builder = llinterp.eval_graph(graph1, graph1args) - r = htshift.hrtyper.getrepr(hs) - llinterp.eval_graph(htshift.ll_close_builder_graph, [builder]) + color = "redvar" + ARGTYPE = htshift.originalconcretetype(v) + residual_argtypes.append(ARGTYPE) + residual_args.append(llvalue) + timeshifted_entrypoint_args_s.append(htshift.s_RedBox) + argdescription.append((color, llvalue)) + + argdescription = unrolling_iterable(argdescription) + RGenOp = self.RGenOp + timeshifted_entrypoint = PseudoHighLevelCallable( + timeshifted_entrypoint_fnptr, + [htshift.s_ResidualGraphBuilder, htshift.s_JITState] + + timeshifted_entrypoint_args_s, + htshift.s_ResidualGraphBuilder) + FUNCTYPE = lltype.FuncType(residual_argtypes, RESTYPE) + gv_functype = RGenOp.constTYPE(FUNCTYPE) + + def ll_runner(): + rgenop = RGenOp.get_rgenop_for_testing() + builder = rtimeshift.make_builder(rgenop) + timeshifted_entrypoint_args = () + for color, llvalue in argdescription: + if color == "green": + timeshifted_entrypoint_args += (llvalue,) + else: + TYPE = lltype.typeOf(llvalue) + gv_type = rgenop.constTYPE(TYPE) + boxcls = rvalue.ll_redboxcls(TYPE) + if color == "redconst": + gv_arg = rgenop.genconst(llvalue) + else: + gv_arg = rtimeshift.ll_geninputarg(builder, gv_type) + box = boxcls(gv_type, gv_arg) + timeshifted_entrypoint_args += (box,) + startblock = rtimeshift.ll_end_setup_builder(builder) + endbuilder = timeshifted_entrypoint(builder, None, + *timeshifted_entrypoint_args) + endbuilder.finish_and_return() + gv_generated = rgenop.gencallableconst("generated", startblock, + gv_functype) + return gv_generated + + annhelper = htshift.annhelper + runner_graph = annhelper.getgraph(ll_runner, [], htshift.s_ConstOrVar) + annhelper.finish() + + # run the runner + llinterp = LLInterpreter(rtyper) + ll_gv_generated = llinterp.eval_graph(runner_graph, []) - # now try to run the blocks produced by the builder - residual_graph = self.RGenOp.buildgraph(startblock) + # now try to run the residual graph generated by the builder + c_generatedfn = RGenOp.reveal(ll_gv_generated) + residual_graph = c_generatedfn.value._obj.graph + if conftest.option.view: + residual_graph.show() insns = summary(residual_graph) - res = self.RGenOp.testgengraph(residual_graph, residual_graph_args, - viewbefore = conftest.option.view) + res = llinterp.eval_graph(residual_graph, residual_args) return insns, res Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Thu Aug 24 18:19:50 2006 @@ -55,6 +55,7 @@ rgenop = RGenOp.get_rgenop_for_testing() return rtimeshift.make_builder(rgenop) + # XXX find a different way to enforce the interface types self.ll_make_builder_graph = self.annhelper.getgraph( ll_make_builder, [], self.s_ResidualGraphBuilder) @@ -88,14 +89,15 @@ rtimeshift.ll_close_builder, [self.s_ResidualGraphBuilder], annmodel.s_None) + self.annhelper.getgraph( + rtimeshift.ll_gencallableconst, + [self.s_ResidualGraphBuilder, annmodel.SomeString(), + self.s_Block, self.s_ConstOrVar], + self.s_ConstOrVar) def s_r_instanceof(self, cls, can_be_None=True): # Return a SomeInstance / InstanceRepr pair correspnding to the specified class. - classdesc = self.rtyper.annotator.bookkeeper.getdesc(cls) - classdef = classdesc.getuniqueclassdef() - s_instance = annmodel.SomeInstance(classdef, can_be_None) - r_instance = self.annhelper.getdelayedrepr(s_instance) - return s_instance, r_instance + return self.annhelper.s_r_instanceof(cls, can_be_None=can_be_None) # creates and numbers reentry_block for block reached by link # argument: Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Thu Aug 24 18:19:50 2006 @@ -268,6 +268,13 @@ class Variable(object): __slots__ = ["_name", "_nr", "concretetype"] +## def getter(x): return x._ct +## def setter(x, ct): +## if repr(ct) == '<* PyObject>': +## import pdb; pdb.set_trace() +## x._ct = ct +## concretetype = property(getter, setter) + dummyname = 'v' namesdict = {dummyname : (dummyname, 0)} Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Thu Aug 24 18:19:50 2006 @@ -285,6 +285,10 @@ r_res = hop.rtyper.getrepr(self.instance.s_result) vlist = hop.inputargs(*args_r) p = self.instance.llfnptr - c_func = Constant(p, lltype.typeOf(p)) + TYPE = lltype.typeOf(p) + c_func = Constant(p, TYPE) + for r_arg, ARGTYPE in zip(args_r, TYPE.TO.ARGS): + assert r_arg.lowleveltype == ARGTYPE + assert r_res.lowleveltype == TYPE.TO.RESULT hop.exception_is_here() return hop.genop('direct_call', [c_func] + vlist, resulttype = r_res) Modified: pypy/dist/pypy/rpython/rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/rtyper.py (original) +++ pypy/dist/pypy/rpython/rtyper.py Thu Aug 24 18:19:50 2006 @@ -67,7 +67,6 @@ self.cache_dummy_values = {} self.typererrors = [] self.typererror_count = 0 - self.seen_graphs_count = 0 # make the primitive_to_repr constant mapping self.primitive_to_repr = {} if self.type_system.offers_exceptiondata: @@ -233,12 +232,6 @@ if self.typererrors: self.dump_typererrors(to_log=True) raise TyperError("there were %d error" % len(self.typererrors)) - # make sure that the return variables of all graphs are concretetype'd - newgraphs = self.annotator.translator.graphs[self.seen_graphs_count:] - self.seen_graphs_count += len(newgraphs) - for graph in newgraphs: - v = graph.getreturnvar() - self.setconcretetype(v) log.event('-=- specialized %d%s blocks -=-' % (blockcount, newtext)) def dump_typererrors(self, num=None, minimize=True, to_log=False): @@ -311,7 +304,11 @@ def specialize_block(self, block): graph = self.annotator.annotated[block] - self.annotator.fixed_graphs[graph] = True + if graph not in self.annotator.fixed_graphs: + self.annotator.fixed_graphs[graph] = True + # make sure that the return variables of all graphs + # are concretetype'd + self.setconcretetype(graph.getreturnvar()) # give the best possible types to the input args try: From arigo at codespeak.net Thu Aug 24 18:20:32 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 Aug 2006 18:20:32 +0200 (CEST) Subject: [pypy-svn] r31615 - pypy/dist/pypy/jit/codegen/llvm/test Message-ID: <20060824162032.65C4E10078@code0.codespeak.net> Author: arigo Date: Thu Aug 24 18:20:30 2006 New Revision: 31615 Modified: pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py Log: Move the skip before the failing import. Modified: pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py Thu Aug 24 18:20:30 2006 @@ -1,3 +1,6 @@ +import py +py.test.skip("WIP") + from pypy.translator.llvm.buildllvm import llvm_is_on_path if not llvm_is_on_path(): import py @@ -9,9 +12,6 @@ from pypy.rpython.module.support import from_opaque_object from pypy.objspace.flow import model as flowmodel -import py -py.test.skip("WIP") - def _runblock(blockcontainer, args, viewbefore=False): from pypy.jit.codegen.llvm.jitcode import JITcode runblock(blockcontainer, args, viewbefore, JITcode) From mwh at codespeak.net Fri Aug 25 14:01:30 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 25 Aug 2006 14:01:30 +0200 (CEST) Subject: [pypy-svn] r31632 - pypy/dist/pypy/doc/discussion Message-ID: <20060825120130.632F71007B@code0.codespeak.net> Author: mwh Date: Fri Aug 25 14:01:28 2006 New Revision: 31632 Modified: pypy/dist/pypy/doc/discussion/GC-performance.txt Log: extremely lazy rest fix Modified: pypy/dist/pypy/doc/discussion/GC-performance.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/GC-performance.txt (original) +++ pypy/dist/pypy/doc/discussion/GC-performance.txt Fri Aug 25 14:01:28 2006 @@ -4,113 +4,115 @@ NewHeuristics is the framework GC with a new heuristics for adjusting the bytes_malloced_threshold -Pystone -StartHeapsize2: -This machine benchmarks at 5426.92 pystones/second -This machine benchmarks at 5193.91 pystones/second -This machine benchmarks at 5403.46 pystones/second -StartHeapsize8: -This machine benchmarks at 6075.33 pystones/second -This machine benchmarks at 6007.21 pystones/second -This machine benchmarks at 6122.45 pystones/second -StartHeapsize32: -This machine benchmarks at 6643.05 pystones/second -This machine benchmarks at 6590.51 pystones/second -This machine benchmarks at 6593.41 pystones/second -StartHeapsize128: -This machine benchmarks at 7065.47 pystones/second -This machine benchmarks at 7102.27 pystones/second -This machine benchmarks at 7082.15 pystones/second -StartHeapsize512: -This machine benchmarks at 7208.07 pystones/second -This machine benchmarks at 7197.7 pystones/second -This machine benchmarks at 7246.38 pystones/second -NewHeuristics: -This machine benchmarks at 6821.28 pystones/second -This machine benchmarks at 6858.71 pystones/second -This machine benchmarks at 6902.9 pystones/second - - -Richards -StartHeapSize2: -Average time per iteration: 5456.21 ms -Average time per iteration: 5529.31 ms -Average time per iteration: 5398.82 ms -StartHeapsize8: -Average time per iteration: 4775.43 ms -Average time per iteration: 4753.25 ms -Average time per iteration: 4781.37 ms -StartHeapsize32: -Average time per iteration: 4554.84 ms -Average time per iteration: 4501.86 ms -Average time per iteration: 4531.59 ms -StartHeapsize128: -Average time per iteration: 4329.42 ms -Average time per iteration: 4360.87 ms -Average time per iteration: 4392.81 ms -StartHeapsize512: -Average time per iteration: 4371.72 ms -Average time per iteration: 4399.70 ms -Average time per iteration: 4354.66 ms -NewHeuristics: -Average time per iteration: 4763.56 ms -Average time per iteration: 4803.49 ms -Average time per iteration: 4840.68 ms - - -translate rpystone - time pypy-c translate --text --batch --backendopt --no-compile targetrpystonedalone.py -StartHeapSize2: -real 1m38.459s -user 1m35.582s -sys 0m0.440s -StartHeapsize8: -real 1m35.398s -user 1m33.878s -sys 0m0.376s -StartHeapsize32: -real 1m5.475s -user 1m5.108s -sys 0m0.180s -StartHeapsize128: -real 0m52.941s -user 0m52.395s -sys 0m0.328s -StartHeapsize512: -real 1m3.727s -user 0m50.031s -sys 0m1.240s -NewHeuristics: -real 0m53.449s -user 0m52.771s -sys 0m0.356s - - -docutils - time pypy-c rst2html doc/coding-guide.txt -StartHeapSize2: -real 0m36.125s -user 0m35.562s -sys 0m0.088s -StartHeapsize8: -real 0m32.678s -user 0m31.106s -sys 0m0.084s -StartHeapsize32: -real 0m22.041s -user 0m21.085s -sys 0m0.132s -StartHeapsize128: -real 0m19.350s -user 0m18.653s -sys 0m0.324s -StartHeapsize512: -real 0m19.116s -user 0m17.517s -sys 0m0.620s -NewHeuristics: -real 0m20.990s -user 0m20.109s -sys 0m0.196s +:: + + Pystone + StartHeapsize2: + This machine benchmarks at 5426.92 pystones/second + This machine benchmarks at 5193.91 pystones/second + This machine benchmarks at 5403.46 pystones/second + StartHeapsize8: + This machine benchmarks at 6075.33 pystones/second + This machine benchmarks at 6007.21 pystones/second + This machine benchmarks at 6122.45 pystones/second + StartHeapsize32: + This machine benchmarks at 6643.05 pystones/second + This machine benchmarks at 6590.51 pystones/second + This machine benchmarks at 6593.41 pystones/second + StartHeapsize128: + This machine benchmarks at 7065.47 pystones/second + This machine benchmarks at 7102.27 pystones/second + This machine benchmarks at 7082.15 pystones/second + StartHeapsize512: + This machine benchmarks at 7208.07 pystones/second + This machine benchmarks at 7197.7 pystones/second + This machine benchmarks at 7246.38 pystones/second + NewHeuristics: + This machine benchmarks at 6821.28 pystones/second + This machine benchmarks at 6858.71 pystones/second + This machine benchmarks at 6902.9 pystones/second + + + Richards + StartHeapSize2: + Average time per iteration: 5456.21 ms + Average time per iteration: 5529.31 ms + Average time per iteration: 5398.82 ms + StartHeapsize8: + Average time per iteration: 4775.43 ms + Average time per iteration: 4753.25 ms + Average time per iteration: 4781.37 ms + StartHeapsize32: + Average time per iteration: 4554.84 ms + Average time per iteration: 4501.86 ms + Average time per iteration: 4531.59 ms + StartHeapsize128: + Average time per iteration: 4329.42 ms + Average time per iteration: 4360.87 ms + Average time per iteration: 4392.81 ms + StartHeapsize512: + Average time per iteration: 4371.72 ms + Average time per iteration: 4399.70 ms + Average time per iteration: 4354.66 ms + NewHeuristics: + Average time per iteration: 4763.56 ms + Average time per iteration: 4803.49 ms + Average time per iteration: 4840.68 ms + + + translate rpystone + time pypy-c translate --text --batch --backendopt --no-compile targetrpystonedalone.py + StartHeapSize2: + real 1m38.459s + user 1m35.582s + sys 0m0.440s + StartHeapsize8: + real 1m35.398s + user 1m33.878s + sys 0m0.376s + StartHeapsize32: + real 1m5.475s + user 1m5.108s + sys 0m0.180s + StartHeapsize128: + real 0m52.941s + user 0m52.395s + sys 0m0.328s + StartHeapsize512: + real 1m3.727s + user 0m50.031s + sys 0m1.240s + NewHeuristics: + real 0m53.449s + user 0m52.771s + sys 0m0.356s + + + docutils + time pypy-c rst2html doc/coding-guide.txt + StartHeapSize2: + real 0m36.125s + user 0m35.562s + sys 0m0.088s + StartHeapsize8: + real 0m32.678s + user 0m31.106s + sys 0m0.084s + StartHeapsize32: + real 0m22.041s + user 0m21.085s + sys 0m0.132s + StartHeapsize128: + real 0m19.350s + user 0m18.653s + sys 0m0.324s + StartHeapsize512: + real 0m19.116s + user 0m17.517s + sys 0m0.620s + NewHeuristics: + real 0m20.990s + user 0m20.109s + sys 0m0.196s From arigo at codespeak.net Fri Aug 25 14:09:47 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Aug 2006 14:09:47 +0200 (CEST) Subject: [pypy-svn] r31634 - pypy/dist/pypy/translator/c Message-ID: <20060825120947.BBA9B1007B@code0.codespeak.net> Author: arigo Date: Fri Aug 25 14:09:44 2006 New Revision: 31634 Modified: pypy/dist/pypy/translator/c/wrapper.py Log: (pedronis, arigo) Oups. Set the concretetype of the new graph's returnvar explicitly. Modified: pypy/dist/pypy/translator/c/wrapper.py ============================================================================== --- pypy/dist/pypy/translator/c/wrapper.py (original) +++ pypy/dist/pypy/translator/c/wrapper.py Fri Aug 25 14:09:44 2006 @@ -144,6 +144,7 @@ translator.graphs.append(wgraph) block.operations[:] = newops block.closeblock(Link([vresult], wgraph.returnblock)) + wgraph.getreturnvar().concretetype = PyObjPtr checkgraph(wgraph) if translator.rtyper is not None: @@ -194,6 +195,7 @@ vres = newgraph.getreturnvar() ann.setbinding(vres, s_None) + vres.concretetype = Void checkgraph(newgraph) # pretend to be the same function, as we actually # will become inlined. From arigo at codespeak.net Fri Aug 25 15:28:29 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Aug 2006 15:28:29 +0200 (CEST) Subject: [pypy-svn] r31641 - in pypy/dist/pypy: annotation jit/codegen/i386 jit/timeshifter/test rpython rpython/lltypesystem translator/c/src Message-ID: <20060825132829.EEDB21007B@code0.codespeak.net> Author: arigo Date: Fri Aug 25 15:28:23 2006 New Revision: 31641 Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/translator/c/src/address.h Log: (pedronis, arigo) Link the timeshifter and the ri398genop tests together. The first test passes! Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Fri Aug 25 15:28:23 2006 @@ -315,6 +315,9 @@ def llmemory_cast_adr_to_int(s): return SomeInteger() # xxx +def llmemory_cast_int_to_adr(s): + return SomeAddress() + def rstack_yield_current_frame_to_caller(): return SomeExternalObject(pypy.rpython.rstack.frame_stack_top) Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Fri Aug 25 15:28:23 2006 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.codegen.i386.codebuf import MachineCodeBlock from pypy.jit.codegen.i386.ri386 import * from pypy.jit.codegen.model import AbstractRGenOp, CodeGenBlock, CodeGenLink @@ -32,6 +32,31 @@ self.kind = kind +##class Const(GenConst): + +## def revealconst(self, TYPE): +## if isinstance(self, IntConst): +## self.revealconst_int(TYPE) +## elif isinstance(self, PtrConst): +## self.revealconst_ptr(TYPE) + +## if isinstance(TYPE, lltype.Ptr): +## if isinstance(self, PtrConst): +## return self.revealconst_ptr(TYPE) +## el +## return self.revealconst_ptr(TYPE) +## elif TYPE is lltype.Float: +## assert isinstance(self, DoubleConst) +## return self.revealconst_double() +## else: +## assert isinstance(TYPE, lltype.Primitive) +## assert TYPE is not lltype.Void, "cannot make red boxes of voids" +## assert isinstance(self, IntConst) +## return self.revealconst_primitive(TYPE) +## return self.value +## revealconst._annspecialcase_ = 'specialize:arg(1)' + + class IntConst(GenConst): def __init__(self, value): @@ -41,7 +66,12 @@ return imm(self.value) def revealconst(self, T): - return lltype.cast_int_to_ptr(T, self.value) + if isinstance(T, lltype.Ptr): + return lltype.cast_int_to_ptr(T, self.value) + elif T is llmemory.Address: + return llmemory.cast_int_to_adr(self.value) + else: + return lltype.cast_primitive(T, self.value) revealconst._annspecialcase_ = 'specialize:arg(1)' Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Fri Aug 25 15:28:23 2006 @@ -132,8 +132,7 @@ FUNCTYPE = lltype.FuncType(residual_argtypes, RESTYPE) gv_functype = RGenOp.constTYPE(FUNCTYPE) - def ll_runner(): - rgenop = RGenOp.get_rgenop_for_testing() + def ll_runner(rgenop): builder = rtimeshift.make_builder(rgenop) timeshifted_entrypoint_args = () for color, llvalue in argdescription: @@ -157,12 +156,24 @@ gv_functype) return gv_generated - annhelper = htshift.annhelper - runner_graph = annhelper.getgraph(ll_runner, [], htshift.s_ConstOrVar) + self.rtyper = rtyper + self.htshift = htshift + self.FUNCTYPE = FUNCTYPE + return self.timeshift_test(ll_runner, residual_args) + + def timeshift_test(self, ll_runner, residual_args): + RGenOp = self.RGenOp + def ll_main(): + rgenop = RGenOp.get_rgenop_for_testing() + return ll_runner(rgenop) + + annhelper = self.htshift.annhelper + runner_graph = annhelper.getgraph(ll_main, [], + self.htshift.s_ConstOrVar) annhelper.finish() # run the runner - llinterp = LLInterpreter(rtyper) + llinterp = LLInterpreter(self.rtyper) ll_gv_generated = llinterp.eval_graph(runner_graph, []) # now try to run the residual graph generated by the builder Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Fri Aug 25 15:28:23 2006 @@ -485,6 +485,10 @@ def cast_adr_to_int(adr): return adr._cast_to_int() +def cast_int_to_adr(int): + raise NotImplementedError("cast_int_to_adr") + + # ____________________________________________________________ import weakref Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Fri Aug 25 15:28:23 2006 @@ -340,7 +340,7 @@ 'cast_weakadr_to_ptr': LLOp(canfold=True), 'cast_weakadr_to_int': LLOp(canfold=True), 'cast_adr_to_int': LLOp(canfold=True), - #'cast_int_to_adr': LLOp(canfold=True), + 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp # __________ GC operations __________ Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Fri Aug 25 15:28:23 2006 @@ -279,6 +279,9 @@ checkadr(adr) return llmemory.cast_adr_to_int(adr) +def op_cast_int_to_adr(int): + return llmemory.cast_int_to_adr(int) + ##def op_cast_int_to_adr(x): ## assert type(x) is int ## return llmemory.cast_int_to_adr(x) Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Fri Aug 25 15:28:23 2006 @@ -636,6 +636,13 @@ return hop.genop('cast_adr_to_int', [adr], resulttype = lltype.Signed) +def rtype_cast_int_to_adr(hop): + assert hop.args_s[0].is_constant() + v_input, = hop.inputargs(lltype.Signed) + hop.exception_cannot_occur() + return hop.genop('cast_int_to_adr', [v_input], + resulttype = llmemory.Address) + def rtype_cast_ptr_to_weakadr(hop): vlist = hop.inputargs(hop.args_r[0]) assert isinstance(vlist[0].concretetype, lltype.Ptr) Modified: pypy/dist/pypy/translator/c/src/address.h ============================================================================== --- pypy/dist/pypy/translator/c/src/address.h (original) +++ pypy/dist/pypy/translator/c/src/address.h Fri Aug 25 15:28:23 2006 @@ -17,6 +17,7 @@ #define OP_ADR_GE(x,y,r) r = ((x) >= (y)) #define OP_CAST_ADR_TO_INT(x, r) r = ((long)x) +#define OP_CAST_INT_TO_ADR(x, r) r = ((void *)(x)) #define OP_CAST_WEAKADR_TO_INT(x, r) r = ((long)x) #ifndef HIDE_POINTER From arigo at codespeak.net Fri Aug 25 16:44:01 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Aug 2006 16:44:01 +0200 (CEST) Subject: [pypy-svn] r31642 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20060825144401.51CD41007B@code0.codespeak.net> Author: arigo Date: Fri Aug 25 16:43:58 2006 New Revision: 31642 Modified: pypy/dist/pypy/rpython/memory/gc.py pypy/dist/pypy/rpython/memory/gctransform.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: (pedronis, arigo) Fix. See failure in rpython.memory.test.test_transformed_gc, test_llinterp_lists. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Fri Aug 25 16:43:58 2006 @@ -448,7 +448,9 @@ hdr = next self.collect_in_progress = False - STATISTICS_NUMBERS = 2 + STAT_HEAP_USAGE = 0 + STAT_BYTES_MALLOCED = 1 + STATISTICS_NUMBERS = 2 def add_reachable_to_stack(self, obj, objects): size_gc_header = self.gcheaderbuilder.size_gc_header @@ -477,8 +479,13 @@ j += 1 i += 1 - def statistics(self): - return self.heap_usage, self.bytes_malloced + def statistics(self, index): + # no memory allocation here! + if index == self.STAT_HEAP_USAGE: + return self.heap_usage + if index == self.STAT_BYTES_MALLOCED: + return self.bytes_malloced + return -1 def size_gc_header(self, typeid=0): return self.gcheaderbuilder.size_gc_header Modified: pypy/dist/pypy/rpython/memory/gctransform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform.py Fri Aug 25 16:43:58 2006 @@ -922,9 +922,9 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc], annmodel.s_None) - statics_s = (annmodel.SomeInteger(),)*GCClass.STATISTICS_NUMBERS self.statistics_ptr = getfn(GCClass.statistics.im_func, - [s_gc], annmodel.SomeTuple(statics_s)) + [s_gc, annmodel.SomeInteger()], + annmodel.SomeInteger()) # experimental gc_x_* operations s_x_pool = annmodel.SomePtr(gc.X_POOL_PTR) Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Fri Aug 25 16:43:58 2006 @@ -219,8 +219,8 @@ if statistics: statisticsgraph = db.gctransformer.statistics_ptr.value._obj.graph ll_gc = db.gctransformer.c_const_gc.value - def statistics(): - return llinterp.eval_graph(statisticsgraph, [ll_gc]) + def statistics(index): + return llinterp.eval_graph(statisticsgraph, [ll_gc, index]) return run, statistics else: return run @@ -230,7 +230,11 @@ class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(gctransform.FrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } - + + def heap_usage(self, statistics): + return statistics( + self.gcpolicy.transformerclass.GCClass.STAT_HEAP_USAGE) + def test_llinterp_lists(self): def malloc_a_lot(): i = 0 @@ -244,7 +248,7 @@ return 0 run, statistics = self.runner(malloc_a_lot, statistics=True) run([]) - heap_size = statistics().item0 + heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx def test_llinterp_tuples(self): @@ -261,7 +265,7 @@ return 0 run, statistics = self.runner(malloc_a_lot, statistics=True) run([]) - heap_size = statistics().item0 + heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx def test_global_list(self): @@ -290,7 +294,7 @@ run, statistics = self.runner(concat, nbargs=2, statistics=True) res = run([100, 0]) assert res == concat(100, 0) - heap_size = statistics().item0 + heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx def test_finalizer(self): From ac at codespeak.net Fri Aug 25 17:06:56 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Fri, 25 Aug 2006 17:06:56 +0200 (CEST) Subject: [pypy-svn] r31644 - pypy/dist/pypy/rpython/memory Message-ID: <20060825150656.246EA10089@code0.codespeak.net> Author: ac Date: Fri Aug 25 17:06:55 2006 New Revision: 31644 Modified: pypy/dist/pypy/rpython/memory/gc.py Log: Fix division by zero problem. Modified: pypy/dist/pypy/rpython/memory/gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc.py (original) +++ pypy/dist/pypy/rpython/memory/gc.py Fri Aug 25 17:06:55 2006 @@ -350,18 +350,13 @@ compute_time = start_time - self.prev_collect_end_time collect_time = end_time - start_time + garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) - - - collect_time_fraction = collect_time / compute_time - old_heapsize = self.heap_usage - garbage_generated = old_malloced - (curr_heap_size - old_heapsize) - garbage_fraction = float(garbage_generated) / float(curr_heap_size) - - - if collect_time_fraction > 0.02 * garbage_fraction: + if (collect_time * curr_heap_size > + 0.02 * garbage_collected * compute_time): self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 - if collect_time_fraction < 0.005 * garbage_fraction: + if (collect_time * curr_heap_size < + 0.005 * garbage_collected * compute_time): self.bytes_malloced_threshold /= 2 # Use atleast as much memory as current live objects. @@ -390,11 +385,11 @@ " total time spent collecting: ", self.total_collection_time, "seconds") llop.debug_print(lltype.Void, - " collecting time fraction: ", - collect_time_fraction) + " collecting time: ", + collect_time) llop.debug_print(lltype.Void, - " garbage fraction: ", - garbage_fraction) + " computing time: ", + collect_time) llop.debug_print(lltype.Void, " new threshold: ", self.bytes_malloced_threshold) From fijal at codespeak.net Fri Aug 25 17:07:16 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 17:07:16 +0200 (CEST) Subject: [pypy-svn] r31645 - in pypy/dist/pypy/rpython: . test Message-ID: <20060825150716.402881008B@code0.codespeak.net> Author: fijal Date: Fri Aug 25 17:07:13 2006 New Revision: 31645 Modified: pypy/dist/pypy/rpython/nonconst.py pypy/dist/pypy/rpython/test/test_nonconst.py Log: Fixed non-const list, but I'm not sure if it works. Modified: pypy/dist/pypy/rpython/nonconst.py ============================================================================== --- pypy/dist/pypy/rpython/nonconst.py (original) +++ pypy/dist/pypy/rpython/nonconst.py Fri Aug 25 17:07:13 2006 @@ -14,7 +14,10 @@ _about_ = NonConstant def compute_result_annotation(self, arg): - return getbookkeeper().annotation_from_example(arg.const) + if hasattr(arg, 'const'): + return getbookkeeper().annotation_from_example(arg.const) + else: + return arg def specialize_call(self, hop): v = Variable() Modified: pypy/dist/pypy/rpython/test/test_nonconst.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_nonconst.py (original) +++ pypy/dist/pypy/rpython/test/test_nonconst.py Fri Aug 25 17:07:13 2006 @@ -16,3 +16,13 @@ s = a.build_types(nonconst_f, []) assert s.knowntype is int assert not hasattr(s, 'const') + +def test_nonconst_list(): + def nonconst_l(): + a = NonConstant([1, 2, 3]) + return a[0] + + a = RPythonAnnotator() + s = a.build_types(nonconst_l, []) + assert s.knowntype is int + assert not hasattr(s, 'const') From ericvrp at codespeak.net Fri Aug 25 17:16:51 2006 From: ericvrp at codespeak.net (ericvrp at codespeak.net) Date: Fri, 25 Aug 2006 17:16:51 +0200 (CEST) Subject: [pypy-svn] r31646 - pypy/dist/pypy/translator/js/turbogears Message-ID: <20060825151651.1957510084@code0.codespeak.net> Author: ericvrp Date: Fri Aug 25 17:16:46 2006 New Revision: 31646 Added: pypy/dist/pypy/translator/js/turbogears/ pypy/dist/pypy/translator/js/turbogears/__init__.py (contents, props changed) pypy/dist/pypy/translator/js/turbogears/setup.py (contents, props changed) pypy/dist/pypy/translator/js/turbogears/templateplugin.py (contents, props changed) pypy/dist/pypy/translator/js/turbogears/upload.sh (contents, props changed) Log: Some WIP for nice TurboGears integration of PyPy-JS. * the setup script to create a Cheeseshop egg. * An upload shell script (will replace with Pythoncode later) because the egg is currently actually stored on codespeak. (because Cheeseshop allows only 5Mb per egg) * A TG plugin that will (hopefully) someday soon allow @expose(template="asjavascript") Added: pypy/dist/pypy/translator/js/turbogears/__init__.py ============================================================================== Added: pypy/dist/pypy/translator/js/turbogears/setup.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/turbogears/setup.py Fri Aug 25 17:16:46 2006 @@ -0,0 +1,75 @@ +from setuptools import setup, find_packages +from turbogears.finddata import find_package_data +import os + +svninfo = os.popen('svn info rpython2javascript').read().split() +version = int(svninfo[svninfo.index('Revision:') + 1]) + +setup( + name="rpython2javascript", + version="0.%d" % version, + + description="RPython to JavaScript translator", + #description_long="""This is a derivative of the PyPy project. Some words about RPython can be found at http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#restricted-python""", + author="PyPy developers, Maciek Fijalkowski, Eric van Riet Paap", + author_email="pypy-dev at codespeak.net", + url="http://codespeak.net/pypy", + download_url="http://codespeak.net/~ericvrp/rpython2javascript/", + license="MIT", + + #install_requires = [ + # "TurboGears >= 1.1a0", + #], + + #scripts = [ + # "start-topl.py" + #], + + zip_safe=False, + + packages=find_packages(), + package_data=find_package_data( + where='rpython2javascript', package='rpython2javascript', + only_in_packages=False, exclude=('*.pyc', '*~', '.*', '*.bak')), + + entry_points=""" + [python.templating.engines] + asjavascript = rpython2javascript.pypy.translator.js.turbogears.templateplugin:TemplatePlugin""", + + #keywords = [ + # # Use keywords if you'll be adding your package to the + # # Python Cheeseshop + # + # # if this has widgets, uncomment the next line + # # 'turbogears.widgets', + # + # # if this has a tg-admin command, uncomment the next line + # # 'turbogears.command', + # + # # if this has identity providers, uncomment the next line + # # 'turbogears.identity.provider', + # + # # If this is a template plugin, uncomment the next line + # # 'python.templating.engines', + # + # # If this is a full application, uncomment the next line + # # 'turbogears.app', + #], + + classifiers = [ + 'Development Status :: 3 - Alpha', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + #'Framework :: TurboGears', + # if this is an application that you'll distribute through + # the Cheeseshop, uncomment the next line + # 'Framework :: TurboGears :: Applications', + + # if this is a package that includes widgets that you'll distribute + # through the Cheeseshop, uncomment the next line + # 'Framework :: TurboGears :: Widgets', + ], + #test_suite = 'nose.collector', + ) + Added: pypy/dist/pypy/translator/js/turbogears/templateplugin.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/turbogears/templateplugin.py Fri Aug 25 17:16:46 2006 @@ -0,0 +1,53 @@ +import cherrypy +from rpython2javascript import rpython2javascript + +class TemplatePlugin: + + def __init__(self, extra_vars_func=None, options=None): + """The standard constructor takes an 'extra_vars_func', + which is a callable that is called for additional + variables with each rendering. Options is a dictionary + that provides options specific to template engines + (encoding, for example). The options should be + prefixed with the engine's scheme name to allow the + same dictionary to be passed in to multiple engines + without ill effects.""" + pass + + # the template name will be in python "dot" notation + # eg "package1.package2.templatename". It will *not* + # have the extension on it. You may want to cache the + # template. This method is only called directly if a + # template is specified in turbogears.view.baseTemplates. + # You might call this yourself from render. + # This doesn't *have* to return anything, but + # existing implementations return a template class. + # (this does not necessarily make sense for all template + # engines, though, which is why no return value is + # required.) + def load_template(self, templatename): + "Find a template specified in python 'dot' notation." + pass + + # info is the dictionary returned by the user's controller. + # format may only make sense for template engines that can + # produce different styles of output based on the same + # template. + # fragment is used if there are special rules about rendering + # a part of a page (don't include headers and declarations). + # template is the name of the template to render. + # You should incorporate extra_vars_func() output + # into the namespace in your template if at all possible. + def render(self, info, format="html", fragment=False, template=None): + "Renders the template to a string using the provided info." + cherrypy.response.headers["Content-Type"] = "text/javascript" + return 'alert("JavascriptCodeGoesHere")' + + # This method is not required for most uses of templates. + # It is specifically used for efficiently inserting widget + # output into Kid pages. It does the same thing render does, + # except the output is a generator of ElementTree Elements + # rather than a string. + def transform(self, info, template): + "Render the output to Elements" + pass Added: pypy/dist/pypy/translator/js/turbogears/upload.sh ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/turbogears/upload.sh Fri Aug 25 17:16:46 2006 @@ -0,0 +1,11 @@ +#!/bin/bash + +#This should be run from the directory where you checked out pypy-dist + +svn up rpython2javascript + +rm -f dist/*.egg +python ./rpython2javascript/pypy/translator/js/turbogears/setup.py bdist_egg +scp dist/*.egg ericvrp at codespeak.net:public_html/rpython2javascript + +#python ./rpython2javascript/pypy/translator/js/turbogears/setup.py register From arigo at codespeak.net Fri Aug 25 17:21:48 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Aug 2006 17:21:48 +0200 (CEST) Subject: [pypy-svn] r31647 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20060825152148.33E5C10089@code0.codespeak.net> Author: arigo Date: Fri Aug 25 17:21:46 2006 New Revision: 31647 Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Log: (arre, arigo) Converted all tests to a format that is suitable for being reused from codegen/jit/test/test_timeshift. Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Fri Aug 25 17:21:46 2006 @@ -181,9 +181,15 @@ residual_graph = c_generatedfn.value._obj.graph if conftest.option.view: residual_graph.show() - insns = summary(residual_graph) + self.insns = summary(residual_graph) res = llinterp.eval_graph(residual_graph, residual_args) - return insns, res + return res + + def check_insns(self, expected=None, **counts): + if expected is not None: + assert self.insns == expected + for opname, count in counts.items(): + assert self.insns.get(opname, 0) == count class TestTimeshift(TimeshiftingTests): @@ -192,16 +198,16 @@ py.test.skip("green return not working") def ll_function(x, y): return hint(x + y, concrete=True) - insns, res = self.timeshift(ll_function, [5, 7]) + res = self.timeshift(ll_function, [5, 7]) assert res == 12 - assert insns == {} + self.check_insns({}) def test_very_simple(self): def ll_function(x, y): return x + y - insns, res = self.timeshift(ll_function, [5, 7]) + res = self.timeshift(ll_function, [5, 7]) assert res == 12 - assert insns == {'int_add': 1} + self.check_insns({'int_add': 1}) def test_convert_const_to_redbox(self): def ll_function(x, y): @@ -211,23 +217,23 @@ tot += y x -= 1 return tot - insns, res = self.timeshift(ll_function, [7, 2]) + res = self.timeshift(ll_function, [7, 2]) assert res == 14 - assert insns == {'int_add': 7} + self.check_insns({'int_add': 7}) def test_simple_opt_const_propagation2(self): def ll_function(x, y): return x + y - insns, res = self.timeshift(ll_function, [5, 7], [0, 1]) + res = self.timeshift(ll_function, [5, 7], [0, 1]) assert res == 12 - assert insns == {} + self.check_insns({}) def test_simple_opt_const_propagation1(self): def ll_function(x): return -x - insns, res = self.timeshift(ll_function, [5], [0]) + res = self.timeshift(ll_function, [5], [0]) assert res == -5 - assert insns == {} + self.check_insns({}) def test_loop_folding(self): def ll_function(x, y): @@ -237,9 +243,9 @@ tot += y x -= 1 return tot - insns, res = self.timeshift(ll_function, [7, 2], [0, 1]) + res = self.timeshift(ll_function, [7, 2], [0, 1]) assert res == 14 - assert insns == {} + self.check_insns({}) def test_loop_merging(self): def ll_function(x, y): @@ -248,25 +254,25 @@ tot += y x -= 1 return tot - insns, res = self.timeshift(ll_function, [7, 2], []) + res = self.timeshift(ll_function, [7, 2], []) assert res == 14 - assert insns['int_add'] == 2 - assert insns['int_is_true'] == 2 + self.check_insns(int_add = 2, + int_is_true = 2) - insns, res = self.timeshift(ll_function, [7, 2], [0]) + res = self.timeshift(ll_function, [7, 2], [0]) assert res == 14 - assert insns['int_add'] == 2 - assert insns['int_is_true'] == 1 + self.check_insns(int_add = 2, + int_is_true = 1) - insns, res = self.timeshift(ll_function, [7, 2], [1]) + res = self.timeshift(ll_function, [7, 2], [1]) assert res == 14 - assert insns['int_add'] == 1 - assert insns['int_is_true'] == 2 + self.check_insns(int_add = 1, + int_is_true = 2) - insns, res = self.timeshift(ll_function, [7, 2], [0, 1]) + res = self.timeshift(ll_function, [7, 2], [0, 1]) assert res == 14 - assert insns['int_add'] == 1 - assert insns['int_is_true'] == 1 + self.check_insns(int_add = 1, + int_is_true = 1) def test_two_loops_merging(self): def ll_function(x, y): @@ -278,21 +284,21 @@ tot += y y -= 1 return tot - insns, res = self.timeshift(ll_function, [7, 3], []) + res = self.timeshift(ll_function, [7, 3], []) assert res == 27 - assert insns['int_add'] == 3 - assert insns['int_is_true'] == 3 + self.check_insns(int_add = 3, + int_is_true = 3) def test_convert_greenvar_to_redvar(self): def ll_function(x, y): hint(x, concrete=True) return x - y - insns, res = self.timeshift(ll_function, [70, 4], [0]) + res = self.timeshift(ll_function, [70, 4], [0]) assert res == 66 - assert insns['int_sub'] == 1 - insns, res = self.timeshift(ll_function, [70, 4], [0, 1]) + self.check_insns(int_sub = 1) + res = self.timeshift(ll_function, [70, 4], [0, 1]) assert res == 66 - assert insns == {} + self.check_insns({}) def test_green_across_split(self): def ll_function(x, y): @@ -302,10 +308,10 @@ else: z = x + y return z - insns, res = self.timeshift(ll_function, [70, 4], [0]) + res = self.timeshift(ll_function, [70, 4], [0]) assert res == 66 - assert insns['int_add'] == 1 - assert insns['int_sub'] == 1 + self.check_insns(int_add = 1, + int_sub = 1) def test_merge_const_before_return(self): def ll_function(x): @@ -316,9 +322,9 @@ x -= 1 y += 1 return y+x - insns, res = self.timeshift(ll_function, [-70], []) + res = self.timeshift(ll_function, [-70], []) assert res == 23-71 - assert insns == {'int_gt': 1, 'int_add': 2, 'int_sub': 2} + self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) def test_merge_3_redconsts_before_return(self): def ll_function(x): @@ -331,11 +337,11 @@ x -= 1 y += 1 return y+x - insns, res = self.timeshift(ll_function, [-70], []) + res = self.timeshift(ll_function, [-70], []) assert res == ll_function(-70) - insns, res = self.timeshift(ll_function, [1], []) + res = self.timeshift(ll_function, [1], []) assert res == ll_function(1) - insns, res = self.timeshift(ll_function, [-70], []) + res = self.timeshift(ll_function, [-70], []) assert res == ll_function(-70) def test_merge_const_at_return(self): @@ -345,9 +351,9 @@ return 17 else: return 22 - insns, res = self.timeshift(ll_function, [-70], []) + res = self.timeshift(ll_function, [-70], []) assert res == 22 - assert insns == {'int_gt': 1} + self.check_insns({'int_gt': 1}) def test_arith_plus_minus(self): def ll_plus_minus(encoded_insn, nb_insn, x, y): @@ -363,10 +369,9 @@ pc += 1 return acc assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - insns, res = self.timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) + res = self.timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) assert res == 42 - assert insns == {'int_add': 2, - 'int_sub': 1} + self.check_insns({'int_add': 2, 'int_sub': 1}) def test_simple_struct(self): S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), @@ -377,13 +382,12 @@ s1 = lltype.malloc(S) s1.hello = 6 s1.world = 7 - insns, res = self.timeshift(ll_function, [s1], []) + res = self.timeshift(ll_function, [s1], []) assert res == 42 - assert insns == {'getfield': 2, - 'int_mul': 1} - insns, res = self.timeshift(ll_function, [s1], [0]) + self.check_insns({'getfield': 2, 'int_mul': 1}) + res = self.timeshift(ll_function, [s1], [0]) assert res == 42 - assert insns == {} + self.check_insns({}) def test_simple_array(self): A = lltype.GcArray(lltype.Signed, @@ -393,13 +397,12 @@ a1 = lltype.malloc(A, 2) a1[0] = 6 a1[1] = 7 - insns, res = self.timeshift(ll_function, [a1], []) + res = self.timeshift(ll_function, [a1], []) assert res == 42 - assert insns == {'getarrayitem': 2, - 'int_mul': 1} - insns, res = self.timeshift(ll_function, [a1], [0]) + self.check_insns({'getarrayitem': 2, 'int_mul': 1}) + res = self.timeshift(ll_function, [a1], [0]) assert res == 42 - assert insns == {} + self.check_insns({}) def test_simple_struct_malloc(self): py.test.skip("blue containers: to be reimplemented") @@ -410,13 +413,13 @@ s.hello = x return s.hello + s.world - insns, res = self.timeshift(ll_function, [3], []) + res = self.timeshift(ll_function, [3], []) assert res == 3 - assert insns == {'int_add': 1} + self.check_insns({'int_add': 1}) - insns, res = self.timeshift(ll_function, [3], [0]) + res = self.timeshift(ll_function, [3], [0]) assert res == 3 - assert insns == {} + self.check_insns({}) def test_inlined_substructure(self): py.test.skip("blue containers: to be reimplemented") @@ -427,13 +430,13 @@ t.s.n = k l = t.s.n return l - insns, res = self.timeshift(ll_function, [7], []) + res = self.timeshift(ll_function, [7], []) assert res == 7 - assert insns == {} + self.check_insns({}) - insns, res = self.timeshift(ll_function, [7], [0]) + res = self.timeshift(ll_function, [7], [0]) assert res == 7 - assert insns == {} + self.check_insns({}) def test_degenerated_before_return(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) @@ -448,9 +451,9 @@ s = t.s s.n += 1 return s.n * t.s.n - insns, res = self.timeshift(ll_function, [0], []) + res = self.timeshift(ll_function, [0], []) assert res == 5 * 3 - insns, res = self.timeshift(ll_function, [1], []) + res = self.timeshift(ll_function, [1], []) assert res == 4 * 4 def test_degenerated_before_return_2(self): @@ -468,9 +471,9 @@ s = t.s s.n += 1 return s.n * t.s.n - insns, res = self.timeshift(ll_function, [1], []) + res = self.timeshift(ll_function, [1], []) assert res == 5 * 3 - insns, res = self.timeshift(ll_function, [0], []) + res = self.timeshift(ll_function, [0], []) assert res == 4 * 4 def test_degenerated_at_return(self): @@ -486,10 +489,10 @@ if flag: s = t.s return s - insns, res = self.timeshift(ll_function, [0], []) + res = self.timeshift(ll_function, [0], []) assert res.n == 4 assert lltype.parentlink(res._obj) == (None, None) - insns, res = self.timeshift(ll_function, [1], []) + res = self.timeshift(ll_function, [1], []) assert res.n == 3 parent, parentindex = lltype.parentlink(res._obj) assert parentindex == 's' @@ -510,9 +513,9 @@ s = t.s t.s.n += 1 return s.n * t.s.n - insns, res = self.timeshift(ll_function, [1], []) + res = self.timeshift(ll_function, [1], []) assert res == 7 * 4 - insns, res = self.timeshift(ll_function, [0], []) + res = self.timeshift(ll_function, [0], []) assert res == 4 * 4 def test_plus_minus_all_inlined(self): @@ -530,9 +533,9 @@ pc += 1 return acc s = rstr.string_repr.convert_const("+-+") - insns, res = self.timeshift(ll_plus_minus, [s, 0, 2], [0], inline=999) + res = self.timeshift(ll_plus_minus, [s, 0, 2], [0], inline=999) assert res == ll_plus_minus("+-+", 0, 2) - assert insns == {'int_add': 2, 'int_sub': 1} + self.check_insns({'int_add': 2, 'int_sub': 1}) def test_red_virtual_container(self): # this checks that red boxes are able to be virtualized dynamically by @@ -543,9 +546,9 @@ s = lltype.malloc(S) s.n = n return s.n - insns, res = self.timeshift(ll_function, [42], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [42], [], policy=P_NOVIRTUAL) assert res == 42 - assert insns == {} + self.check_insns({}) def test_red_propagate(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) @@ -555,9 +558,9 @@ if k < 0: return -123 return s.n * k - insns, res = self.timeshift(ll_function, [3, 8], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [3, 8], [], policy=P_NOVIRTUAL) assert res == 24 - assert insns == {'int_lt': 1, 'int_mul': 1} + self.check_insns({'int_lt': 1, 'int_mul': 1}) def test_red_subcontainer(self): S = lltype.Struct('S', ('n', lltype.Signed)) @@ -571,9 +574,9 @@ result = s.n * (k-1) keepalive_until_here(t) return result - insns, res = self.timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) assert res == 42 - assert insns == {'int_lt': 1, 'int_mul': 1, 'int_sub': 1} + self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) def test_merge_structures(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) @@ -593,30 +596,30 @@ t.s = s t.n = 6 return t.n + t.s.n - insns, res = self.timeshift(ll_function, [0], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [0], [], policy=P_NOVIRTUAL) assert res == 5 + 6 - assert insns == {'int_is_true': 1, 'int_add': 1} - insns, res = self.timeshift(ll_function, [1], [], policy=P_NOVIRTUAL) + self.check_insns({'int_is_true': 1, 'int_add': 1}) + res = self.timeshift(ll_function, [1], [], policy=P_NOVIRTUAL) assert res == 1 + 2 - assert insns == {'int_is_true': 1, 'int_add': 1} + self.check_insns({'int_is_true': 1, 'int_add': 1}) def test_call_simple(self): def ll_add_one(x): return x + 1 def ll_function(y): return ll_add_one(y) - insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) assert res == 6 - assert insns == {'int_add': 1} + self.check_insns({'int_add': 1}) def test_call_2(self): def ll_add_one(x): return x + 1 def ll_function(y): return ll_add_one(y) + y - insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) assert res == 11 - assert insns == {'int_add': 2} + self.check_insns({'int_add': 2}) def test_call_3(self): def ll_add_one(x): @@ -625,6 +628,6 @@ return ll_add_one(ll_add_one(x)) - x def ll_function(y): return ll_two(y) * y - insns, res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) + res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) assert res == 10 - assert insns == {'int_add': 2, 'int_sub': 1, 'int_mul': 1} + self.check_insns({'int_add': 2, 'int_sub': 1, 'int_mul': 1}) From mwh at codespeak.net Fri Aug 25 17:27:13 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 25 Aug 2006 17:27:13 +0200 (CEST) Subject: [pypy-svn] r31649 - in pypy/extradoc/talk: . sprint-introduction.key sprint-introduction.key/thumbs Message-ID: <20060825152713.6928E10089@code0.codespeak.net> Author: mwh Date: Fri Aug 25 17:27:02 2006 New Revision: 31649 Modified: pypy/extradoc/talk/sprint-introduction.key/index.apxl.gz pypy/extradoc/talk/sprint-introduction.key/thumbs/st0.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st1-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st10.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st11-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st14.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st15.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st16-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st16.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st17.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st18.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st19.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st2.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st20.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st22-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st22.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st24-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st24.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st26.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st27-1.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st27.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st29.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st3-4.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st30.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st6.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st8.tiff pypy/extradoc/talk/sprint-introduction.key/thumbs/st9.tiff pypy/extradoc/talk/sprint-introduction.pdf pypy/extradoc/talk/sprint-introduction.ppt Log: (mwh, pedronis) minor updates to the sprint introduction for the Limerick sprint. Modified: pypy/extradoc/talk/sprint-introduction.key/index.apxl.gz ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st0.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st1-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st10.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st11-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st14.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st15.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st16-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st16.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st17.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st18.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st19.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st2.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st20.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st22-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st22.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st24-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st24.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st26.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st27-1.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st27.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st29.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st3-4.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st30.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st6.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st8.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.key/thumbs/st9.tiff ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/sprint-introduction.pdf ============================================================================== Files pypy/extradoc/talk/sprint-introduction.pdf (original) and pypy/extradoc/talk/sprint-introduction.pdf Fri Aug 25 17:27:02 2006 differ Modified: pypy/extradoc/talk/sprint-introduction.ppt ============================================================================== Binary files. No diff available. From micktwomey at codespeak.net Fri Aug 25 18:09:55 2006 From: micktwomey at codespeak.net (micktwomey at codespeak.net) Date: Fri, 25 Aug 2006 18:09:55 +0200 (CEST) Subject: [pypy-svn] r31650 - in pypy/dist/pypy: bin translator/js Message-ID: <20060825160955.9ED3310080@code0.codespeak.net> Author: micktwomey Date: Fri Aug 25 18:09:50 2006 New Revision: 31650 Added: pypy/dist/pypy/translator/js/main.py Modified: pypy/dist/pypy/bin/jscompile.py Log: Refactored js backend main call rpython2javascript. Modified: pypy/dist/pypy/bin/jscompile.py ============================================================================== --- pypy/dist/pypy/bin/jscompile.py (original) +++ pypy/dist/pypy/bin/jscompile.py Fri Aug 25 18:09:50 2006 @@ -6,55 +6,7 @@ import autopath import sys -from pypy.translator.js.test.runtest import compile_function -#from pypy.translator.translator import TranslationContext -from pypy.translator.driver import TranslationDriver -from pypy.translator.js.js import JS -from pypy.tool.error import AnnotatorError, FlowingError, debug -from pypy.rpython.nonconst import NonConstant -from pypy.annotation.policy import AnnotatorPolicy +from pypy.translator.js.main import rpython2javascript_main -class FunctionNotFound(Exception): - pass - -class BadSignature(Exception): - pass - -class JsPolicy(AnnotatorPolicy): - allow_someobjects = False - -def get_args(func_data): - l = [] - for i in xrange(func_data.func_code.co_argcount): - l.append("NonConstant(%s)" % repr(func_data.func_defaults[i])) - return "(%s)" % ",".join(l) - -def rpython2javascript(argv): - if len(argv) < 2: - print __doc__ - sys.exit(0) - module_name = argv[0] - function_names = argv[1:] - mod = __import__(module_name, None, None, ["Module"]) - for func_name in function_names: - if func_name not in mod.__dict__: - raise FunctionNotFound("function %r was not found in module %r" % (func_name, module_name)) - func_code = mod.__dict__[func_name] - if func_code.func_code.co_argcount > 0 and func_code.func_code.co_argcount != len(func_code.func_defaults): - raise BadSignature("Function %s does not have default arguments" % func_name) - source_ssf = "\n".join(["import %s" % module_name, "def some_strange_function_which_will_never_be_called():"] + [" "+\ - module_name+"."+fun_name+get_args(mod.__dict__[func_name]) for fun_name in function_names]) - print source_ssf - exec(source_ssf) in globals() - #fn = compile_function([mod.__dict__[f_name] for f_name in function_names], [[] for i in function_names]) - # now we gonna just cut off not needed function - driver = TranslationDriver() - try: - driver.setup(some_strange_function_which_will_never_be_called, [], policy = JsPolicy()) - driver.proceed(["compile_js"]) - except Exception, e: - # do something nice with it - debug(driver) - if __name__ == '__main__': - rpython2javascript(sys.argv[1:]) + rpython2javascript_main(sys.argv[1:]) Added: pypy/dist/pypy/translator/js/main.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/main.py Fri Aug 25 18:09:50 2006 @@ -0,0 +1,57 @@ +"""Contains high level javascript compilation function +""" + +from pypy.translator.js.test.runtest import compile_function +#from pypy.translator.translator import TranslationContext +from pypy.translator.driver import TranslationDriver +from pypy.translator.js.js import JS +from pypy.tool.error import AnnotatorError, FlowingError, debug +from pypy.rpython.nonconst import NonConstant +from pypy.annotation.policy import AnnotatorPolicy + +class FunctionNotFound(Exception): + pass + +class BadSignature(Exception): + pass + +class JsPolicy(AnnotatorPolicy): + allow_someobjects = False + +def get_args(func_data): + l = [] + for i in xrange(func_data.func_code.co_argcount): + l.append("NonConstant(%s)" % repr(func_data.func_defaults[i])) + return "(%s)" % ",".join(l) + +def rpython2javascript_main(argv): + if len(argv) < 2: + print __doc__ + sys.exit(0) + module_name = argv[0] + function_names = argv[1:] + mod = __import__(module_name, None, None, ["Module"]) + rpython2javascript(mod, function_names) + +def rpython2javascript(mod, function_names): + module_name = mod.__name__ + for func_name in function_names: + if func_name not in mod.__dict__: + raise FunctionNotFound("function %r was not found in module %r" % (func_name, module_name)) + func_code = mod.__dict__[func_name] + if func_code.func_code.co_argcount > 0 and func_code.func_code.co_argcount != len(func_code.func_defaults): + raise BadSignature("Function %s does not have default arguments" % func_name) + source_ssf = "\n".join(["import %s" % module_name, "def some_strange_function_which_will_never_be_called():"] + [" "+\ + module_name+"."+fun_name+get_args(mod.__dict__[func_name]) for fun_name in function_names]) + print source_ssf + exec(source_ssf) in globals() + #fn = compile_function([mod.__dict__[f_name] for f_name in function_names], [[] for i in function_names]) + # now we gonna just cut off not needed function + driver = TranslationDriver() + try: + driver.setup(some_strange_function_which_will_never_be_called, [], policy = JsPolicy()) + driver.proceed(["compile_js"]) + return driver.gen.tmpfile.open().read() + except Exception, e: + # do something nice with it + debug(driver) From mwh at codespeak.net Fri Aug 25 18:30:24 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 25 Aug 2006 18:30:24 +0200 (CEST) Subject: [pypy-svn] r31652 - in pypy/branch/no-zeroing-assumption/pypy: annotation rpython/lltypesystem rpython/lltypesystem/test rpython/ootypesystem Message-ID: <20060825163024.F285A10084@code0.codespeak.net> Author: mwh Date: Fri Aug 25 18:30:19 2006 New Revision: 31652 Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py Log: (pedronis, mwh) increase sanity in lltype and in the process make raw mallocs not zero pointer fields. see new test for intent. also, this lets me revert the changes i made to places that i shouldn't have needed to change in the first place. Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/binaryop.py Fri Aug 25 18:30:19 2006 @@ -669,7 +669,7 @@ example = p.ll_ptrtype._example() if example[0] is not None: # ignore Void s_value v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl(example=True) + example[0] = v_lltype._defl() setitem.can_only_throw = [] class __extend__(pairtype(SomePtr, SomeObject)): Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/builtin.py Fri Aug 25 18:30:19 2006 @@ -416,7 +416,7 @@ def cast_primitive(T, s_v): assert T.is_constant() - return ll_to_annotation(lltype.cast_primitive(T.const, annotation_to_lltype(s_v)._defl(example=True))) + return ll_to_annotation(lltype.cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) def nullptr(T): assert T.is_constant() @@ -426,13 +426,13 @@ def cast_pointer(PtrT, s_p): assert isinstance(s_p, SomePtr), "casting of non-pointer: %r" % s_p assert PtrT.is_constant() - cast_p = lltype.cast_pointer(PtrT.const, s_p.ll_ptrtype._defl(example=True)) + cast_p = lltype.cast_pointer(PtrT.const, s_p.ll_ptrtype._defl()) return SomePtr(ll_ptrtype=lltype.typeOf(cast_p)) def cast_opaque_ptr(PtrT, s_p): assert isinstance(s_p, SomePtr), "casting of non-pointer: %r" % s_p assert PtrT.is_constant() - cast_p = lltype.cast_opaque_ptr(PtrT.const, s_p.ll_ptrtype._defl(example=True)) + cast_p = lltype.cast_opaque_ptr(PtrT.const, s_p.ll_ptrtype._defl()) return SomePtr(ll_ptrtype=lltype.typeOf(cast_p)) def direct_fieldptr(s_p, s_fieldname): Modified: pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/annotation/unaryop.py Fri Aug 25 18:30:19 2006 @@ -605,10 +605,10 @@ example = p.ll_ptrtype._example() if getattr(example, s_attr.const) is not None: # ignore Void s_value v_lltype = annotation_to_lltype(s_value) - setattr(example, s_attr.const, v_lltype._defl(example=True)) + setattr(example, s_attr.const, v_lltype._defl()) def simple_call(p, *args_s): - llargs = [annotation_to_lltype(arg_s)._defl(example=True) for arg_s in args_s] + llargs = [annotation_to_lltype(arg_s)._defl() for arg_s in args_s] v = p.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/llmemory.py Fri Aug 25 18:30:19 2006 @@ -56,7 +56,7 @@ if (isinstance(self.TYPE, lltype.ContainerType) and self.TYPE._gckind == 'gc'): assert self.repeat == 1 - p = lltype.malloc(self.TYPE) + p = lltype.malloc(self.TYPE, flavor='raw') return cast_ptr_to_adr(p) else: T = lltype.FixedSizeArray(self.TYPE, self.repeat) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/lltype.py Fri Aug 25 18:30:19 2006 @@ -115,7 +115,11 @@ def _short_name(self): return str(self) - def _defl(self, parent=None, parentindex=None, example=False): + def _defl(self, parent=None, parentindex=None): + raise NotImplementedError + + def _allocate(self, initialization, parent=None, parentindex=None): + assert initialization in ('raw', 'malloc', 'example') raise NotImplementedError def _freeze_(self): @@ -254,16 +258,19 @@ def _short_name(self): return "%s %s" % (self.__class__.__name__, self._name) - def _defl(self, parent=None, parentindex=None, example=False): - return _struct(self, parent=parent, parentindex=parentindex, - example=example) +## def _defl(self, parent=None, parentindex=None): +## return _struct(self, parent=parent, parentindex=parentindex) + + def _allocate(self, initialization, parent=None, parentindex=None): + return _struct(self, initialization=initialization, + parent=parent, parentindex=parentindex) def _container_example(self): if self._arrayfld is None: n = None else: n = 1 - return _struct(self, n, example=True) + return _struct(self, n, initialization='example') class RttiStruct(Struct): _runtime_type_info = None @@ -359,7 +366,7 @@ _short_name = saferecursive(_short_name, '...') def _container_example(self): - return _array(self, 1, example=True) + return _array(self, 1, initialization='example') class GcArray(Array): _gckind = 'gc' @@ -425,7 +432,7 @@ def _container_example(self): def ex(*args): - return self.RESULT._defl(example=True) + return self.RESULT._defl() return _func(self, _callable=ex) def _trueargs(self): @@ -448,9 +455,12 @@ def _container_example(self): return _opaque(self) - def _defl(self, parent=None, parentindex=None, example=False): + def _defl(self, parent=None, parentindex=None): return _opaque(self, parent=parent, parentindex=parentindex) + def _allocate(self, initialization, parent=None, parentindex=None): + return self._defl(parent=parent, parentindex=parentindex) + RuntimeTypeInfo = OpaqueType("RuntimeTypeInfo") class GcOpaqueType(OpaqueType): @@ -469,8 +479,10 @@ return "PyObject" def _inline_is_varsize(self, last): return False - def _defl(self, parent=None, parentindex=None, example=False): + def _defl(self, parent=None, parentindex=None): return _pyobjheader(parent, parentindex) + def _allocate(self, initialization, parent=None, parentindex=None): + return self._defl(parent=parent, parentindex=parentindex) PyObject = PyObjectType() @@ -508,15 +520,19 @@ def __str__(self): return self._name - def _defl(self, parent=None, parentindex=None, example=False): - if not example and self is not Void: - return Uninitialized + def _defl(self, parent=None, parentindex=None): return self._default + def _allocate(self, initialization, parent=None, parentindex=None): + if self is not Void and initialization != 'example': + return Uninitialized + else: + return self._default + def _is_atomic(self): return True - def _example(self, parent=None, parentindex=None, example=False): + def _example(self, parent=None, parentindex=None): return self._default class Number(Primitive): @@ -580,8 +596,13 @@ def _is_atomic(self): return self.TO._gckind == 'raw' - def _defl(self, parent=None, parentindex=None, example=False): - if example or self._needsgc: + def _defl(self, parent=None, parentindex=None): + return _ptr(self, None) + + def _allocate(self, initialization, parent=None, parentindex=None): + if initialization == 'example': + return _ptr(self, None) + elif initialization == 'malloc' and self._needsgc(): return _ptr(self, None) else: return Uninitialized @@ -1210,11 +1231,11 @@ __slots__ = () - def __new__(self, TYPE, n=None, parent=None, parentindex=None, example=False): + def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, parent=None, parentindex=None, example=False): + def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1223,9 +1244,9 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, parent=self, parentindex=fld, example=example) + value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) else: - value = typ._defl(parent=self, parentindex=fld, example=example) + value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1287,13 +1308,13 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, parent=None, parentindex=None, example=False): + def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): if not isinstance(n, int): raise TypeError, "array length must be an int" if n < 0: raise ValueError, "negative array length" _parentable.__init__(self, TYPE) - self.items = [TYPE.OF._defl(parent=self, parentindex=j, example=example) + self.items = [TYPE.OF._allocate(initialization=initialization, parent=self, parentindex=j) for j in range(n)] if parent is not None: self._setparentstructure(parent, parentindex) @@ -1525,10 +1546,16 @@ def malloc(T, n=None, flavor='gc', immortal=False, extra_args=(), zero=False): + if zero or immortal: + initialization = 'example' + elif flavor == 'raw': + initialization = 'raw' + else: + initialization = 'malloc' if isinstance(T, Struct): - o = _struct(T, n, example=zero or immortal) + o = _struct(T, n, initialization=initialization) elif isinstance(T, Array): - o = _array(T, n, example=zero or immortal) + o = _array(T, n, initialization=initialization) else: raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): @@ -1555,7 +1582,7 @@ return _ptr(Ptr(TYPE), o) def nullptr(T): - return Ptr(T)._defl(example=True) + return Ptr(T)._defl() def opaqueptr(TYPE, name, **attrs): if not isinstance(TYPE, OpaqueType): Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rclass.py Fri Aug 25 18:30:19 2006 @@ -440,7 +440,7 @@ if attrvalue is None: warning("prebuilt instance %r has no attribute %r" % ( value, name)) - llattrvalue = r.lowleveltype._defl(example=True) + llattrvalue = r.lowleveltype._defl() else: llattrvalue = r.convert_desc_or_const(attrvalue) else: Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/rpbc.py Fri Aug 25 18:30:19 2006 @@ -73,7 +73,7 @@ return malloc(self.EMPTY, immortal=True) def null_instance(self): - return llmemory.Address._defl(example=True) + return llmemory.Address._defl() class __extend__(pairtype(MultipleUnrelatedFrozenPBCRepr, MultipleUnrelatedFrozenPBCRepr), Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/lltypesystem/test/test_llmemory.py Fri Aug 25 18:30:19 2006 @@ -295,6 +295,20 @@ py.test.raises(IndexError, "adr.signed[-1]") py.test.raises(IndexError, "adr.signed[1]") +def test_raw_malloc_access(): + S = lltype.GcStruct("S", ('x', lltype.Signed)) + T = lltype.GcStruct("T", ('y', lltype.Signed), ('s', lltype.Ptr(S))) + # regular malloc zeros GC pointers + p_t = lltype.malloc(T) + assert p_t.s == lltype.nullptr(S) + # raw malloc does not + p_raw_t = lltype.malloc(T, flavor="raw") + py.test.raises(lltype.UninitializedMemoryAccess, "p_raw_t.s") + # this sort of raw_malloc too + p_raw_t = cast_adr_to_ptr(raw_malloc(sizeof(T)), lltype.Ptr(T)) + py.test.raises(lltype.UninitializedMemoryAccess, "p_raw_t.s") + + def test_raw_malloc_signed_bunch(): adr = raw_malloc(sizeof(lltype.Signed) * 50) p = cast_adr_to_ptr(adr, Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/ootypesystem/ootype.py Fri Aug 25 18:30:19 2006 @@ -30,7 +30,7 @@ class Class(OOType): - def _defl(self, example=False): + def _defl(self): return nullruntimeclass Class = Class() @@ -66,7 +66,7 @@ def __hash__(self): return object.__hash__(self) - def _defl(self, example=False): + def _defl(self): return self._null def _example(self): return new(self) @@ -92,7 +92,7 @@ if isinstance(defn, Meth): raise TypeError("Attempting to store method in field") - fields[name] = (defn, defn._defl(example=True)) + fields[name] = (defn, defn._defl()) else: ootype, default = defn @@ -195,7 +195,7 @@ _retval = self.RESULT._example() return _static_meth(self, _callable=lambda *args: _retval) - def _defl(self, example=False): + def _defl(self): return null(self) def __repr__(self): @@ -219,7 +219,7 @@ def _example(self): return new(self) - def _defl(self, example=False): + def _defl(self): return self._null def _get_interp_class(self): @@ -233,10 +233,10 @@ def __init__(self, fields): self._fields = frozendict() for name, ITEMTYPE in fields.items(): - self._fields[name] = ITEMTYPE, ITEMTYPE._defl(example=True) + self._fields[name] = ITEMTYPE, ITEMTYPE._defl() self._null = _null_record(self) - def _defl(self, example=False): + def _defl(self): return self._null def _get_interp_class(self): @@ -321,7 +321,7 @@ return BuiltinADTType._enforce(self, value) # TODO: should it return _null or ''? - def _defl(self, example=False): + def _defl(self): return make_string("") def _example(self): return self._defl() @@ -344,7 +344,7 @@ }) self._setup_methods({}) - def _defl(self, example=False): + def _defl(self): return self._null def _get_interp_class(self): @@ -430,7 +430,7 @@ ITEMTYPE = self._specialize_type(self._ITEMTYPE, generic_types) return self.__class__(ITEMTYPE) - def _defl(self, example=False): + def _defl(self): return self._null def _set_itemtype(self, ITEMTYPE): From micktwomey at codespeak.net Fri Aug 25 18:30:46 2006 From: micktwomey at codespeak.net (micktwomey at codespeak.net) Date: Fri, 25 Aug 2006 18:30:46 +0200 (CEST) Subject: [pypy-svn] r31653 - pypy/dist/pypy/translator/js/tools Message-ID: <20060825163046.5682610084@code0.codespeak.net> Author: micktwomey Date: Fri Aug 25 18:30:44 2006 New Revision: 31653 Added: pypy/dist/pypy/translator/js/tools/__init__.py Log: Making tools a package. Added: pypy/dist/pypy/translator/js/tools/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/tools/__init__.py Fri Aug 25 18:30:44 2006 @@ -0,0 +1 @@ +# \ No newline at end of file From mwh at codespeak.net Fri Aug 25 18:55:59 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 25 Aug 2006 18:55:59 +0200 (CEST) Subject: [pypy-svn] r31654 - pypy/branch/no-zeroing-assumption/pypy/translator/c Message-ID: <20060825165559.95B9310080@code0.codespeak.net> Author: mwh Date: Fri Aug 25 18:55:58 2006 New Revision: 31654 Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py Log: (mwh, pedronis) disable some code we're *almost* certain isn't needed. Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/gc.py Fri Aug 25 18:55:58 2006 @@ -402,10 +402,7 @@ return defnode.db.gctransformer.gc_field_values_for(o) def zero_malloc(self, TYPE, esize, eresult): - assert TYPE._gckind == 'gc' # we don't really support this - typename = self.db.gettype(TYPE) - erestype = cdecl(typename, '*') - return 'OP_ZERO_MALLOC(%s, %s, %s);' % (esize, eresult, erestype) + assert False, "a malloc operation in a framework build??" malloc = zero_malloc From fijal at codespeak.net Fri Aug 25 19:06:38 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 19:06:38 +0200 (CEST) Subject: [pypy-svn] r31655 - pypy/dist/pypy/translator/js Message-ID: <20060825170638.C47DC10084@code0.codespeak.net> Author: fijal Date: Fri Aug 25 19:06:37 2006 New Revision: 31655 Modified: pypy/dist/pypy/translator/js/commproxy.py Log: (fijal, mtwomey) - Fixed xmlhttp request when used with decorators. Modified: pypy/dist/pypy/translator/js/commproxy.py ============================================================================== --- pypy/dist/pypy/translator/js/commproxy.py (original) +++ pypy/dist/pypy/translator/js/commproxy.py Fri Aug 25 19:06:37 2006 @@ -4,6 +4,7 @@ from pypy.objspace.flow.model import Variable, Constant +from pypy.rpython.ootypesystem.bltregistry import ArgDesc METHOD_BODY = """ %(class)s.prototype.%(method)s = function ( %(args)s ) { @@ -84,8 +85,10 @@ ilasm.begin_function(self.name, []) ilasm.end_function() - def render_method(self, method_name, method, ilasm): + def render_method(self, method_name, method, ilasm): args, retval = method.args, method.retval.name + if args[-1].name != 'callback': + args.append(ArgDesc('callback', lambda : None)) real_args = list(arg.name for arg in args) # FIXME: dirty JS here data = "{%s}" % ",".join(["'%s':%s" % (i,i) for i in real_args if i != 'callback']) From mwh at codespeak.net Fri Aug 25 19:16:52 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Fri, 25 Aug 2006 19:16:52 +0200 (CEST) Subject: [pypy-svn] r31656 - in pypy/branch/no-zeroing-assumption/pypy: rpython/memory translator/c translator/c/src translator/c/test Message-ID: <20060825171652.4A56C10084@code0.codespeak.net> Author: mwh Date: Fri Aug 25 19:16:50 2006 New Revision: 31656 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gc.py pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py Log: (mwh, a bit of pedronis) baby steps towards allocating non-zeroed memory. Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gc.py Fri Aug 25 19:16:50 2006 @@ -1,4 +1,4 @@ -from pypy.rpython.memory.lladdress import raw_malloc, raw_free, raw_memcopy +from pypy.rpython.memory.lladdress import raw_malloc, raw_free, raw_memcopy, raw_memclear from pypy.rpython.memory.lladdress import NULL, _address, raw_malloc_usage from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gcheader import GCHeaderBuilder @@ -195,6 +195,33 @@ # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + tot_size = size_gc_header + size + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + raw_memclear(result, tot_size) + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: @@ -227,6 +254,38 @@ # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, + can_collect, has_finalizer=False): + if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + size_gc_header = self.gcheaderbuilder.size_gc_header + try: + fixsize = size_gc_header + size + varsize = ovfcheck(itemsize * length) + tot_size = ovfcheck(fixsize + varsize) + usage = raw_malloc_usage(tot_size) + bytes_malloced = ovfcheck(self.bytes_malloced+usage) + ovfcheck(self.heap_usage + bytes_malloced) + except OverflowError: + raise memoryError + result = raw_malloc(tot_size) + (result + size_gc_header + offset_to_length).signed[0] = length + hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) + hdr.typeid = typeid << 1 + if has_finalizer: + hdr.next = self.malloced_objects_with_finalizer + self.malloced_objects_with_finalizer = hdr + else: + hdr.next = self.malloced_objects + self.malloced_objects = hdr + self.bytes_malloced = bytes_malloced + + result += size_gc_header + #llop.debug_print(lltype.Void, 'malloc_varsize length', length, + # 'typeid', typeid, + # '->', llmemory.cast_adr_to_int(result)) + return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + def collect(self): # 1. mark from the roots, and also the objects that objects-with-del # point to (using the list of malloced_objects_with_finalizer) Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py Fri Aug 25 19:16:50 2006 @@ -915,10 +915,20 @@ annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, inline = False) + self.malloc_fixedsize_clear_ptr = getfn( + GCClass.malloc_fixedsize_clear.im_func, + [s_gc, annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, + inline = False) self.malloc_varsize_ptr = getfn( GCClass.malloc_varsize.im_func, [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) + self.malloc_varsize_clear_ptr = getfn( + GCClass.malloc_varsize_clear.im_func, + [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc], annmodel.s_None) @@ -1205,15 +1215,21 @@ info = self.type_info_list[type_id] c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) if not op.opname.endswith('_varsize'): - args = [self.malloc_fixedsize_ptr, self.c_const_gc, c_type_id, - c_size, c_can_collect] + if op.opname.startswith('zero'): + p = self.malloc_fixedsize_clear_ptr + else: + p = self.malloc_fixedsize_clear_ptr + args = [p, self.c_const_gc, c_type_id, c_size, c_can_collect] else: v_length = op.args[-1] c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) - args = [self.malloc_varsize_ptr, self.c_const_gc, c_type_id, - v_length, c_size, c_varitemsize, c_ofstolength, - c_can_collect] + if op.opname.startswith('zero'): + p = self.malloc_varsize_clear_ptr + else: + p = self.malloc_varsize_clear_ptr + args = [p, self.c_const_gc, c_type_id, v_length, c_size, + c_varitemsize, c_ofstolength, c_can_collect] c_has_finalizer = rmodel.inputconst( lltype.Bool, bool(self.finalizer_funcptr_for_type(TYPE))) args.append(c_has_finalizer) @@ -1224,7 +1240,9 @@ ops.append(SpaceOperation("cast_opaque_ptr", [v], op.result)) return ops + replace_zero_malloc = replace_malloc replace_malloc_varsize = replace_malloc + replace_zero_malloc_varsize = replace_malloc replace_flavored_malloc = replace_malloc replace_flavored_malloc_varsize = replace_malloc Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/funcgen.py Fri Aug 25 19:16:50 2006 @@ -561,8 +561,7 @@ def OP_RAW_MALLOC(self, op): eresult = self.expr(op.result) esize = self.expr(op.args[0]) - return ("OP_RAW_MALLOC(%s, %s, void *);\n\t" % (esize, eresult) + - "if (%s) OP_RAW_MEM_ZERO(%s, %s);" % (eresult, eresult, esize)) + return "OP_RAW_MALLOC(%s, %s, void *);" % (esize, eresult) def OP_FLAVORED_MALLOC(self, op): TYPE = self.lltypemap(op.result).TO Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Fri Aug 25 19:16:50 2006 @@ -15,7 +15,7 @@ #define OP_RAW_FREE(p, r) PyObject_Free(p); COUNT_FREE; -#define OP_RAW_MEM_ZERO(p, size) memset((void*)p, 0, size) +#define OP_RAW_MEMCLEAR(p, size, r) memset((void*)p, 0, size) #define OP_RAW_MALLOC_USAGE(size, r) r = size @@ -55,7 +55,7 @@ #define OP_ZERO_MALLOC(size, r, restype) { \ OP_RAW_MALLOC(size, r, restype); \ - if (r != NULL) OP_RAW_MEM_ZERO(r, size); \ + if (r != NULL) OP_RAW_MEMCLEAR(r, size, /* */); \ } #define OP_FREE(p) OP_RAW_FREE(p, do_not_use) @@ -136,12 +136,12 @@ /************************************************************/ /* rcpy support */ -#define OP_CPY_MALLOC(cpytype, r, restype) { \ - /* XXX add tp_itemsize later */ \ - OP_RAW_MALLOC(((PyTypeObject *)cpytype)->tp_basicsize, r, restype); \ - if (r) { \ - OP_RAW_MEM_ZERO(r, ((PyTypeObject *)cpytype)->tp_basicsize); \ - PyObject_Init((PyObject *)r, (PyTypeObject *)cpytype); \ - } \ - } +#define OP_CPY_MALLOC(cpytype, r, restype) { \ + /* XXX add tp_itemsize later */ \ + OP_RAW_MALLOC(((PyTypeObject *)cpytype)->tp_basicsize, r, restype); \ + if (r) { \ + OP_RAW_MEMCLEAR(r, ((PyTypeObject *)cpytype)->tp_basicsize, /* */); \ + PyObject_Init((PyObject *)r, (PyTypeObject *)cpytype); \ + } \ + } #define OP_CPY_FREE(x) OP_RAW_FREE(x, /*nothing*/) Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/test/test_newgc.py Fri Aug 25 19:16:50 2006 @@ -859,8 +859,8 @@ p.a = lltype.malloc(A) return p.a.x fn = self.getcompiled(f) + # the point is just not to segfault res = fn() - assert res == 123 class TestUsingStacklessFramework(TestUsingFramework): from pypy.translator.c.gc import StacklessFrameworkGcPolicy as gcpolicy From fijal at codespeak.net Fri Aug 25 20:01:54 2006 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 Aug 2006 20:01:54 +0200 (CEST) Subject: [pypy-svn] r31659 - pypy/dist/pypy/interpreter Message-ID: <20060825180154.4EB2110084@code0.codespeak.net> Author: fijal Date: Fri Aug 25 20:01:52 2006 New Revision: 31659 Modified: pypy/dist/pypy/interpreter/error.py Log: We do not need autpath here. Modified: pypy/dist/pypy/interpreter/error.py ============================================================================== --- pypy/dist/pypy/interpreter/error.py (original) +++ pypy/dist/pypy/interpreter/error.py Fri Aug 25 20:01:52 2006 @@ -1,4 +1,3 @@ -import autopath import os, sys from pypy.rpython.objectmodel import we_are_translated From mwh at codespeak.net Sat Aug 26 11:10:51 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 26 Aug 2006 11:10:51 +0200 (CEST) Subject: [pypy-svn] r31669 - pypy/dist/pypy/translator/stackless Message-ID: <20060826091051.4855910078@code0.codespeak.net> Author: mwh Date: Sat Aug 26 11:10:50 2006 New Revision: 31669 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: turn off saving of stackless statistics (oops) Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Sat Aug 26 11:10:50 2006 @@ -20,7 +20,7 @@ from pypy.translator.stackless.frame import STATE_HEADER, null_state from pypy.translator.stackless.frame import storage_type -SAVE_STATISTICS = True +SAVE_STATISTICS = False # we do this _a lot_: def copyvar(var): From arigo at codespeak.net Sat Aug 26 11:22:47 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 26 Aug 2006 11:22:47 +0200 (CEST) Subject: [pypy-svn] r31670 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20060826092247.A686B10078@code0.codespeak.net> Author: arigo Date: Sat Aug 26 11:22:45 2006 New Revision: 31670 Modified: pypy/dist/pypy/jit/timeshifter/test/test_tl.py pypy/dist/pypy/jit/timeshifter/test/test_tlr.py pypy/dist/pypy/jit/timeshifter/test/test_vlist.py Log: (arre, pedronis, arigo) Forgot to change the style of the other tests in this directory. Modified: pypy/dist/pypy/jit/timeshifter/test/test_tl.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_tl.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_tl.py Sat Aug 26 11:22:45 2006 @@ -12,7 +12,6 @@ import py; py.test.skip("in-progress") code = tl.compile(FACTORIAL_SOURCE) ll_code = string_repr.convert_const(code) - insns, res = self.timeshift(tl.interp_without_call, - [ll_code, 0, 5], [0, 1], - policy=P_OOPSPEC) + res = self.timeshift(tl.interp_without_call, [ll_code, 0, 5], [0, 1], + policy=P_OOPSPEC) assert res == 120 Modified: pypy/dist/pypy/jit/timeshifter/test/test_tlr.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_tlr.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_tlr.py Sat Aug 26 11:22:45 2006 @@ -9,6 +9,6 @@ def test_tlr(self): bytecode = string_repr.convert_const(tlr.SQUARE) - insns, res = self.timeshift(tlr.interpret, [bytecode, 9], [0], - policy=P_OOPSPEC) + res = self.timeshift(tlr.interpret, [bytecode, 9], [0], + policy=P_OOPSPEC) assert res == 81 Modified: pypy/dist/pypy/jit/timeshifter/test/test_vlist.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_vlist.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_vlist.py Sat Aug 26 11:22:45 2006 @@ -13,9 +13,9 @@ lst = [] lst.append(12) return lst[0] - insns, res = self.timeshift(ll_function, [], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [], [], policy=P_OOPSPEC) assert res == 12 - assert insns == {} + self.check_insns({}) def test_enter_block(self): def ll_function(flag): @@ -26,12 +26,12 @@ return lst[0] else: return lst[1] - insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) assert res == 6 - assert insns == {'int_is_true': 1} - insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + self.check_insns({'int_is_true': 1}) + res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) assert res == 131 - assert insns == {'int_is_true': 1} + self.check_insns({'int_is_true': 1}) def test_merge(self): def ll_function(flag): @@ -41,12 +41,12 @@ else: lst.append(131) return lst[-1] - insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) assert res == 6 - assert insns == {'int_is_true': 1} - insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + self.check_insns({'int_is_true': 1}) + res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) assert res == 131 - assert insns == {'int_is_true': 1} + self.check_insns({'int_is_true': 1}) def test_replace(self): def ll_function(flag): @@ -56,12 +56,12 @@ else: lst.append(131) return lst[-1] - insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) assert res == 12 - assert insns == {'int_is_true': 1} - insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + self.check_insns({'int_is_true': 1}) + res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) assert res == 131 - assert insns == {'int_is_true': 1} + self.check_insns({'int_is_true': 1}) def test_force(self): def ll_function(n): @@ -70,7 +70,7 @@ if n: lst.append(12) return lst[-1] - insns, res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [6], [], policy=P_OOPSPEC) assert res == 12 - insns, res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) + res = self.timeshift(ll_function, [0], [], policy=P_OOPSPEC) assert res == 0 From arigo at codespeak.net Sat Aug 26 12:21:07 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 26 Aug 2006 12:21:07 +0200 (CEST) Subject: [pypy-svn] r31672 - in pypy/dist/pypy: annotation jit/codegen/i386 jit/codegen/i386/test jit/codegen/llgraph jit/timeshifter jit/timeshifter/test rpython Message-ID: <20060826102107.7588C10078@code0.codespeak.net> Author: arigo Date: Sat Aug 26 12:21:00 2006 New Revision: 31672 Added: pypy/dist/pypy/jit/codegen/i386/test/test_timeshift.py (contents, props changed) Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/jit/codegen/i386/ri386genop.py pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py pypy/dist/pypy/jit/codegen/llgraph/rgenop.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/timeshift.py pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/callparse.py Log: (pedronis, arre, arigo) Started porting the timeshifting tests to codegen/i386. Lots of small refactorings and fixes, in-progress. Mix-level annotation is "fun". Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Sat Aug 26 12:21:00 2006 @@ -98,23 +98,32 @@ return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - def annotate_helper(self, function, args_s, policy=None, complete_now=True): - args_s = args_s[:] - saved = self.policy, self.added_blocks - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() + def get_call_parameters(self, function, args_s, policy): + desc = self.bookkeeper.getdesc(function) + args = self.bookkeeper.build_args("simple_call", args_s[:]) + result = [] + def schedule(graph, inputcells): + result.append((graph, inputcells)) + return annmodel.s_ImpossibleValue + + prevpolicy = self.policy self.policy = policy + self.bookkeeper.enter(None) try: - self.added_blocks = {} - desc = self.bookkeeper.getdesc(function) - graph = desc.specialize(args_s) - if complete_now: - self.build_graph_types(graph, args_s) - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) + desc.pycall(schedule, args, annmodel.s_ImpossibleValue) finally: - self.policy, self.added_blocks = saved + self.bookkeeper.leave() + self.policy = prevpolicy + [(graph, inputcells)] = result + return graph, inputcells + + def annotate_helper(self, function, args_s, policy=None): + if policy is None: + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() + graph, inputcells = self.get_call_parameters(function, args_s, policy) + self.build_graph_types(graph, inputcells, complete_now=False) + self.complete_helpers(policy) return graph def complete_helpers(self, policy): @@ -123,6 +132,7 @@ try: self.added_blocks = {} self.complete() + # invoke annotation simplifications for the new blocks self.simplify(block_subset=self.added_blocks) finally: self.policy, self.added_blocks = saved Modified: pypy/dist/pypy/jit/codegen/i386/ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/ri386genop.py Sat Aug 26 12:21:00 2006 @@ -236,7 +236,7 @@ T = lltype.typeOf(llvalue) assert T is lltype.Signed return IntConst(llvalue) - genconst._annspecialcase_ = 'specialize:ll' + genconst._annspecialcase_ = 'specialize:genconst(0)' genconst = staticmethod(genconst) def constTYPE(T): @@ -247,6 +247,8 @@ constTYPE._annspecialcase_ = 'specialize:memo' constTYPE = staticmethod(constTYPE) + constPrebuiltGlobal = genconst + def gencallableconst(self, name, block, gv_FUNCTYPE): prologue = self.newblock() #prologue.mc.BREAKPOINT() Modified: pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py (original) +++ pypy/dist/pypy/jit/codegen/i386/test/test_ri386genop.py Sat Aug 26 12:21:00 2006 @@ -1,11 +1,14 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.objectmodel import keepalive_until_here +from pypy.rpython.annlowlevel import MixLevelAnnotatorPolicy from pypy.translator.c.test.test_genc import compile from pypy.jit.codegen.i386.ri386genop import RI386GenOp from ctypes import c_void_p, cast, CFUNCTYPE, c_int +GENOP_POLICY = MixLevelAnnotatorPolicy(None) # XXX clean up + # ____________________________________________________________ FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) @@ -50,7 +53,7 @@ assert res == 42 def test_adder_compile(): - fn = compile(runner, [int, int]) + fn = compile(runner, [int, int], annotatorpolicy=GENOP_POLICY) res = fn(9080983, -9080941) assert res == 42 @@ -110,7 +113,7 @@ assert res == 42 def test_dummy_compile(): - fn = compile(dummy_runner, [int, int]) + fn = compile(dummy_runner, [int, int], annotatorpolicy=GENOP_POLICY) res = fn(40, 37) assert res == 42 @@ -172,7 +175,7 @@ assert res == 17 def test_branching_compile(): - fn = compile(branching_runner, [int, int]) + fn = compile(branching_runner, [int, int], annotatorpolicy=GENOP_POLICY) res = fn(30, 17) assert res == 29 res = fn(3, 17) Added: pypy/dist/pypy/jit/codegen/i386/test/test_timeshift.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/i386/test/test_timeshift.py Sat Aug 26 12:21:00 2006 @@ -0,0 +1,48 @@ +import os +from pypy.annotation import model as annmodel +from pypy.annotation.listdef import s_list_of_strings +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.objectmodel import keepalive_until_here +from pypy.jit.timeshifter.test import test_timeshift +from pypy.translator.c.genc import CStandaloneBuilder + +import py; py.test.skip("in-progress") + + +class TestTimeshiftI386(test_timeshift.TestTimeshift): + from pypy.jit.codegen.i386.ri386genop import RI386GenOp as RGenOp + + def timeshift_test(self, ll_runner, residual_args): + RGenOp = self.RGenOp + FUNC = self.FUNCTYPE + SEPLINE = 'running residual graph...\n' + + def ll_main(argv): + rgenop = RGenOp.get_rgenop_for_testing() + gv_generated = ll_runner(rgenop) + generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + os.write(1, SEPLINE) + res = generated(*residual_args) + os.write(1, str(res) + '\n') + keepalive_until_here(rgenop) # to keep the code blocks alive + return 0 + + annhelper = self.htshift.annhelper + annhelper.getgraph(ll_main, [s_list_of_strings], + annmodel.SomeInteger()) + annhelper.finish() + + t = self.rtyper.annotator.translator + cbuilder = CStandaloneBuilder(t, ll_main) + cbuilder.generate_source() + cbuilder.compile() + output = cbuilder.cmdexec() + assert output.startswith(SEPLINE) + lastline = output[len(SEPLINE):].strip() + return int(lastline) + + def check_insns(self, expected=None, **counts): + "Cannot check instructions in the generated assembler." + + # for the individual tests see + # ====> ../../../timeshifter/test/test_timeshift.py Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Sat Aug 26 12:21:00 2006 @@ -65,7 +65,7 @@ def genconst(llvalue): return LLConst(llimpl.genconst(llvalue)) - genconst._annspecialcase_ = 'specialize:ll' + genconst._annspecialcase_ = 'specialize:genconst(0)' genconst = staticmethod(genconst) def constTYPE(T): Modified: pypy/dist/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/rtimeshift.py Sat Aug 26 12:21:00 2006 @@ -305,7 +305,7 @@ def genconst(self, llvalue): return self.rgenop.genconst(llvalue) - genconst._annspecialcase_ = 'specialize:ll' + genconst._annspecialcase_ = 'specialize:genconst(1)' def genvoidconst(self, dummy): return self.rgenop.placeholder(dummy) Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Sat Aug 26 12:21:00 2006 @@ -57,132 +57,194 @@ del cls._cache del cls._cache_order - def timeshift_cached(self, ll_function, values, inline, policy): + def timeshift_cached(self, ll_function, values, inline=None, policy=None): key = ll_function, inline, policy try: - result, argtypes = self._cache[key] + cache, argtypes = self._cache[key] except KeyError: - if len(self._cache_order) >= 3: - del self._cache[self._cache_order.pop(0)] - hs, ha, rtyper = hannotate(ll_function, values, - inline=inline, policy=policy) - htshift = HintTimeshift(ha, rtyper, self.RGenOp) - RESTYPE = htshift.originalconcretetype( - ha.translator.graphs[0].getreturnvar()) - htshift.timeshift() - t = rtyper.annotator.translator - for graph in ha.translator.graphs: - checkgraph(graph) - t.graphs.append(graph) - if conftest.option.view: - from pypy.translator.tool.graphpage import FlowGraphPage - FlowGraphPage(t, ha.translator.graphs).display() - result = hs, ha, rtyper, htshift, RESTYPE - self._cache[key] = result, getargtypes(rtyper.annotator, values) - self._cache_order.append(key) + pass else: - hs, ha, rtyper, htshift, RESTYPE = result - assert argtypes == getargtypes(rtyper.annotator, values) - return result - - def timeshift(self, ll_function, values, opt_consts=[], inline=None, - policy=None): - hs, ha, rtyper, htshift, RESTYPE = self.timeshift_cached( - ll_function, values, inline, policy) - # build a runner function - original_entrypoint_graph = rtyper.annotator.translator.graphs[0] - graph1 = ha.translator.graphs[0] - timeshifted_entrypoint_fnptr = rtyper.type_system.getcallable(graph1) + self.__dict__.update(cache) + assert argtypes == getargtypes(self.rtyper.annotator, values) + return + + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] + hs, ha, rtyper = hannotate(ll_function, values, + inline=inline, policy=policy) + + # make the timeshifted graphs + htshift = HintTimeshift(ha, rtyper, self.RGenOp) + RESTYPE = htshift.originalconcretetype( + ha.translator.graphs[0].getreturnvar()) + htshift.timeshift() + t = rtyper.annotator.translator + for graph in ha.translator.graphs: + checkgraph(graph) + t.graphs.append(graph) + if conftest.option.view: + from pypy.translator.tool.graphpage import FlowGraphPage + FlowGraphPage(t, ha.translator.graphs).display() + + # make an interface to the timeshifted graphs: + # + # - a green input arg in the timeshifted entry point + # must be provided as a value in 'args' + # + # - a redbox input arg in the timeshifted entry point must + # be provided as two entries in 'args': a boolean flag + # (True=constant, False=variable) and a value + # + graph1 = ha.translator.graphs[0] # the timeshifted entry point assert len(graph1.getargs()) == 2 + len(values) graph1varargs = graph1.getargs()[2:] - - argdescription = [] # list of: ("green"/"redconst"/"redvar", value) - residual_args = [] - residual_argtypes = [] timeshifted_entrypoint_args_s = [] + residual_argtypes = [] + argcolors = [] + generate_code_args_s = [] - for i, (v, llvalue) in enumerate(zip(graph1varargs, values)): + for v, llvalue in zip(graph1varargs, values): + s_var = annmodel.ll_to_annotation(llvalue) r = htshift.hrtyper.bindingrepr(v) residual_v = r.residual_values(llvalue) if len(residual_v) == 0: - # green color = "green" - s_var = annmodel.ll_to_annotation(llvalue) timeshifted_entrypoint_args_s.append(s_var) else: - # red + color = "red" assert residual_v == [llvalue], "XXX for now" - if i in opt_consts: - color = "redconst" - else: - color = "redvar" - ARGTYPE = htshift.originalconcretetype(v) - residual_argtypes.append(ARGTYPE) - residual_args.append(llvalue) + ARGTYPE = htshift.originalconcretetype(v) + residual_argtypes.append(ARGTYPE) timeshifted_entrypoint_args_s.append(htshift.s_RedBox) - argdescription.append((color, llvalue)) + generate_code_args_s.append(annmodel.SomeBool()) + argcolors.append(color) + generate_code_args_s.append(s_var) - argdescription = unrolling_iterable(argdescription) - RGenOp = self.RGenOp + timeshifted_entrypoint_fnptr = rtyper.type_system.getcallable( + graph1) timeshifted_entrypoint = PseudoHighLevelCallable( timeshifted_entrypoint_fnptr, [htshift.s_ResidualGraphBuilder, htshift.s_JITState] + timeshifted_entrypoint_args_s, htshift.s_ResidualGraphBuilder) - FUNCTYPE = lltype.FuncType(residual_argtypes, RESTYPE) - gv_functype = RGenOp.constTYPE(FUNCTYPE) + FUNC = lltype.FuncType(residual_argtypes, RESTYPE) + argcolors = unrolling_iterable(argcolors) + self.argcolors = argcolors - def ll_runner(rgenop): - builder = rtimeshift.make_builder(rgenop) + def ml_generate_code(rgenop, *args): timeshifted_entrypoint_args = () - for color, llvalue in argdescription: + builder = rtimeshift.make_builder(rgenop) + for color in argcolors: if color == "green": + llvalue = args[0] + args = args[1:] timeshifted_entrypoint_args += (llvalue,) else: + is_constant = args[0] + llvalue = args[1] + args = args[2:] TYPE = lltype.typeOf(llvalue) gv_type = rgenop.constTYPE(TYPE) boxcls = rvalue.ll_redboxcls(TYPE) - if color == "redconst": + gv_arg = rtimeshift.ll_geninputarg(builder, gv_type) + if is_constant: + # ignore the gv_arg above, which is still present + # to give the residual graph a uniform signature gv_arg = rgenop.genconst(llvalue) - else: - gv_arg = rtimeshift.ll_geninputarg(builder, gv_type) box = boxcls(gv_type, gv_arg) timeshifted_entrypoint_args += (box,) startblock = rtimeshift.ll_end_setup_builder(builder) endbuilder = timeshifted_entrypoint(builder, None, - *timeshifted_entrypoint_args) + *timeshifted_entrypoint_args) endbuilder.finish_and_return() + gv_functype = rgenop.constTYPE(FUNC) gv_generated = rgenop.gencallableconst("generated", startblock, gv_functype) - return gv_generated + generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + return generated + + ml_generate_code.args_s = ["XXX rgenop"] + generate_code_args_s + ml_generate_code.s_result = annmodel.lltype_to_annotation( + lltype.Ptr(FUNC)) + +## def ml_extract_residual_args(*args): +## result = () +## for color in argcolors: +## if color == "green": +## args = args[1:] +## else: +## is_constant = args[0] +## llvalue = args[1] +## args = args[2:] +## result += (llvalue,) +## return result + +## def ml_call_residual_graph(generated, *allargs): +## residual_args = ml_extract_residual_args(*allargs) +## return generated(*residual_args) + +## ml_call_residual_graph.args_s = ( +## [ml_generate_code.s_result, ...]) +## ml_call_residual_graph.s_result = annmodel.lltype_to_annotation( +## RESTYPE) + self.ml_generate_code = ml_generate_code +## self.ml_call_residual_graph = ml_call_residual_graph self.rtyper = rtyper self.htshift = htshift - self.FUNCTYPE = FUNCTYPE - return self.timeshift_test(ll_runner, residual_args) + self.annotate_interface_functions() - def timeshift_test(self, ll_runner, residual_args): + cache = self.__dict__.copy() + self._cache[key] = cache, getargtypes(rtyper.annotator, values) + self._cache_order.append(key) + + def annotate_interface_functions(self): + annhelper = self.htshift.annhelper RGenOp = self.RGenOp - def ll_main(): + ml_generate_code = self.ml_generate_code +## ml_call_residual_graph = self.ml_call_residual_graph + + def ml_main(*args): rgenop = RGenOp.get_rgenop_for_testing() - return ll_runner(rgenop) + return ml_generate_code(rgenop, *args) + + ml_main.args_s = ml_generate_code.args_s[1:] + ml_main.s_result = ml_generate_code.s_result + + self.maingraph = annhelper.getgraph( + ml_main, + ml_main.args_s, + ml_main.s_result) +## self.callresidualgraph = annhelper.getgraph( +## ml_call_residual_graph, +## ml_call_residual_graph.args_s, +## ml_call_residual_graph.s_result) - annhelper = self.htshift.annhelper - runner_graph = annhelper.getgraph(ll_main, [], - self.htshift.s_ConstOrVar) annhelper.finish() - # run the runner + def timeshift(self, ll_function, values, opt_consts=[], *args, **kwds): + self.timeshift_cached(ll_function, values, *args, **kwds) + + mainargs = [] + residualargs = [] + for i, (color, llvalue) in enumerate(zip(self.argcolors, values)): + if color == "green": + mainargs.append(llvalue) + else: + mainargs.append(i in opt_consts) + mainargs.append(llvalue) + residualargs.append(llvalue) + + # run the graph generator llinterp = LLInterpreter(self.rtyper) - ll_gv_generated = llinterp.eval_graph(runner_graph, []) + ll_generated = llinterp.eval_graph(self.maingraph, mainargs) # now try to run the residual graph generated by the builder - c_generatedfn = RGenOp.reveal(ll_gv_generated) - residual_graph = c_generatedfn.value._obj.graph + residual_graph = ll_generated._obj.graph if conftest.option.view: residual_graph.show() self.insns = summary(residual_graph) - res = llinterp.eval_graph(residual_graph, residual_args) + res = llinterp.eval_graph(residual_graph, residualargs) return res def check_insns(self, expected=None, **counts): Modified: pypy/dist/pypy/jit/timeshifter/timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/timeshift.py Sat Aug 26 12:21:00 2006 @@ -413,12 +413,12 @@ for r, newvar in zip(args_r, newinputargs): if isinstance(r, GreenRepr): r_from = getrepr(r.annotation()) - erased_s = r.erased_annotation() - r_to = getrepr(erased_s) - items_s.append(erased_s) - erased_v = llops.convertvar(newvar, r_from, r_to) + s_erased = r.erased_annotation() + r_to = getrepr(s_erased) + items_s.append(s_erased) + v_erased = llops.convertvar(newvar, r_from, r_to) orig_key_v.append(newvar) - key_v.append(erased_v) + key_v.append(v_erased) s_key_tuple = annmodel.SomeTuple(items_s) r_key = getrepr(s_key_tuple) Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Sat Aug 26 12:21:00 2006 @@ -121,12 +121,19 @@ return super(MixLevelAnnotatorPolicy, pol).default_specialize( funcdesc, args_s) else: - return funcdesc.cachedgraph(None) + return AnnotatorPolicy.default_specialize(funcdesc, args_s) def specialize__arglltype(pol, funcdesc, args_s, i): key = pol.rtyper.getrepr(args_s[i]).lowleveltype alt_name = funcdesc.name+"__for_%sLlT" % key._short_name() - return funcdesc.cachedgraph(key, alt_name=valid_identifier(alt_name)) + return funcdesc.cachedgraph(key, alt_name=valid_identifier(alt_name)) + + def specialize__genconst(pol, funcdesc, args_s, i): + # XXX this is specific to the JIT + TYPE = annmodel.annotation_to_lltype(args_s[i], 'genconst') + args_s[i] = annmodel.lltype_to_annotation(TYPE) + alt_name = funcdesc.name + "__%s" % (TYPE._short_name(),) + return funcdesc.cachedgraph(TYPE, alt_name=valid_identifier(alt_name)) class MixLevelHelperAnnotator: @@ -144,12 +151,12 @@ # get the graph of the mix-level helper ll_function and prepare it for # being annotated. Annotation and RTyping should be done in a single shot # at the end with finish(). - graph = self.rtyper.annotator.annotate_helper(ll_function, args_s, - policy = self.policy, - complete_now = False) + graph, args_s = self.rtyper.annotator.get_call_parameters( + ll_function, args_s, policy = self.policy) for v_arg, s_arg in zip(graph.getargs(), args_s): self.rtyper.annotator.setbinding(v_arg, s_arg) self.rtyper.annotator.setbinding(graph.getreturnvar(), s_result) + #self.rtyper.annotator.annotated[graph.returnblock] = graph self.pending.append((ll_function, graph, args_s, s_result)) return graph Modified: pypy/dist/pypy/rpython/callparse.py ============================================================================== --- pypy/dist/pypy/rpython/callparse.py (original) +++ pypy/dist/pypy/rpython/callparse.py Sat Aug 26 12:21:00 2006 @@ -125,7 +125,7 @@ break else: if 0 < len(holders) == len(holders[0].holder.items()): - return h[0].holder + return holders[0].holder inst = Holder.__new__(cls) inst.holders = tuple(holders) return inst From arigo at codespeak.net Sat Aug 26 12:45:42 2006 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 26 Aug 2006 12:45:42 +0200 (CEST) Subject: [pypy-svn] r31673 - pypy/extradoc/sprintinfo/ireland-2006 Message-ID: <20060826104542.0561F10088@code0.codespeak.net> Author: arigo Date: Sat Aug 26 12:45:41 2006 New Revision: 31673 Modified: pypy/extradoc/sprintinfo/ireland-2006/planning.txt Log: Status meeting. Modified: pypy/extradoc/sprintinfo/ireland-2006/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ireland-2006/planning.txt (original) +++ pypy/extradoc/sprintinfo/ireland-2006/planning.txt Sat Aug 26 12:45:41 2006 @@ -9,32 +9,41 @@ bea arranges a dinner table friday evening * core optimizations - (mwh, arre) + (mwh) - (done) did a lot of profiling, main finding: GC - - (next) optimising GC + - optimising GC: (done) by mostly just collecting + less often :-/ - experiment with optimising builtin lookups - e.g caching method lookups or callback-dict + e.g caching method lookups or callback-dict + - remove the zeroing assumption (more should be + done on this) * ext compiler maybe a topic for the weekend * JIT - (armin, samuele) - - (done) first machine code generation tests are passing - (next) more of the same - - produce machine code in-memory from running the generated CFGs - - maybe think about refining the interface to generating - I386 machine code (also considering a PPC asm backend) + - first machine code generation tests are passing + - (samuele, arre) produce machine code in-memory from running the generated CFGs + - (michael, arigo) implement more of the interface to generate I386 + machine code; maybe think about refining the interface (also + considering a PPC asm backend) * distributed testing - (maciej, hpk) + (maciej, mick) more or less done, but many things : - use "/usr/bin/rsync" to sync to remote - master does collection of items - "leaf" items are asynchronously send to "Executors" (running on nodes) - super-simple reporting + (next) boxing of processes to catch stdout, segfaults, etc. + (next) http server interface to running tests + (later) make more test options work, e.g. --pdb, --view... + + +* (mick, maciej) writing more JS examples + - (arigo) wrote a generic interface that works via greenlets or threads interchangeable + (arigo) wrote a generic interface that works via greenlets or threads interchangeable (svn/user/arigo/hack/pypeers/bluesock.py) From mwh at codespeak.net Sat Aug 26 13:02:18 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 26 Aug 2006 13:02:18 +0200 (CEST) Subject: [pypy-svn] r31675 - pypy/dist/pypy/translator/stackless Message-ID: <20060826110218.4A8E310089@code0.codespeak.net> Author: mwh Date: Sat Aug 26 13:02:17 2006 New Revision: 31675 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: fix stackless build. why on earth did this start failing a few days ago? oh well. Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Sat Aug 26 13:02:17 2006 @@ -260,6 +260,8 @@ return True elif op.opname == 'resume_state_invoke': return True + elif op.opname == 'resume_state_create': + return True return self.stackless_gc and LL_OPERATIONS[op.opname].canunwindgc def analyze_external_call(self, op): From mwh at codespeak.net Sat Aug 26 13:08:51 2006 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 26 Aug 2006 13:08:51 +0200 (CEST) Subject: [pypy-svn] r31676 - in pypy/branch/no-zeroing-assumption/pypy: rpython/memory translator/c/src Message-ID: <20060826110851.B251110089@code0.codespeak.net> Author: mwh Date: Sat Aug 26 13:08:50 2006 New Revision: 31676 Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Log: actually allocate some non zeroed memory. Modified: pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py (original) +++ pypy/branch/no-zeroing-assumption/pypy/rpython/memory/gctransform.py Sat Aug 26 13:08:50 2006 @@ -20,6 +20,7 @@ from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.rpython.rtyper import LowLevelOpList import sets, os def var_ispyobj(var): @@ -1216,28 +1217,33 @@ c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) if not op.opname.endswith('_varsize'): if op.opname.startswith('zero'): - p = self.malloc_fixedsize_clear_ptr + malloc_ptr = self.malloc_fixedsize_clear_ptr else: - p = self.malloc_fixedsize_clear_ptr - args = [p, self.c_const_gc, c_type_id, c_size, c_can_collect] + malloc_ptr = self.malloc_fixedsize_ptr + args = [self.c_const_gc, c_type_id, c_size, c_can_collect] else: v_length = op.args[-1] c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) - if op.opname.startswith('zero'): - p = self.malloc_varsize_clear_ptr - else: - p = self.malloc_varsize_clear_ptr - args = [p, self.c_const_gc, c_type_id, v_length, c_size, + malloc_ptr = self.malloc_varsize_clear_ptr +## if op.opname.startswith('zero'): +## p = self.malloc_varsize_clear_ptr +## else: +## p = self.malloc_varsize_clear_ptr + args = [self.c_const_gc, c_type_id, v_length, c_size, c_varitemsize, c_ofstolength, c_can_collect] c_has_finalizer = rmodel.inputconst( lltype.Bool, bool(self.finalizer_funcptr_for_type(TYPE))) args.append(c_has_finalizer) v = varoftype(llmemory.GCREF) - newop = SpaceOperation("direct_call", args, v) + newop = SpaceOperation("direct_call", [malloc_ptr] + args, v) ops, index = self.protect_roots(newop, livevars, block, block.operations.index(op)) ops.append(SpaceOperation("cast_opaque_ptr", [v], op.result)) + if malloc_ptr == self.malloc_fixedsize_ptr: + llops = LowLevelOpList(None) + gen_zero_gc_pointers(TYPE, op.result, llops) + ops.extend(llops) return ops replace_zero_malloc = replace_malloc @@ -1342,6 +1348,21 @@ offsets.append(0) return offsets +def gen_zero_gc_pointers(TYPE, v, llops): + assert isinstance(TYPE, lltype.Struct) + for name in TYPE._names: + FIELD = getattr(TYPE, name) + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + c_name = Constant(name, lltype.Void) + c_null = Constant(lltype.nullptr(FIELD.TO), FIELD) + llops.genop('bare_setfield', [v, c_name, c_null]) + elif isinstance(FIELD, lltype.Struct): + c_name = Constant(name, lltype.Void) + v1 = llops.genop('getsubstruct', [v, c_name], + resulttype = lltype.Ptr(FIELD)) + gen_zero_gc_pointers(FIELD, v1, llops) + + # ____________________________________________________________ Modified: pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h (original) +++ pypy/branch/no-zeroing-assumption/pypy/translator/c/src/mem.h Sat Aug 26 13:08:50 2006 @@ -23,7 +23,11 @@ #define alloca _alloca #endif +#ifdef USING_BOEHM_GC #define MALLOC_ZERO_FILLED 1 +#else +#define MALLOC_ZERO_FILLED 0 +#endif #define OP_STACK_MALLOC(size,r,restype) \ r = (restype) alloca(size); \ From ac at codespeak.net Sat Aug 26 14:25:09 2006 From: ac at codespeak.net (ac at codespeak.net) Date: Sat, 26 Aug 2006 14:25:09 +0200 (CEST) Subject: [pypy-svn] r31677 - in pypy/dist/pypy/rpython: . test Message-ID: <20060826122509.17A9110089@code0.codespeak.net> Author: ac Date: Sat Aug 26 14:25:09 2006 New Revision: 31677 Modified: pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/rptr.py pypy/dist/pypy/rpython/test/test_rptr.py Log: (arre, pedronis) support ll-ptr(*args) calls, with tests. Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Sat Aug 26 14:25:09 2006 @@ -49,6 +49,42 @@ # to it. return (self.__class__, self.methodname, id(self.s_self)) +def call_args_expand(hop, takes_kwds = True): + hop = hop.copy() + from pypy.interpreter.argument import Arguments + arguments = Arguments.fromshape(None, hop.args_s[1].const, # shape + range(hop.nb_args-2)) + if arguments.w_starstararg is not None: + raise TyperError("**kwds call not implemented") + if arguments.w_stararg is not None: + # expand the *arg in-place -- it must be a tuple + from pypy.rpython.rtuple import AbstractTupleRepr + if arguments.w_stararg != hop.nb_args - 3: + raise TyperError("call pattern too complex") + hop.nb_args -= 1 + v_tuple = hop.args_v.pop() + s_tuple = hop.args_s.pop() + r_tuple = hop.args_r.pop() + if not isinstance(r_tuple, AbstractTupleRepr): + raise TyperError("*arg must be a tuple") + for i in range(len(r_tuple.items_r)): + v_item = r_tuple.getitem_internal(hop.llops, v_tuple, i) + hop.nb_args += 1 + hop.args_v.append(v_item) + hop.args_s.append(s_tuple.items[i]) + hop.args_r.append(r_tuple.items_r[i]) + + kwds = arguments.kwds_w or {} + if not takes_kwds and kwds: + raise TyperError("kwds args not supported") + # prefix keyword arguments with 'i_' + kwds_i = {} + for key, index in kwds.items(): + kwds_i['i_'+key] = index + + return hop, kwds_i + + class BuiltinFunctionRepr(Repr): lowleveltype = lltype.Void @@ -81,34 +117,7 @@ # calling a built-in function with keyword arguments: # mostly for rpython.objectmodel.hint() and for constructing # rctypes structures - from pypy.interpreter.argument import Arguments - arguments = Arguments.fromshape(None, hop.args_s[1].const, # shape - range(hop.nb_args-2)) - if arguments.w_starstararg is not None: - raise TyperError("**kwds call not implemented") - if arguments.w_stararg is not None: - # expand the *arg in-place -- it must be a tuple - from pypy.rpython.rtuple import AbstractTupleRepr - if arguments.w_stararg != hop.nb_args - 3: - raise TyperError("call pattern too complex") - hop.nb_args -= 1 - v_tuple = hop.args_v.pop() - s_tuple = hop.args_s.pop() - r_tuple = hop.args_r.pop() - if not isinstance(r_tuple, AbstractTupleRepr): - raise TyperError("*arg must be a tuple") - for i in range(len(r_tuple.items_r)): - v_item = r_tuple.getitem_internal(hop.llops, v_tuple, i) - hop.nb_args += 1 - hop.args_v.append(v_item) - hop.args_s.append(s_tuple.items[i]) - hop.args_r.append(r_tuple.items_r[i]) - - kwds = arguments.kwds_w or {} - # prefix keyword arguments with 'i_' - kwds_i = {} - for key, index in kwds.items(): - kwds_i['i_'+key] = index + hop, kwds_i = call_args_expand(hop) bltintyper = self.findbltintyper(hop.rtyper) hop2 = hop.copy() Modified: pypy/dist/pypy/rpython/rptr.py ============================================================================== --- pypy/dist/pypy/rpython/rptr.py (original) +++ pypy/dist/pypy/rpython/rptr.py Sat Aug 26 14:25:09 2006 @@ -63,6 +63,11 @@ if not isinstance(self.lowleveltype.TO, FuncType): raise TyperError("calling a non-function %r", self.lowleveltype.TO) vlist = hop.inputargs(*hop.args_r) + nexpected = len(self.lowleveltype.TO.ARGS) + nactual = len(vlist)-1 + if nactual != nexpected: + raise TyperError("argcount mismatch: expected %d got %d" % + (nexpected, nactual)) if isinstance(vlist[0], flowmodel.Constant): if hasattr(vlist[0].value, 'graph'): hop.llops.record_extra_call(vlist[0].value.graph) @@ -74,6 +79,13 @@ return hop.genop(opname, vlist, resulttype = self.lowleveltype.TO.RESULT) + def rtype_call_args(self, hop): + from pypy.rpython.rbuiltin import call_args_expand + hop, _ = call_args_expand(hop, takes_kwds=False) + hop.swap_fst_snd_args() + hop.r_s_popfirstarg() + return self.rtype_simple_call(hop) + class __extend__(pairtype(PtrRepr, IntegerRepr)): Modified: pypy/dist/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rptr.py (original) +++ pypy/dist/pypy/rpython/test/test_rptr.py Sat Aug 26 14:25:09 2006 @@ -1,3 +1,4 @@ +import py from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy from pypy.rpython.lltypesystem.lltype import * @@ -181,3 +182,25 @@ res = interpret(fn, [23]) assert res == 23 + + +def test_call_ptr(): + def f(x,y,z): + return x+y+z + FTYPE = FuncType([Signed, Signed, Signed], Signed) + fptr = functionptr(FTYPE, "f", _callable=f) + + def g(x,y,z): + tot = 0 + tot += fptr(x,y,z) + tot += fptr(*(x,y,z)) + tot += fptr(x, *(x,z)) + return tot + + res = interpret(g, [1,2,4]) + assert res == g(1,2,4) + + def wrong(x,y): + fptr(*(x,y)) + + py.test.raises(TypeError, "interpret(wrong, [1, 2])") From micktwomey at codespeak.net Sat Aug 26 14:36:50 2006 From: micktwomey at codespeak.net (micktwomey at codespeak.net) Date: Sat, 26 Aug 2006 14:36:50 +0200 (CEST) Subject: [pypy-svn] r31678 - in pypy/dist/pypy/translator/js/demo/jsdemo/djangoping: . templates Message-ID: <20060826123650.75A5C10068@code0.codespeak.net> Author: micktwomey Date: Sat Aug 26 14:36:43 2006 New Revision: 31678 Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/__init__.py pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/client.py pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/manage.py (contents, props changed) pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/settings.py pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/templates/ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/templates/index.html pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/urls.py pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/views.py Log: (fijal, micktwomey) Created a simple django javascript ping example. Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/__init__.py ============================================================================== Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/client.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/client.py Sat Aug 26 14:36:43 2006 @@ -0,0 +1,58 @@ +"""rpython javascript code""" + +from pypy.rpython.ootypesystem.bltregistry import BasicExternal, MethodDesc, described + +from pypy.translator.js.modules import mochikit +from pypy.translator.js.modules import _dom as dom + +class PingHandler(BasicExternal): + """Server side code which handles javascript calls""" + _render_xmlhttp = True + + @described(retval={"aa":"aa"}) + def ping(self, ping_str="aa"): + """Simply returns the string prefixed with a PONG""" + return dict(response="PONG: %s" % ping_str) + +ping_handler = PingHandler() + +class Pinger(object): + """Client side javascript ping code which pings the server""" + def initialise(self): + self.log = dom.get_document().getElementById("log") + + def ping(self): + mochikit.logDebug("pinging") + ping_handler.ping("PING", callback) + +pinger = Pinger() + +def callback(data): + mochikit.logDebug("Got response: " + data["response"]) + log = pinger.log + mochikit.logDebug("got log element") + try: + s = "

" + data["response"] + "

" + except KeyError: + mochikit.logDebug("Can't find data") + s = "

" + "Error" + "

" + mochikit.logDebug("Adding: " + s) + log.innerHTML += s + mochikit.logDebug("added message") + +def doping_onclick(event): + mochikit.logDebug("calling pinger") + pinger.ping() + +def ping_init(): + mochikit.createLoggingPane(True) + pinger.initialise() + button = dom.get_document().getElementById("doping") + button.onclick = doping_onclick + mochikit.logDebug("Ping button setup") + +if __name__ == "__main__": + # circular import + from pypy.translator.js.demo.jsdemo.djangoping import client + from pypy.translator.js.main import rpython2javascript + print rpython2javascript(client, ["ping_init"]) Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/manage.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/manage.py Sat Aug 26 14:36:43 2006 @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/settings.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/settings.py Sat Aug 26 14:36:43 2006 @@ -0,0 +1,76 @@ +# Django settings for djangoping project. + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email at domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = '' # 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +DATABASE_NAME = '' # Or path to database file if using sqlite3. +DATABASE_USER = '' # Not used with sqlite3. +DATABASE_PASSWORD = '' # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. All choices can be found here: +# http://www.postgresql.org/docs/current/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes +# http://blogs.law.harvard.edu/tech/stories/storyReader$15 +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. +# Example: "http://media.lawrence.com" +MEDIA_URL = '' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = 'z-n46o5v2gd^n2!tqlher5)w5!l at 0em+d_o+-3qhdulg7qc_lk' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.middleware.doc.XViewMiddleware', +) + +ROOT_URLCONF = 'pypy.translator.js.demo.jsdemo.djangoping.urls' + +import os +# grab a sister package to get its __file__ +from pypy.translator.js.demo.jsdemo.djangoping import client +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates". + # Always use forward slashes, even on Windows. + os.path.join(os.path.dirname(client.__file__), "templates"), +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', +) Added: pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/templates/index.html ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/js/demo/jsdemo/djangoping/templates/index.html Sat Aug 26 14:36:43 2006 @@ -0,0 +1,14 @@ + + + + + Python pinger +