From antocuni at codespeak.net Mon Sep 1 10:37:49 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 10:37:49 +0200 (CEST) Subject: [pypy-svn] r57708 - in pypy/branch/oo-jit/pypy: annotation rpython/ootypesystem/test Message-ID: <20080901083749.C929516842C@codespeak.net> Author: antocuni Date: Mon Sep 1 10:37:46 2008 New Revision: 57708 Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py Log: allow mixing ootype.Class of Instances and Records Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/binaryop.py (original) +++ pypy/branch/oo-jit/pypy/annotation/binaryop.py Mon Sep 1 10:37:46 2008 @@ -951,6 +951,8 @@ common = r2.ootype elif r2.ootype is None: common = r1.ootype + elif isinstance(r1.ootype, ootype.Record) or isinstance(r2.ootype, ootype.Record): + common = ootype.Object else: common = ootype.commonBaseclass(r1.ootype, r2.ootype) assert common is not None, ('Mixing of incompatible classes %r, %r' Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py Mon Sep 1 10:37:46 2008 @@ -363,3 +363,20 @@ res = interpret(fn, [], type_system='ootype') assert not res + +def test_mix_class_record_instance(): + I = Instance("test", ROOT, {"a": Signed}) + R = Record({"x": Signed}) + + c1 = runtimeClass(I) + c2 = runtimeClass(R) + def fn(flag): + if flag: + return c1 + else: + return c2 + + res = interpret(fn, [True], type_system='ootype') + assert res is c1 + res = interpret(fn, [False], type_system='ootype') + assert res is c2 From antocuni at codespeak.net Mon Sep 1 10:47:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 10:47:25 +0200 (CEST) Subject: [pypy-svn] r57709 - in pypy/branch/oo-jit/pypy: annotation jit/codegen/cli translator/cli Message-ID: <20080901084725.456DE169E48@codespeak.net> Author: antocuni Date: Mon Sep 1 10:47:23 2008 New Revision: 57709 Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/translator/cli/constant.py pypy/branch/oo-jit/pypy/translator/cli/database.py Log: use the ootype.Class of the Records to get their CLI type, as we did for instances Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/binaryop.py (original) +++ pypy/branch/oo-jit/pypy/annotation/binaryop.py Mon Sep 1 10:47:23 2008 @@ -951,6 +951,8 @@ common = r2.ootype elif r2.ootype is None: common = r1.ootype + elif r1.ootype is ootype.Object or r2.ootype is ootype.Object: + common = ootype.Object elif isinstance(r1.ootype, ootype.Record) or isinstance(r2.ootype, ootype.Record): common = ootype.Object else: Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Mon Sep 1 10:47:23 2008 @@ -180,10 +180,7 @@ from pypy.jit.codegen.cli.rgenop import class2type self.meth = meth self.gv_obj = gv_obj - if fieldtoken.cls is None: - clitype = gv_obj.getCliType() # XXX: it's a Record, need to think how to fix - else: - clitype = class2type(fieldtoken.cls) + clitype = class2type(fieldtoken.ooclass) self.fieldinfo = clitype.GetField(str(fieldtoken.name)) def restype(self): @@ -202,10 +199,7 @@ self.meth = meth self.gv_obj = gv_obj self.gv_value = gv_value - if fieldtoken.cls is None: - clitype = gv_obj.getCliType() # XXX: it's a Record, need to think how to fix - else: - clitype = class2type(fieldtoken.cls) + clitype = class2type(fieldtoken.ooclass) self.fieldinfo = clitype.GetField(str(fieldtoken.name)) def restype(self): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 1 10:47:23 2008 @@ -35,8 +35,8 @@ self.funcclass = funcclass class FieldToken: - def __init__(self, cls, name): - self.cls = cls + def __init__(self, ooclass, name): + self.ooclass = ooclass self.name = name class AllocToken: @@ -335,11 +335,8 @@ @staticmethod @specialize.memo() def fieldToken(T, name): - if isinstance(T, ootype.Record): - cls = ootype.nullruntimeclass - else: - cls = ootype.runtimeClass(T) - return FieldToken(cls, name) + ooclass = ootype.runtimeClass(T) + return FieldToken(ooclass, name) @staticmethod @specialize.memo() Modified: pypy/branch/oo-jit/pypy/translator/cli/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/constant.py Mon Sep 1 10:47:23 2008 @@ -348,8 +348,8 @@ FUNC = self.value._FUNC classname = self.db.record_delegate(FUNC) else: - INSTANCE = self.value._INSTANCE - classname = self.db.class_name(INSTANCE) + TYPE = self.value._INSTANCE + classname = self.db.class_or_record_name(TYPE) gen.ilasm.opcode('ldtoken', classname) gen.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') return Modified: pypy/branch/oo-jit/pypy/translator/cli/database.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/database.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/database.py Mon Sep 1 10:47:23 2008 @@ -129,6 +129,14 @@ self.classnames.add((namespace, name)) return name + def class_or_record_name(self, TYPE): + if isinstance(TYPE, ootype.Instance): + return self.class_name(TYPE) + elif isinstance(TYPE, ootype.Record): + return self.get_record_name(TYPE) + else: + assert False + def class_name(self, INSTANCE): try: NATIVE_INSTANCE = INSTANCE._hints['NATIVE_INSTANCE'] From fijal at codespeak.net Mon Sep 1 13:07:32 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Sep 2008 13:07:32 +0200 (CEST) Subject: [pypy-svn] r57713 - pypy/dist/pypy/lib Message-ID: <20080901110732.D31E8169E0E@codespeak.net> Author: fijal Date: Mon Sep 1 13:07:30 2008 New Revision: 57713 Modified: pypy/dist/pypy/lib/_hashlib.py Log: Use ctypes_configure instead of hacky "let's hope it's enough" ctypes trickery. Hopefully this makes 64bit segfault go away. Modified: pypy/dist/pypy/lib/_hashlib.py ============================================================================== --- pypy/dist/pypy/lib/_hashlib.py (original) +++ pypy/dist/pypy/lib/_hashlib.py Mon Sep 1 13:07:30 2008 @@ -1,5 +1,6 @@ from ctypes import * import ctypes.util +from ctypes_configure import configure # Note: OpenSSL on OS X only provides md5 and sha1 libpath = ctypes.util.find_library('ssl') @@ -11,34 +12,34 @@ else: return buffer(x)[:] -# FIXME do we really need this anywhere here? -class ENV_MD(Structure): - # XXX Should there be more to this object?. - _fields_ = [ - ('type', c_int), - ('pkey_type', c_int), - ('md_size', c_int), - ] - -class _dummy_env_md(Structure): - # XXX used for OS X, a bit hairy - _fields_ = [ - ('digest', ENV_MD), - ('two', c_int), - ('three', c_int), - ('four', c_int), - ('five', c_int), - ] +class CConfig: + _compilation_info_ = configure.ExternalCompilationInfo( + includes=['openssl/evp.h'], + ) + EVP_MD = configure.Struct('EVP_MD', + []) + EVP_MD_CTX = configure.Struct('EVP_MD_CTX', + [('digest', c_void_p)]) +c = configure.configure(CConfig) +EVP_MD_CTX = c['EVP_MD_CTX'] +EVP_MD = c['EVP_MD'] + +def patch_fields(fields): + res = [] + for k, v in fields: + if k == 'digest': + res.append((k, POINTER(EVP_MD))) + else: + res.append((k, v)) + return res -def _new_ENV_MD(): - return _dummy_env_md() +class EVP_MD_CTX(Structure): + _fields_ = patch_fields(EVP_MD_CTX._fields_) +del c # OpenSSL initialization lib.OpenSSL_add_all_digests() -def _get_digest(ctx): - return ctx.digest - # taken from evp.h, max size is 512 bit, 64 chars lib.EVP_MAX_MD_SIZE = 64 @@ -62,10 +63,10 @@ def __init__(self, obj, name): self.name = name # part of API #print 'obj is ', obj - if isinstance(obj, _dummy_env_md): + if isinstance(obj, EVP_MD_CTX): self._obj = obj.digest else: - self._obj = obj # private + self._obj = obj def __repr__(self): # format is not the same as in C module @@ -73,7 +74,7 @@ def copy(self): "Return a copy of the hash object." - ctxnew = _new_ENV_MD() + ctxnew = EVP_MD_CTX() lib.EVP_MD_CTX_copy(byref(ctxnew), byref(self._obj)) return hash(ctxnew, self.name) @@ -135,10 +136,10 @@ if not digest: raise ValueError("unknown hash function") - ctx = _new_ENV_MD() + ctx = EVP_MD_CTX() lib.EVP_DigestInit(pointer(ctx), digest) - h = hash(_get_digest(ctx), name) + h = hash(ctx.digest, name) if string: h.update(string) return hash(ctx, name) From fijal at codespeak.net Mon Sep 1 13:16:31 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Sep 2008 13:16:31 +0200 (CEST) Subject: [pypy-svn] r57715 - pypy/dist/pypy/lib Message-ID: <20080901111631.55AB8169F04@codespeak.net> Author: fijal Date: Mon Sep 1 13:16:30 2008 New Revision: 57715 Modified: pypy/dist/pypy/lib/_hashlib.py Log: Fix segfault on 64 bit. I think our ctypes are not too smart though and we need to fix it anyway at some point. Modified: pypy/dist/pypy/lib/_hashlib.py ============================================================================== --- pypy/dist/pypy/lib/_hashlib.py (original) +++ pypy/dist/pypy/lib/_hashlib.py Mon Sep 1 13:16:30 2008 @@ -5,6 +5,8 @@ # Note: OpenSSL on OS X only provides md5 and sha1 libpath = ctypes.util.find_library('ssl') lib = CDLL(libpath) # Linux, OS X +lib.EVP_get_digestbyname.restype = c_void_p +lib.EVP_DigestInit.argtypes = [c_void_p, c_void_p] def bufferstr(x): if isinstance(x, basestring): From fijal at codespeak.net Mon Sep 1 14:35:25 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 1 Sep 2008 14:35:25 +0200 (CEST) Subject: [pypy-svn] r57720 - in pypy/branch/cross-compilation/pypy: config rlib translator/c translator/tool translator/tool/test Message-ID: <20080901123525.99ED2169E86@codespeak.net> Author: fijal Date: Mon Sep 1 14:35:23 2008 New Revision: 57720 Modified: pypy/branch/cross-compilation/pypy/config/translationoption.py pypy/branch/cross-compilation/pypy/rlib/pyplatform.py pypy/branch/cross-compilation/pypy/translator/c/genc.py pypy/branch/cross-compilation/pypy/translator/tool/cbuild.py pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py Log: Refactor platform dependencies a bit. Now there is a global in rlib.pyplatform.platform which should contain all platform-dependent configuration details. cbuild.py now uses cc from translation config (needs a test) Modified: pypy/branch/cross-compilation/pypy/config/translationoption.py ============================================================================== --- pypy/branch/cross-compilation/pypy/config/translationoption.py (original) +++ pypy/branch/cross-compilation/pypy/config/translationoption.py Mon Sep 1 14:35:23 2008 @@ -346,7 +346,8 @@ ] def set_platform(config, platform): - from pypy.rlib.pyplatform import Platform, Maemo + from pypy.rlib.pyplatform import Platform, Maemo, OverloadCompilerPlatform + from pypy.rlib import pyplatform from pypy.translator.tool.cbuild import ExternalCompilationInfo if isinstance(platform, str): if platform == 'maemo': @@ -356,9 +357,8 @@ else: raise NotImplementedError('Platform = %s' % (platform,)) assert isinstance(platform, Platform) - # XXX evil hackery - func_defs = list(ExternalCompilationInfo.__init__.func_defaults) - func_defs[-1] = platform - ExternalCompilationInfo.__init__.im_func.func_defaults = tuple(func_defs) + pyplatform.platform = platform + if config.translation.cc: + pyplatform.platform = OverloadCompilerPlatform(platform, + config.translation.cc) - Modified: pypy/branch/cross-compilation/pypy/rlib/pyplatform.py ============================================================================== --- pypy/branch/cross-compilation/pypy/rlib/pyplatform.py (original) +++ pypy/branch/cross-compilation/pypy/rlib/pyplatform.py Mon Sep 1 14:35:23 2008 @@ -25,7 +25,22 @@ class Maemo(Platform): def get_compiler(self): + # XXX how to make this reliable??? return '/scratchbox/compilers/cs2005q3.2-glibc-arm/bin/sbox-arm-linux-gcc' def execute(self, cmd): return py.process.cmdexec('/scratchbox/login ' + cmd) + +class OverloadCompilerPlatform(Platform): + def __init__(self, previous_platform, cc): + self.previous_platform = previous_platform + self.cc = cc + + def get_compiler(self): + return self.cc + + def execute(self, cmd): + return self.previous_platform.execute(cmd) + +platform = Platform() + Modified: pypy/branch/cross-compilation/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/c/genc.py (original) +++ pypy/branch/cross-compilation/pypy/translator/c/genc.py Mon Sep 1 14:35:23 2008 @@ -313,9 +313,6 @@ return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) def getccompiler(self): - # XXX note that overwritten cc here will not affect already - # performed steps in cbuild.py - # completely unsure how to get rid of this inconsistency cc = self.config.translation.cc # Copy extrafiles to target directory, if needed extrafiles = [] Modified: pypy/branch/cross-compilation/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/cross-compilation/pypy/translator/tool/cbuild.py Mon Sep 1 14:35:23 2008 @@ -9,7 +9,6 @@ log = py.log.Producer("cbuild") py.log.setconsumer("cbuild", ansi_log) from pypy.tool.udir import udir -from pypy.rlib.pyplatform import Platform debug = 0 @@ -41,7 +40,7 @@ compile_extra = [], link_extra = [], frameworks = [], - platform = Platform()): + platform = None): """ pre_include_bits: list of pieces of text that should be put at the top of the generated .c files, before any #include. They shouldn't @@ -89,6 +88,9 @@ value = locals()[name] assert isinstance(value, (list, tuple)) setattr(self, name, tuple(value)) + if platform is None: + from pypy.rlib import pyplatform + platform = pyplatform.platform self.platform = platform def from_compiler_flags(cls, flags): Modified: pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py Mon Sep 1 14:35:23 2008 @@ -213,3 +213,4 @@ py.test.raises(py.process.cmdexec.Error, py.process.cmdexec, output) result = eci.platform.execute(output) assert result.startswith('4.0') + From exarkun at codespeak.net Mon Sep 1 14:39:45 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Mon, 1 Sep 2008 14:39:45 +0200 (CEST) Subject: [pypy-svn] r57721 - pypy/build/bot Message-ID: <20080901123945.7482A169EAF@codespeak.net> Author: exarkun Date: Mon Sep 1 14:39:41 2008 New Revision: 57721 Modified: pypy/build/bot/master.cfg pypy/build/bot/pypybuilders.py Log: django and new optimization options Modified: pypy/build/bot/master.cfg ============================================================================== --- pypy/build/bot/master.cfg (original) +++ pypy/build/bot/master.cfg Mon Sep 1 14:39:41 2008 @@ -42,26 +42,26 @@ "slavenames": ["charm"], "builddir": "pypy-c-allworkingmodules-32", "factory": POSIXPyPyBuildFactory( - ["--boxed"], [], ["--allworkingmodules"])}, + ["--boxed"], ["-O0"], [])}, {"name": "pypy-c-allworkingmodules-O3-32", "slavenames": ["charm"], "builddir": "pypy-c-allworkingmodules-faassen-32", "factory": POSIXPyPyBuildFactory( - None, ["--gc=hybrid", "--gcrootfinder=asmgcc", "-O3"], - ["--allworkingmodules"])}, + None, ["-O3", "--gcrootfinder=asmgcc"], + [])}, {"name": "pypy-c-allworkingmodules-O3-64", "slavenames": ["linux-dvs0"], "builddir": "pypy-c-allworkingmodules-faassen-64", "factory": POSIXPyPyBuildFactory( - ["--boxed"], ["-O3"], ["--allworkingmodules"])}, + ["--boxed"], ["-O3"], [], django=True)}, {"name": "pypy-c-allworkingmodules-winxp32", "slavenames": ["winxp32-py2.5"], "builddir": "pypy-c-allworkingmodules-faassen-winxp32", "factory": WindowsPyPyBuildFactory( - ["--boxed"], [], ["--allworkingmodules"])}, + ["--boxed"], ["-O0"], [], django=True)}, ], 'buildbotURL': 'http://office.divmod.com:%d/' % (httpPortNumber,), Modified: pypy/build/bot/pypybuilders.py ============================================================================== --- pypy/build/bot/pypybuilders.py (original) +++ pypy/build/bot/pypybuilders.py Mon Sep 1 14:39:41 2008 @@ -105,6 +105,7 @@ def __init__(self, pytestArguments, translationArguments, targetArguments, *a, **kw): + django = kw.pop('django', False) BuildFactory.__init__(self, *a, **kw) self.addStep( @@ -175,13 +176,35 @@ # tests=["twisted"], # env={"PATH": "."}) + if django: + self.addStep( + SVN, + workdir="build/Django-src", + svnurl="http://code.djangoproject.com/svn/django/trunk/", + mode="copy") + self.addStep( + FileDownload, + mastersrc="django-settings.py", + slavedest="tests/settings.py", + workdir="build/Django-src") + self.addStep( + ShellCommand, + command=[self.sep.join([ + "..", "..", "pypy-src", "pypy", "translator", + "goal", self.executable]), + "runtests.py", "-v", "2", + "--settings=settings"], + workdir="build/Django-src/tests", + env={'PYTHONPATH': '..'}) class POSIXPyPyBuildFactory(PyPyBuildFactory): PyTestStep = POSIXPyTest + sep = '/' class WindowsPyPyBuildFactory(PyPyBuildFactory): PyTestStep = WindowsPyTest executable = "pypy-c.exe" + sep = '\\' From antocuni at codespeak.net Mon Sep 1 15:02:57 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 15:02:57 +0200 (CEST) Subject: [pypy-svn] r57723 - pypy/branch/oo-jit/pypy/jit/codegen/cli/test Message-ID: <20080901130257.65B69169ED9@codespeak.net> Author: antocuni Date: Mon Sep 1 15:02:53 2008 New Revision: 57723 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: skip one test, and unskip a bunch of ones that pass out of the box :-) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 1 15:02:53 2008 @@ -77,11 +77,9 @@ res = self.interpret(ll_function, [], []) assert res.class_name == 'S' - test_arith_plus_minus = skip - test_plus_minus = skip - test_red_virtual_container = skip - test_red_propagate = skip - test_merge_structures = skip + def test_arith_plus_minus(self): + py.test.skip("Cannot work unless we add support for constant arguments in compiled tests") + test_green_with_side_effects = skip test_compile_time_const_tuple = skip test_green_deepfrozen_oosend = skip From antocuni at codespeak.net Mon Sep 1 15:14:33 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 15:14:33 +0200 (CEST) Subject: [pypy-svn] r57724 - in pypy/branch/oo-jit/pypy/jit/codegen/cli: . test Message-ID: <20080901131433.E9109169F5F@codespeak.net> Author: antocuni Date: Mon Sep 1 15:14:32 2008 New Revision: 57724 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: finally remove an "assert False" that has been there for ages before I could find a failing test Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 1 15:14:32 2008 @@ -184,14 +184,9 @@ return dotnet.cast_to_native_object(self.obj) def load(self, meth): - assert False, 'XXX' -## import pdb;pdb.set_trace() -## index = self._get_index(builder) -## if self.obj is None: -## t = typeof(System.Object) -## else: -## t = self.obj.GetType() -## self._load_from_array(builder, index, t) + index = self._get_index(meth) + clitype = self.getCliType() + self._load_from_array(meth, index, clitype) @specialize.arg(1) def revealconst(self, T): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 1 15:14:32 2008 @@ -80,7 +80,6 @@ def test_arith_plus_minus(self): py.test.skip("Cannot work unless we add support for constant arguments in compiled tests") - test_green_with_side_effects = skip test_compile_time_const_tuple = skip test_green_deepfrozen_oosend = skip test_direct_oosend_with_green_self = skip From antocuni at codespeak.net Mon Sep 1 15:21:20 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 15:21:20 +0200 (CEST) Subject: [pypy-svn] r57725 - pypy/branch/oo-jit/pypy/rpython/ootypesystem/test Message-ID: <20080901132120.D8159169E3D@codespeak.net> Author: antocuni Date: Mon Sep 1 15:21:19 2008 New Revision: 57725 Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Log: this test is already present in test_oorecord Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Mon Sep 1 15:21:19 2008 @@ -40,13 +40,6 @@ assert typeOf(i) == I assert typeOf(c) == Class -def test_record_equivalence(): - R1 = Record({"a": Signed}) - R2 = Record({"a": Signed}) - assert R1 == R2 - assert hash(R1) == hash(R2) - assert R1._class is R2._class - def test_runtime_record_instantiation(): R = Record({"a": Signed}) c = runtimeClass(R) From antocuni at codespeak.net Mon Sep 1 15:30:59 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 15:30:59 +0200 (CEST) Subject: [pypy-svn] r57726 - in pypy/branch/oo-jit/pypy: annotation rpython/ootypesystem rpython/ootypesystem/test Message-ID: <20080901133059.19AA5169F66@codespeak.net> Author: antocuni Date: Mon Sep 1 15:30:57 2008 New Revision: 57726 Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Log: make it possible to get the ootype.Class of all BuiltinTypes (including Dict and List), not only Records Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/binaryop.py (original) +++ pypy/branch/oo-jit/pypy/annotation/binaryop.py Mon Sep 1 15:30:57 2008 @@ -953,7 +953,7 @@ common = r1.ootype elif r1.ootype is ootype.Object or r2.ootype is ootype.Object: common = ootype.Object - elif isinstance(r1.ootype, ootype.Record) or isinstance(r2.ootype, ootype.Record): + elif isinstance(r1.ootype, ootype.BuiltinType) or isinstance(r2.ootype, ootype.BuiltinType): common = ootype.Object else: common = ootype.commonBaseclass(r1.ootype, r2.ootype) Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Mon Sep 1 15:30:57 2008 @@ -294,6 +294,17 @@ class BuiltinType(SpecializableType): + _classes = {} + + @property + def _class(self): + try: + return self._classes[self] + except KeyError: + cls = _class(self) + self._classes[self] = cls + return cls + def _example(self): return new(self) @@ -308,8 +319,6 @@ # We try to keep Record as similar to Instance as possible, so backends # can treat them polymorphically, if they choose to do so. - _classes = {} - def __init__(self, fields, _hints={}): self._fields = frozendict() for name, ITEMTYPE in fields.items(): @@ -317,15 +326,6 @@ self._null = _null_record(self) self._hints = frozendict(_hints) - @property - def _class(self): - try: - return self._classes[self] - except KeyError: - cls = _class(self) - self._classes[self] = cls - return cls - def _defl(self): return self._null @@ -1827,7 +1827,7 @@ INSTANCE._override_default_for_fields(fields) def runtimeClass(INSTANCE): - assert isinstance(INSTANCE, (Instance, Record)) + assert isinstance(INSTANCE, (Instance, BuiltinType)) return INSTANCE._class def isSubclass(C1, C2): Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py Mon Sep 1 15:30:57 2008 @@ -367,16 +367,22 @@ def test_mix_class_record_instance(): I = Instance("test", ROOT, {"a": Signed}) R = Record({"x": Signed}) + L = List(Signed) c1 = runtimeClass(I) c2 = runtimeClass(R) + c3 = runtimeClass(L) def fn(flag): - if flag: + if flag == 0: return c1 - else: + elif flag == 1: return c2 + else: + return c3 - res = interpret(fn, [True], type_system='ootype') + res = interpret(fn, [0], type_system='ootype') assert res is c1 - res = interpret(fn, [False], type_system='ootype') + res = interpret(fn, [1], type_system='ootype') assert res is c2 + res = interpret(fn, [2], type_system='ootype') + assert res is c3 Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Mon Sep 1 15:30:57 2008 @@ -53,6 +53,12 @@ R2 = Record({"a": Signed}) assert R1 == R2 assert runtimeClass(R1) is runtimeClass(R2) + +def test_class_builtintype(): + L1 = List(Signed) + L2 = List(Signed) + assert L1 == L2 + assert runtimeClass(L1) is runtimeClass(L2) def test_classof(): I = Instance("test", ROOT, {"a": Signed}) From tverwaes at codespeak.net Mon Sep 1 16:50:16 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 1 Sep 2008 16:50:16 +0200 (CEST) Subject: [pypy-svn] r57730 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080901145016.53384169E88@codespeak.net> Author: tverwaes Date: Mon Sep 1 16:50:12 2008 New Revision: 57730 Modified: pypy/dist/pypy/lang/gameboy/cartridge.py pypy/dist/pypy/lang/gameboy/interrupt.py pypy/dist/pypy/lang/gameboy/test/test_cpu.py pypy/dist/pypy/lang/gameboy/test/test_interrupt.py pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/video.py Log: small refactorings + removed some print statements Modified: pypy/dist/pypy/lang/gameboy/cartridge.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/cartridge.py (original) +++ pypy/dist/pypy/lang/gameboy/cartridge.py Mon Sep 1 16:50:12 2008 @@ -545,7 +545,7 @@ raise InvalidMemoryAccessException("MBC3.read_clock_data invalid address %i") def write(self, address, data): - print hex(address), hex(data) + #print hex(address), hex(data) # 0000-1FFF if address <= 0x1FFF: self.write_ram_enable(address, data) Modified: pypy/dist/pypy/lang/gameboy/interrupt.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/interrupt.py (original) +++ pypy/dist/pypy/lang/gameboy/interrupt.py Mon Sep 1 16:50:12 2008 @@ -6,8 +6,8 @@ """ An Interrupt Flag handles a single interrupt channel """ - def __init__(self, _reset, mask, call_code): - self._reset = _reset + def __init__(self, reset, mask, call_code): + self._reset = reset self.mask = mask self.call_code = call_code self.reset() @@ -19,8 +19,8 @@ def is_pending(self): return self._is_pending - def set_pending(self, _is_pending=True): - self._is_pending = _is_pending + def set_pending(self, is_pending=True): + self._is_pending = is_pending def is_enabled(self): return self.enabled @@ -65,14 +65,14 @@ self.reset() def create_interrupt_flags(self): - self.vblank = InterruptFlag(True, constants.VBLANK, 0x40) - self.lcd = InterruptFlag(False, constants.LCD, 0x48) - self.timer = InterruptFlag(False, constants.TIMER, 0x50) - self.serial = InterruptFlag(False, constants.SERIAL, 0x58) - self.joypad = InterruptFlag(False, constants.JOYPAD, 0x60) + self.v_blank = InterruptFlag(True, constants.VBLANK, 0x40) + self.lcd = InterruptFlag(False, constants.LCD, 0x48) + self.timer = InterruptFlag(False, constants.TIMER, 0x50) + self.serial = InterruptFlag(False, constants.SERIAL, 0x58) + self.joypad = InterruptFlag(False, constants.JOYPAD, 0x60) def create_flag_list(self): - self.interrupt_flags = [ self.vblank, self.lcd, self.timer, self.serial, + self.interrupt_flags = [ self.v_blank, self.lcd, self.timer, self.serial, self.joypad] def create_flag_mask_mapping(self): @@ -104,7 +104,7 @@ return (self.get_enable_mask() & self.get_interrupt_flag() & mask) != 0 def raise_interrupt(self, mask): - self.mask_mapping[mask].set_pending(True) + self.mask_mapping[mask].set_pending() def lower(self, mask): self.mask_mapping[mask].set_pending(False) Modified: pypy/dist/pypy/lang/gameboy/test/test_cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_cpu.py Mon Sep 1 16:50:12 2008 @@ -1141,7 +1141,7 @@ cpu.halted = True cpu.cycles = 4 cpu.interrupt.set_enable_mask(0xFF) - cpu.interrupt.vblank.set_pending() + cpu.interrupt.v_blank.set_pending() assert cpu.interrupt.is_pending() == True assert cpu.halted == True cpu.handle_pending_interrupts() @@ -1156,14 +1156,14 @@ cpu.sp.set(0x02) sp = cpu.sp.get() cpu.interrupt.set_enable_mask(0xFF) - cpu.interrupt.vblank.set_pending() + cpu.interrupt.v_blank.set_pending() cpu.interrupt.lcd.set_pending() assert cpu.interrupt.is_pending() == True cpu.cycles = 0 cpu.handle_pending_interrupts() assert cpu.cycles == 0 assert cpu.halted == False - assert_default_registers(cpu, pc=cpu.interrupt.vblank.call_code, sp=sp-2) + assert_default_registers(cpu, pc=cpu.interrupt.v_blank.call_code, sp=sp-2) assert cpu.pop() == 0x34 assert cpu.pop() == 0x12 @@ -1245,7 +1245,7 @@ cpu.ime = False cpu.halted = False prepare_for_fetch(cpu, 0x00) # nop 1 cycle - print cpu.interrupt.get_enable_mask() + #print cpu.interrupt.get_enable_mask() assert cpu.interrupt.is_pending() == False cycle_test(cpu, 0xFB, 1+1) assert cpu.interrupt.is_pending() == False @@ -1257,7 +1257,7 @@ cpu.ime = True cpu.halted = False prepare_for_fetch(cpu, 0x00) # nop 1 cycle - cpu.interrupt.vblank.set_pending() + cpu.interrupt.v_blank.set_pending() cpu.interrupt.serial.set_pending() cpu.interrupt.set_enable_mask(0x1F) assert cpu.interrupt.is_pending() == True @@ -1265,14 +1265,14 @@ assert cpu.ime == True cycle_test(cpu, 0xFB, 1+1) assert cpu.interrupt.is_pending() == True - assert cpu.interrupt.vblank.is_pending() == False + assert cpu.interrupt.v_blank.is_pending() == False assert cpu.interrupt.serial.is_pending() == True - assert cpu.pc.get() == cpu.interrupt.vblank.call_code + assert cpu.pc.get() == cpu.interrupt.v_blank.call_code assert cpu.ime == False cpu.ime = True cycle_test(cpu, 0xFB, 1+1) - assert cpu.interrupt.vblank.is_pending() == False + assert cpu.interrupt.v_blank.is_pending() == False assert cpu.interrupt.serial.is_pending() == False assert cpu.interrupt.is_pending() == False @@ -1551,8 +1551,8 @@ if register == cpu.hli: cycles = 4 cpu.reset() - if registerOpCode ==0xFF: - print "6544444444444444" + #if registerOpCode ==0xFF: + # print "6544444444444444" register.set(0) fetch_execute_cycle_test_second_order(cpu, registerOpCode, cycles) @@ -1580,8 +1580,8 @@ assert (register.get() & (1< Author: tverwaes Date: Mon Sep 1 17:11:35 2008 New Revision: 57732 Modified: pypy/dist/pypy/lang/gameboy/interrupt.py pypy/dist/pypy/lang/gameboy/joypad.py pypy/dist/pypy/lang/gameboy/serial.py pypy/dist/pypy/lang/gameboy/test/test_interrupt.py pypy/dist/pypy/lang/gameboy/test/test_joypad.py pypy/dist/pypy/lang/gameboy/test/test_serial.py pypy/dist/pypy/lang/gameboy/test/test_timer.py pypy/dist/pypy/lang/gameboy/timer.py Log: refactored constants.DEVICE to use direct links to InterruptFlag objects Modified: pypy/dist/pypy/lang/gameboy/interrupt.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/interrupt.py (original) +++ pypy/dist/pypy/lang/gameboy/interrupt.py Mon Sep 1 17:11:35 2008 @@ -61,7 +61,6 @@ def __init__(self): self.create_interrupt_flags() self.create_flag_list() - self.create_flag_mask_mapping() self.reset() def create_interrupt_flags(self): @@ -72,14 +71,12 @@ self.joypad = InterruptFlag(False, constants.JOYPAD, 0x60) def create_flag_list(self): - self.interrupt_flags = [ self.v_blank, self.lcd, self.timer, self.serial, + self.interrupt_flags = [ self.v_blank, + self.lcd, + self.timer, + self.serial, self.joypad] - def create_flag_mask_mapping(self): - self.mask_mapping = {} - for flag in self.interrupt_flags: - self.mask_mapping[flag.mask] = flag - def reset(self): self.set_enable_mask(0x0) for flag in self.interrupt_flags: @@ -103,12 +100,6 @@ def is_pending(self, mask=0xFF): return (self.get_enable_mask() & self.get_interrupt_flag() & mask) != 0 - def raise_interrupt(self, mask): - self.mask_mapping[mask].set_pending() - - def lower(self, mask): - self.mask_mapping[mask].set_pending(False) - def get_enable_mask(self): enabled = 0x00 for interrupt_flag in self.interrupt_flags: @@ -118,7 +109,7 @@ def set_enable_mask(self, enable_mask): for flag in self.interrupt_flags: - flag.set_enabled((enable_mask & flag.mask) != 0) + flag.set_enabled(bool(enable_mask & flag.mask)) self.enable_rest_data = enable_mask & 0xE0; Modified: pypy/dist/pypy/lang/gameboy/joypad.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/joypad.py (original) +++ pypy/dist/pypy/lang/gameboy/joypad.py Mon Sep 1 17:11:35 2008 @@ -28,7 +28,7 @@ assert isinstance(joypad_driver, JoypadDriver) assert isinstance(interrupt, Interrupt) self.driver = joypad_driver - self.interrupt = interrupt + self.joypad_interrupt_flag = interrupt.joypad self.reset() def reset(self): @@ -65,7 +65,7 @@ elif self.read_control & 0x3 == 3: self.button_code = 0xF if old_buttons != self.button_code: - self.interrupt.raise_interrupt(constants.JOYPAD) + self.joypad_interrupt_flag.set_pending() # ------------------------------------------------------------------------------ @@ -213,4 +213,4 @@ self.pressed = True def is_pressed(self): - return self.pressed \ No newline at end of file + return self.pressed Modified: pypy/dist/pypy/lang/gameboy/serial.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/serial.py (original) +++ pypy/dist/pypy/lang/gameboy/serial.py Mon Sep 1 17:11:35 2008 @@ -11,7 +11,7 @@ def __init__(self, interrupt): assert isinstance(interrupt, Interrupt) - self.interrupt = interrupt + self.serial_interrupt_flag = interrupt.serial self.reset() def reset(self): @@ -30,7 +30,7 @@ self.serial_data = 0xFF self.serial_control &= 0x7F self.cycles = constants.SERIAL_IDLE_CLOCK - self.interrupt.raise_interrupt(constants.SERIAL) + self.serial_interrupt_flag.set_pending() def write(self, address, data): if address == constants.SERIAL_TRANSFER_DATA: @@ -58,4 +58,4 @@ return self.serial_data def get_serial_control(self): - return self.serial_control \ No newline at end of file + return self.serial_control Modified: pypy/dist/pypy/lang/gameboy/test/test_interrupt.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_interrupt.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_interrupt.py Mon Sep 1 17:11:35 2008 @@ -54,19 +54,6 @@ flag.set_pending(True) assert interrupt.is_pending(flag.mask) -def test_raise_lower_interrupt(): - interrupt = get_interrupt() - masks= [constants.LCD, constants.TIMER, - constants.JOYPAD, constants.SERIAL] - interrupt.set_enable_mask(0xFF) - interrupt.v_blank.set_pending(True) - for mask in masks: - interrupt.raise_interrupt(mask) - assert interrupt.mask_mapping[mask].is_pending() == True - assert interrupt.is_pending(mask) == True - interrupt.lower(mask) - assert interrupt.is_pending(mask) == False - def test_read_write(): interrupt = get_interrupt() interrupt.write(constants.IE, 0x12) Modified: pypy/dist/pypy/lang/gameboy/test/test_joypad.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_joypad.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_joypad.py Mon Sep 1 17:11:35 2008 @@ -263,7 +263,7 @@ assert joypad.cycles == constants.JOYPAD_CLOCK assert joypad.read_control == 2 assert joypad.button_code == 0xF - assert joypad.interrupt.joypad.is_pending() == False + assert not joypad.joypad_interrupt_flag.is_pending() joypad.driver.raised = True joypad.cycles = 2 @@ -272,7 +272,7 @@ assert joypad.cycles == constants.JOYPAD_CLOCK assert joypad.read_control == 2 assert joypad.button_code == constants.BUTTON_UP - assert joypad.interrupt.joypad.is_pending() == True + assert joypad.joypad_interrupt_flag.is_pending() def test_read_write(): joypad = get_joypad() @@ -308,13 +308,13 @@ joypad.write(constants.JOYP, 0x10) joypad.update() assert joypad.button_code == constants.BUTTON_SELECT - assert joypad.interrupt.joypad.is_pending() + assert joypad.joypad_interrupt_flag.is_pending() - joypad.interrupt.joypad.set_pending(False) + joypad.joypad_interrupt_flag.set_pending(False) joypad.write(constants.JOYP, 0x10) joypad.update() assert joypad.button_code == constants.BUTTON_SELECT - assert joypad.interrupt.joypad.is_pending() == False + assert not joypad.joypad_interrupt_flag.is_pending() joypad.write(constants.JOYP, 0x20) joypad.update() @@ -325,4 +325,4 @@ assert joypad.button_code == 0xF - \ No newline at end of file + Modified: pypy/dist/pypy/lang/gameboy/test/test_serial.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_serial.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_serial.py Mon Sep 1 17:11:35 2008 @@ -44,7 +44,7 @@ assert cycles-serial.cycles == 2 assert serial.serial_data == 0 assert serial.serial_control == 0x81 - assert serial.interrupt.serial.is_pending() == False + assert not serial.serial_interrupt_flag.is_pending() serial.reset() serial.serial_control = 0x81 @@ -53,7 +53,7 @@ assert serial.serial_data == 0xFF assert serial.serial_control == 0x81 & 0x7F assert serial.cycles == constants.SERIAL_IDLE_CLOCK - assert serial.interrupt.serial.is_pending() == True + assert serial.serial_interrupt_flag.is_pending() def test_read_write(): @@ -69,4 +69,4 @@ assert serial.serial_control == value assert serial.read(0) == 0xFF - \ No newline at end of file + Modified: pypy/dist/pypy/lang/gameboy/test/test_timer.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_timer.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_timer.py Mon Sep 1 17:11:35 2008 @@ -216,12 +216,11 @@ timer.timer_control = 0x04 timer.timer_counter = -1 # raise an interupt as we pass 0 - assert timer.interrupt.is_pending(constants.TIMER) == False - assert timer.interrupt.timer.is_pending() == False + assert not timer.timer_interrupt_flag.is_pending() timer.timer_cycles = -timer.timer_clock+1 timer.emulate_timer(ticks) assert timer.timer_cycles == 1 assert timer.timer_counter == timer.timer_modulo - assert timer.interrupt.timer.is_pending() + assert timer.timer_interrupt_flag.is_pending() + - \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/timer.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/timer.py (original) +++ pypy/dist/pypy/lang/gameboy/timer.py Mon Sep 1 17:11:35 2008 @@ -23,7 +23,7 @@ def __init__(self, interrupt): assert isinstance(interrupt, Interrupt) - self.interrupt = interrupt + self.timer_interrupt_flag = interrupt.timer self.reset() def reset(self): @@ -145,7 +145,7 @@ """ if self.timer_counter == 0x00: self.timer_counter = self.timer_modulo - self.interrupt.raise_interrupt(constants.TIMER) + self.timer_interrupt_flag.set_pending() #def emulate_timer(self, ticks): # if (self.timer_control & 0x04) == 0: From tverwaes at codespeak.net Mon Sep 1 18:52:00 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 1 Sep 2008 18:52:00 +0200 (CEST) Subject: [pypy-svn] r57735 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080901165200.CA972169F73@codespeak.net> Author: tverwaes Date: Mon Sep 1 18:51:57 2008 New Revision: 57735 Modified: pypy/dist/pypy/lang/gameboy/gameboy_implementation.py pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/video.py Log: refactoring -> moving to mode objects rather than ids and manual switching Modified: pypy/dist/pypy/lang/gameboy/gameboy_implementation.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/gameboy_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/gameboy_implementation.py Mon Sep 1 18:51:57 2008 @@ -38,7 +38,8 @@ try: while self.is_running and self.handle_events(): self.emulate(constants.GAMEBOY_CLOCK >> 2) - time.sleep(10/1000) + RSDL.Delay(1) + #time.sleep(10/1000) except : self.is_running = False lltype.free(self.event, flavor='raw') Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Mon Sep 1 18:51:57 2008 @@ -51,7 +51,7 @@ # StatusRegister --------------------------------------------------------------- def test_video_status_reset(): - status = StatusRegister() + status = StatusRegister(None) assert status.read(extend=True) == 0x02 + 0x80 status.write(0x00, write_all=True) @@ -65,11 +65,11 @@ assert status.read(extend=True) == 0x02 + 0x80 def test_video_status_mode(): - status = StatusRegister() + status = StatusRegister(None) assert status.get_mode() == 2 for i in range(3): status.set_mode(i) assert status.get_mode() == i status.set_mode(4) - assert status.get_mode() == 0 \ No newline at end of file + assert status.get_mode() == 0 Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Mon Sep 1 18:51:57 2008 @@ -82,6 +82,36 @@ # ----------------------------------------------------------------------------- +class Mode(object): + def id(self): + raise Exception() + def __init__(self, video): + self.video = video + +class Mode0(Mode): + def id(self): + return 0 + def emulate(self): + self.video.emulate_hblank() + +class Mode1(Mode): + def id(self): + return 1 + def emulate(self): + self.video.emulate_v_blank() + +class Mode2(Mode): + def id(self): + return 2 + def emulate(self): + self.video.emulate_oam() + +class Mode3(Mode): + def id(self): + return 3 + def emulate(self): + self.video.emulate_transfer() + class StatusRegister(object): """ Bit 6 - LYC=LY Coincidence Interrupt (1=Enable) (Read/Write) @@ -95,11 +125,15 @@ 2: During Searching OAM-RAM 3: During Transfering Data to LCD Driver """ - def __init__(self): + def __init__(self, video): + self.modes = [Mode0(video), + Mode1(video), + Mode2(video), + Mode3(video)] self.reset() def reset(self): - self._mode = 0x02 + self.set_mode(0x02) self.line_y_compare_flag = False self.mode_0_h_blank_interrupt = False self.mode_1_v_blank_interrupt = False @@ -109,7 +143,7 @@ def read(self, extend=False): - value = self._mode + value = self.get_mode() value += self.line_y_compare_flag << 2 value += self.mode_0_h_blank_interrupt << 3 value += self.mode_1_v_blank_interrupt << 4 @@ -123,7 +157,7 @@ def write(self, value, write_all=False, keep_mode_0_h_blank_interrupt=False): if write_all: - self._mode = value & 0x03 + self.set_mode(value & 0x03) self.line_y_compare_flag = bool(value & (1 << 2)) self.status = bool(value & (1 << 7)) self.mode_0_h_blank_interrupt = bool(value & (1 << 3)) @@ -132,14 +166,14 @@ self.line_y_compare_interrupt = bool(value & (1 << 6)) def get_mode(self): - return self._mode + return self.current_mode.id() def set_mode(self, mode): - self._mode = mode & 0x03 + self.current_mode = self.modes[mode & 0x03] def line_y_compare_check(self): return not self.line_y_compare_flag or not self.line_y_compare_interrupt - + # ----------------------------------------------------------------------------- class Sprite(object): @@ -199,7 +233,7 @@ self.v_blank_interrupt_flag = interrupt.v_blank self.lcd_interrupt_flag = interrupt.lcd self.control = ControlRegister() - self.status = StatusRegister() + self.status = StatusRegister(self) self.memory = memory self.reset() @@ -257,9 +291,9 @@ self.set_scroll_y(data) elif address == constants.SCX: self.set_scroll_x(data) - elif address == constants.LY: - # Read Only: line_y - pass + #elif address == constants.LY: + # Read Only: line_y + # pass elif address == constants.LYC: self.set_line_y_compare(data) elif address == constants.DMA: @@ -274,18 +308,9 @@ self.set_window_y(data) elif address == constants.WX: self.set_window_x(data) - else: - self.write_oam(address, data) - - def write_oam(self, address, data): - if address >= constants.OAM_ADDR and \ - address < (constants.OAM_ADDR + constants.OAM_SIZE): - self.oam[address - constants.OAM_ADDR] = data & 0xFF - elif address >= constants.VRAM_ADDR and \ - address < (constants.VRAM_ADDR + constants.VRAM_SIZE): - if (address - constants.VRAM_ADDR) == 0x1910 or \ - (address - constants.VRAM_ADDR) == 0x1911: - pass + elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: + self.oam[address - constants.OAM_ADDR] = data & 0xFF + elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: self.vram[address - constants.VRAM_ADDR] = data & 0xFF def read(self, address): @@ -313,15 +338,9 @@ return self.get_window_y() elif address == constants.WX: return self.get_window_x() - else: - return self.read_oam(address) - - def read_oam(self, address): - if (address >= constants.OAM_ADDR and - address < (constants.OAM_ADDR + constants.OAM_SIZE)): + elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: return self.oam[address - constants.OAM_ADDR] - elif (address >= constants.VRAM_ADDR and - address < (constants.VRAM_ADDR + constants.VRAM_SIZE)): + elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: return self.vram[address - constants.VRAM_ADDR] return 0xFF @@ -343,7 +362,7 @@ self.control.write(data) def reset_control(self, data): - # NOTE: do not reset constants.LY=LYC flag (bit 2) of the STAT register (Mr. Do!) + # NOTE: do not reset LY=LYC flag (bit 2) of the STAT register (Mr. Do!) self.line_y = 0 if (data & 0x80) != 0: self.status.set_mode(0x02) @@ -364,7 +383,7 @@ def set_status_bug(self) : # Gameboy Bug if self.control.lcd_enabled and \ - self.status.get_mode() == 0x01 and \ + self.status.get_mode() == 1 and \ self.status.line_y_compare_check(): self.lcd_interrupt_flag.set_pending() @@ -597,23 +616,14 @@ # emulation ---------------------------------------------------------------- def emulate(self, ticks): - ticks = int(ticks) if self.control.lcd_enabled: - self.cycles -= ticks - self.consume_cycles() - - def consume_cycles(self): - while self.cycles <= 0: - mode = self.status.get_mode() - if mode == 0: - self.emulate_hblank() - elif mode == 1: - self.emulate_v_blank() - elif mode == 2: - self.emulate_oam() - else: - self.emulate_transfer() - + self.cycles -= int(ticks) + while self.cycles <= 0: + self.current_mode().emulate() + + def current_mode(self): + return self.status.current_mode + def emulate_oam(self): self.set_mode_3_begin() From antocuni at codespeak.net Mon Sep 1 20:31:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 1 Sep 2008 20:31:26 +0200 (CEST) Subject: [pypy-svn] r57736 - in pypy/branch/oo-jit/pypy/rpython/ootypesystem: . test Message-ID: <20080901183126.10CDD169F7B@codespeak.net> Author: antocuni Date: Mon Sep 1 20:31:24 2008 New Revision: 57736 Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Log: attach a Class to *all* OOTypes, including Class itself, and all StaticMethods. Not sure if this is the right way to do or just a temporary hack, probably we need to think more about how the various OOTypes are related each other. Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Mon Sep 1 20:31:24 2008 @@ -15,6 +15,18 @@ oopspec_name = None + _classes = {} + + @property + def _class(self): + try: + return self._classes[self] + except KeyError: + cls = _class(self) + self._classes[self] = cls + return cls + + def _is_compatible(TYPE1, TYPE2): if TYPE1 == TYPE2: return True @@ -87,7 +99,7 @@ self._add_methods(methods) self._null = make_null_instance(self) - self._class = _class(self) + self.__dict__['_class'] = _class(self) def __eq__(self, other): return self is other @@ -294,17 +306,6 @@ class BuiltinType(SpecializableType): - _classes = {} - - @property - def _class(self): - try: - return self._classes[self] - except KeyError: - cls = _class(self) - self._classes[self] = cls - return cls - def _example(self): return new(self) @@ -1826,9 +1827,9 @@ def overrideDefaultForFields(INSTANCE, fields): INSTANCE._override_default_for_fields(fields) -def runtimeClass(INSTANCE): - assert isinstance(INSTANCE, (Instance, BuiltinType)) - return INSTANCE._class +def runtimeClass(TYPE): + assert isinstance(TYPE, OOType) + return TYPE._class def isSubclass(C1, C2): c = C1 Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_oortype.py Mon Sep 1 20:31:24 2008 @@ -372,13 +372,16 @@ c1 = runtimeClass(I) c2 = runtimeClass(R) c3 = runtimeClass(L) + c4 = runtimeClass(Class) def fn(flag): if flag == 0: return c1 elif flag == 1: return c2 - else: + elif flag == 2: return c3 + else: + return c4 res = interpret(fn, [0], type_system='ootype') assert res is c1 @@ -386,3 +389,5 @@ assert res is c2 res = interpret(fn, [2], type_system='ootype') assert res is c3 + res = interpret(fn, [3], type_system='ootype') + assert res is c4 Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/test/test_ootype.py Mon Sep 1 20:31:24 2008 @@ -59,7 +59,16 @@ L2 = List(Signed) assert L1 == L2 assert runtimeClass(L1) is runtimeClass(L2) - + +def test_class_class(): + L = List(Signed) + R = Record({"a": Signed}) + c1 = runtimeClass(L) + c2 = runtimeClass(R) + C1 = typeOf(c1) + C2 = typeOf(c2) + assert runtimeClass(C1) is runtimeClass(C2) + def test_classof(): I = Instance("test", ROOT, {"a": Signed}) c = runtimeClass(I) From pedronis at codespeak.net Mon Sep 1 22:30:44 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 1 Sep 2008 22:30:44 +0200 (CEST) Subject: [pypy-svn] r57737 - pypy/branch/garden-call-code-2/pypy/interpreter/test Message-ID: <20080901203044.E815A169EE0@codespeak.net> Author: pedronis Date: Mon Sep 1 22:30:41 2008 New Revision: 57737 Modified: pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py Log: repair tests, some new tests Modified: pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py Mon Sep 1 22:30:41 2008 @@ -457,7 +457,36 @@ w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str) assert space.is_w(w_meth5, w_meth3) -class TestShortcuts(object): +class TestShortcuts(object): + + def test_call_function(self): + space = self.space + + d = {} + for i in range(10): + args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" + exec """ +def f%s: + return %s +""" % (args, args) in d + f = d['f'] + res = f(*range(i)) + code = PyCode._from_code(self.space, f.func_code) + fn = Function(self.space, code, self.space.newdict()) + + assert fn.code.fast_natural_arity == i + if i < 5: + + def bomb(*args): + assert False, "shortcutting should have avoided this" + + code.funcrun = bomb + code.funcrun_obj = bomb + + args_w = map(space.wrap, range(i)) + w_res = space.call_function(fn, *args_w) + check = space.is_true(space.eq(w_res, space.wrap(res))) + assert check def test_fastcall(self): space = self.space @@ -469,28 +498,22 @@ assert fn.code.fast_natural_arity == 1 - called = [] - fastcall_1 = fn.code.fastcall_1 - def witness_fastcall_1(space, w_func, w_arg): - called.append(w_func) - return fastcall_1(space, w_func, w_arg) + def bomb(*args): + assert False, "shortcutting should have avoided this" - fn.code.fastcall_1 = witness_fastcall_1 + code.funcrun = bomb + code.funcrun_obj = bomb w_3 = space.newint(3) w_res = space.call_function(fn, w_3) assert w_res is w_3 - assert called == [fn] - - called = [] w_res = space.appexec([fn, w_3], """(f, x): return f(x) """) assert w_res is w_3 - assert called == [fn] def test_fastcall_method(self): space = self.space @@ -502,13 +525,11 @@ assert fn.code.fast_natural_arity == 2 - called = [] - fastcall_2 = fn.code.fastcall_2 - def witness_fastcall_2(space, w_func, w_arg1, w_arg2): - called.append(w_func) - return fastcall_2(space, w_func, w_arg1, w_arg2) + def bomb(*args): + assert False, "shortcutting should have avoided this" - fn.code.fastcall_2 = witness_fastcall_2 + code.funcrun = bomb + code.funcrun_obj = bomb w_3 = space.newint(3) w_res = space.appexec([fn, w_3], """(f, x): @@ -521,7 +542,6 @@ """) assert space.is_true(w_res) - assert called == [fn, fn] From pedronis at codespeak.net Mon Sep 1 23:57:37 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 1 Sep 2008 23:57:37 +0200 (CEST) Subject: [pypy-svn] r57739 - pypy/branch/garden-call-code-2/pypy/interpreter Message-ID: <20080901215737.66B2F16852D@codespeak.net> Author: pedronis Date: Mon Sep 1 23:57:36 2008 New Revision: 57739 Modified: pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py Log: the shortcut in funccall works for any pycode arity Modified: pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py Mon Sep 1 23:57:36 2008 @@ -163,8 +163,6 @@ def _compute_fastcall(self): # Speed hack! self.fast_natural_arity = -99 - if not (0 <= self.co_argcount <= 4): - return if self.co_flags & (CO_VARARGS | CO_VARKEYWORDS): return if len(self._args_as_cellvars) > 0: From cami at codespeak.net Tue Sep 2 11:47:34 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 11:47:34 +0200 (CEST) Subject: [pypy-svn] r57746 - in pypy/dist/pypy/lang/gameboy: . profiling Message-ID: <20080902094734.F1491169F18@codespeak.net> Author: cami Date: Tue Sep 2 11:47:32 2008 New Revision: 57746 Removed: pypy/dist/pypy/lang/gameboy/profiling/ Modified: pypy/dist/pypy/lang/gameboy/cartridge.py pypy/dist/pypy/lang/gameboy/cpu.py pypy/dist/pypy/lang/gameboy/gameboy.py pypy/dist/pypy/lang/gameboy/gameboy_implementation.py Log: smaller refactorings, removed some print statements Modified: pypy/dist/pypy/lang/gameboy/cartridge.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/cartridge.py (original) +++ pypy/dist/pypy/lang/gameboy/cartridge.py Tue Sep 2 11:47:32 2008 @@ -100,7 +100,7 @@ self.load_battery() self.mbc = self.create_bank_controller(self.get_memory_bank_type(), self.rom, self.ram, self.clock) - print self + #print self def check_rom(self): if not self.verify_header(): Modified: pypy/dist/pypy/lang/gameboy/cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/cpu.py Tue Sep 2 11:47:32 2008 @@ -493,20 +493,20 @@ if use_cycles: self.cycles += 1 - def ld(self, getCaller, setCaller): + def load(self, getCaller, setCaller): # 1 cycle setCaller.set(getCaller.get()) # 1 cycle def load_fetch_register(self, register): - self.ld(CPUFetchCaller(self), RegisterCallWrapper(register)) + self.load(CPUFetchCaller(self), RegisterCallWrapper(register)) def store_hl_in_pc(self): # LD PC,HL, 1 cycle - self.ld(DoubleRegisterCallWrapper(self.hl), + self.load(DoubleRegisterCallWrapper(self.hl), DoubleRegisterCallWrapper(self.pc)) def fetch_load(self, getCaller, setCaller): - self.ld(CPUFetchCaller(self), setCaller) + self.load(CPUFetchCaller(self), setCaller) def add_a(self, getCaller, setCaller=None): data = getCaller.get() @@ -1043,7 +1043,7 @@ return op_codes def load_group_lambda(store_register, load_register): - return lambda s: CPU.ld(s, RegisterCallWrapper(load_register(s)), + return lambda s: CPU.load(s, RegisterCallWrapper(load_register(s)), RegisterCallWrapper(store_register(s))) def create_register_op_codes(table): Modified: pypy/dist/pypy/lang/gameboy/gameboy.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/gameboy.py (original) +++ pypy/dist/pypy/lang/gameboy/gameboy.py Tue Sep 2 11:47:32 2008 @@ -69,7 +69,6 @@ self.sound.stop() def reset(self): - print "python resetting gameboy" self.ram.reset() self.memory_bank_controller.reset() self.interrupt.reset() Modified: pypy/dist/pypy/lang/gameboy/gameboy_implementation.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/gameboy_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/gameboy_implementation.py Tue Sep 2 11:47:32 2008 @@ -36,10 +36,10 @@ self.reset() self.is_running = True try: - while self.is_running and self.handle_events(): + while self.is_running: + self.handle_events() self.emulate(constants.GAMEBOY_CLOCK >> 2) RSDL.Delay(1) - #time.sleep(10/1000) except : self.is_running = False lltype.free(self.event, flavor='raw') @@ -51,11 +51,10 @@ pass def handle_events(self): - while self.poll_event() and self.is_running: - if self.check_for_escape(): - self.is_running = False - self.joypad_driver.update(self.event) - return self.is_running + self.poll_event() + if self.check_for_escape(): + self.is_running = False + self.joypad_driver.update(self.event) def poll_event(self): @@ -100,7 +99,8 @@ #if y%2 == 0 or True: # px = self.get_pixel_color(x, y) # str += ["#", "%", "+", " ", " "][px] - RSDL_helper.set_pixel(self.screen, x, y, self.get_pixel_color(x, y)) + #RSDL_helper.set_pixel(self.screen, x, y, self.get_pixel_color(x, y)) + pass #print str; @specialize.arg(3) From cami at codespeak.net Tue Sep 2 11:51:37 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 11:51:37 +0200 (CEST) Subject: [pypy-svn] r57747 - pypy/dist/pypy/lang/gameboy/profiling Message-ID: <20080902095137.B06D9169F8C@codespeak.net> Author: cami Date: Tue Sep 2 11:51:36 2008 New Revision: 57747 Added: pypy/dist/pypy/lang/gameboy/profiling/ pypy/dist/pypy/lang/gameboy/profiling/gameboyTest.py pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Log: addde new proifling files Added: pypy/dist/pypy/lang/gameboy/profiling/gameboyTest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/gameboyTest.py Tue Sep 2 11:51:36 2008 @@ -0,0 +1,29 @@ +from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import * +from pypy.lang.gameboy.debug import debug +import py +import sys + +from AppKit import NSApplication +NSApplication.sharedApplication() + + +debug.DEBUG_PRINT_LOGS = False +ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" + +filename = "" +if len(sys.argv) > 1: + print sys.argv + filename = sys.argv[1] +else: + pos = str(9) + filename = ROM_PATH+"/rom"+pos+"/rom"+pos+".gb" + +gameBoy = GameBoyProfilingImplementation(500000) + +try: + gameBoy.load_cartridge_file(str(filename)) +except: + gameBoy.load_cartridge_file(str(filename), verify=False) + print "Cartridge is Corrupted!" + +gameBoy.mainLoop() Added: pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py Tue Sep 2 11:51:36 2008 @@ -0,0 +1,45 @@ +#!/usr/bin/env python +from __future__ import generators + +from pypy.lang.gameboy.gameboy_implementation import * +from pypy.lang.gameboy.profiling.profiling_cpu import ProfilingCPU +from pypy.lang.gameboy.debug import debug +from pypy.lang.gameboy.debug.debug_socket_memory import * + +# GAMEBOY ---------------------------------------------------------------------- + +class GameBoyProfilingImplementation(GameBoyImplementation): + + def __init__(self, op_codes): + GameBoyImplementation.__init__(self) + self.op_codes = op_codes + self.cycleLimit = cycleLimit + self.cpu = ProfilingCPU(self.interrupt, self) + self.cpu.cycle_limit = cycleLimit + + def handle_execution_error(self): + self.is_running = False + debug.print_results() + + +# CUSTOM DRIVER IMPLEMENTATIONS currently not used ============================= + +# VIDEO DRIVER ----------------------------------------------------------------- + +class VideoDriverDebugImplementation(VideoDriverImplementation): + pass + + +# JOYPAD DRIVER ---------------------------------------------------------------- + +class JoypadDriverDebugImplementation(JoypadDriverImplementation): + pass + + +# SOUND DRIVER ----------------------------------------------------------------- + +class SoundDriverDebugImplementation(SoundDriverImplementation): + pass + + +# ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Tue Sep 2 11:51:36 2008 @@ -0,0 +1,30 @@ + +from __future__ import generators +from pypy.lang.gameboy.cpu import CPU +from pypy.lang.gameboy.debug import debug + + +class ProfilingCPU(CPU): + + + def __init__(self, interrupt, memory): + CPU.__init__(self, interrupt, memory) + self.cycle_limit = 0 + self.op_code_count = 0 + self.fetch_exec_opcode_histo = [0]*(0xFF+1) + self.opcode_histo = [0]*(0xFF+1) + + def fetch_execute(self): + CPU.fetch_execute(self) + self.op_code_count += 1 + self.fetch_exec_opcode_histo[self.last_fetch_execute_op_code] += 1 + debug.log(self.last_fetch_execute_op_code, is_fetch_execute=True) + + + def execute(self, opCode): + CPU.execute(self, opCode) + debug.log(self.last_op_code) + self.op_code_count += 1 + self.opcode_histo[self.last_op_code] += 1 + #if self.op_code_count >= self.cycle_limit: + # raise Exception("Maximal Cyclecount reached") \ No newline at end of file From cami at codespeak.net Tue Sep 2 11:54:30 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 11:54:30 +0200 (CEST) Subject: [pypy-svn] r57748 - pypy/dist/pypy/lang/gameboy/profiling/evaluation Message-ID: <20080902095430.14B0916855A@codespeak.net> Author: cami Date: Tue Sep 2 11:54:29 2008 New Revision: 57748 Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py Log: added evaluation part Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py Tue Sep 2 11:54:29 2008 @@ -0,0 +1,30 @@ +from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import * +from pypy.lang.gameboy.debug import debug +import py +import sys + +from AppKit import NSApplication +NSApplication.sharedApplication() + + +debug.DEBUG_PRINT_LOGS = False +ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" + +filename = "" +if len(sys.argv) > 1: + print sys.argv + filename = sys.argv[1] +else: + pos = str(9) + filename = ROM_PATH+"/rom"+pos+"/rom"+pos+".gb" + +#print "loading rom: ", str(filename) +gameBoy = GameBoyEvaluationImplementation(500000) + +try: + gameBoy.load_cartridge_file(str(filename)) +except: + gameBoy.load_cartridge_file(str(filename), verify=False) + print "Cartridge is Corrupted!" + +gameBoy.mainLoop() Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py Tue Sep 2 11:54:29 2008 @@ -0,0 +1,44 @@ +#!/usr/bin/env python +from __future__ import generators + +from pypy.lang.gameboy.gameboy_implementation import * +from pypy.lang.gameboy.profiling.evaluation.evaluation_cpu import EvaluationCPU +from pypy.lang.gameboy.debug import debug +from pypy.lang.gameboy.debug.debug_socket_memory import * + +# GAMEBOY ---------------------------------------------------------------------- + +class GameBoyProfilingImplementation(GameBoyImplementation): + + def __init__(self, cycleLimit=0): + GameBoyImplementation.__init__(self) + self.cycleLimit = cycleLimit + self.cpu = EvaluationCPU(self.interrupt, self) + self.cpu.cycle_limit = cycleLimit + + def handle_execution_error(self): + self.is_running = False + debug.print_results() + + +# CUSTOM DRIVER IMPLEMENTATIONS currently not used ============================= + +# VIDEO DRIVER ----------------------------------------------------------------- + +class VideoDriverDebugImplementation(VideoDriverImplementation): + pass + + +# JOYPAD DRIVER ---------------------------------------------------------------- + +class JoypadDriverDebugImplementation(JoypadDriverImplementation): + pass + + +# SOUND DRIVER ----------------------------------------------------------------- + +class SoundDriverDebugImplementation(SoundDriverImplementation): + pass + + +# ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py Tue Sep 2 11:54:29 2008 @@ -0,0 +1,30 @@ + +from __future__ import generators +from pypy.lang.gameboy.cpu import CPU +from pypy.lang.gameboy.debug import debug + + +class ProfilingCPU(CPU): + + + def __init__(self, interrupt, memory): + CPU.__init__(self, interrupt, memory) + self.cycle_limit = 0 + self.op_code_count = 0 + self.fetch_exec_opcode_histo = [0]*(0xFF+1) + self.opcode_histo = [0]*(0xFF+1) + + def fetch_execute(self): + CPU.fetch_execute(self) + self.op_code_count += 1 + self.fetch_exec_opcode_histo[self.last_fetch_execute_op_code] += 1 + debug.log(self.last_fetch_execute_op_code, is_fetch_execute=True) + + + def execute(self, opCode): + CPU.execute(self, opCode) + debug.log(self.last_op_code) + self.op_code_count += 1 + self.opcode_histo[self.last_op_code] += 1 + #if self.op_code_count >= self.cycle_limit: + # raise Exception("Maximal Cyclecount reached") \ No newline at end of file From cami at codespeak.net Tue Sep 2 11:58:21 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 11:58:21 +0200 (CEST) Subject: [pypy-svn] r57749 - in pypy/dist/pypy/lang/gameboy/profiling/evaluation: . logs Message-ID: <20080902095821.04D1F169E12@codespeak.net> Author: cami Date: Tue Sep 2 11:58:20 2008 New Revision: 57749 Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/ pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/kirbysDreamland.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/megaman.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/rom9.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/superMario.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/parseTests.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh (contents, props changed) Log: added loggin part to the evaluation evaluation should be able to create histogram of the executed op codes fo find a working set of opCodes which should run fast Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/kirbysDreamland.txt ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/megaman.txt ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/rom9.txt ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/superMario.txt ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/parseTests.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/parseTests.py Tue Sep 2 11:58:20 2008 @@ -0,0 +1,47 @@ +#!/bin/python + +def add_rank(file): + lines = open(file).readlines() + lastRank = -1; + for line in lines[4:]: + pos = line.find(":") + try: + lastRank = int(line[:pos]) + add_rank_opcodes(lastRank, line[pos+2:]) + except : + add_rank_fetch_opcodes(lastRank, line[pos+2:]) + + + +def add_rank_opcodes(rank, opcodes): + opcodes = opcodes.strip().split(" ") + for opcode in opcodes: + op_codes[int(opcode, 16)] += rank + +def add_rank_fetch_opcodes(rank, opcodes): + opcodes = opcodes.strip().split(" ") + for opcode in opcodes: + fetch_op_codes[int(opcode, 16)] += rank + + +def print_sorted(table): + dict = {} + for i in range(0xFF): + dict[table[i]] = i + keys = dict.keys() + keys.sort() + for key in keys: + if key != 0: + print "0x%2x: %5i" % (dict[key], key) + +# -------------------------------------- +files = ["superMario.txt", "rom9.txt", "megaman.txt", "kirbysDreamland.txt"] + +op_codes = [0] * 0xFF +fetch_op_codes = [0] * 0xFF + +for file in files: + add_rank(file) + +print_sorted(op_codes) + \ No newline at end of file Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh Tue Sep 2 11:58:20 2008 @@ -0,0 +1,13 @@ +#!/bin/bash +rm -rf *.txt + +romPath=~/Ausbildung/08_UNIBE_FS/bachelor/docs/roms +executable=/pypy-dist/pypy/lang/gameboy/gameboyTest.py + +python2.5 $executable $romPath/Megaman.gb >> logs/megaman.txt +python2.5 $executable $romPath/KirbysDreamLand.gb >> logs/kirbysDreamland.txt +python2.5 $executable $romPath/SuperMarioLand.gb >> logs/superMario.txt +python2.5 $executable >> logs/rom9.txt + + +python parseTests.py \ No newline at end of file From cami at codespeak.net Tue Sep 2 12:03:19 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 12:03:19 +0200 (CEST) Subject: [pypy-svn] r57750 - pypy/dist/pypy/lang/gameboy/profiling/evaluation Message-ID: <20080902100319.BEEF6169EB2@codespeak.net> Author: cami Date: Tue Sep 2 12:03:15 2008 New Revision: 57750 Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_test_parser.py - copied unchanged from r57749, pypy/dist/pypy/lang/gameboy/profiling/evaluation/parseTests.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py - copied, changed from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py - copied, changed from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py Removed: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/parseTests.py Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh Log: renamed files Copied: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py (from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py) ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_profiling_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py Tue Sep 2 12:03:15 2008 @@ -1,14 +1,13 @@ #!/usr/bin/env python -from __future__ import generators -from pypy.lang.gameboy.gameboy_implementation import * +from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * from pypy.lang.gameboy.profiling.evaluation.evaluation_cpu import EvaluationCPU from pypy.lang.gameboy.debug import debug from pypy.lang.gameboy.debug.debug_socket_memory import * # GAMEBOY ---------------------------------------------------------------------- -class GameBoyProfilingImplementation(GameBoyImplementation): +class GameBoyEvaluationImplementation(GameBoyImplementation): def __init__(self, cycleLimit=0): GameBoyImplementation.__init__(self) Copied: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py) ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboyTest.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Tue Sep 2 12:03:15 2008 @@ -1,4 +1,5 @@ -from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import * + +from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * from pypy.lang.gameboy.debug import debug import py import sys Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh Tue Sep 2 12:03:15 2008 @@ -1,8 +1,9 @@ #!/bin/bash -rm -rf *.txt + +rm -rf logs/*.txt romPath=~/Ausbildung/08_UNIBE_FS/bachelor/docs/roms -executable=/pypy-dist/pypy/lang/gameboy/gameboyTest.py +executable=/pypy-dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py python2.5 $executable $romPath/Megaman.gb >> logs/megaman.txt python2.5 $executable $romPath/KirbysDreamLand.gb >> logs/kirbysDreamland.txt @@ -10,4 +11,4 @@ python2.5 $executable >> logs/rom9.txt -python parseTests.py \ No newline at end of file +python evaluation_test_parser.py \ No newline at end of file From cami at codespeak.net Tue Sep 2 12:09:37 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 12:09:37 +0200 (CEST) Subject: [pypy-svn] r57751 - pypy/dist/pypy/lang/gameboy/profiling/evaluation Message-ID: <20080902100937.0A6DD169EB2@codespeak.net> Author: cami Date: Tue Sep 2 12:09:35 2008 New Revision: 57751 Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py - copied, changed from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py Removed: pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh Log: rearranging sources for the evaluation Copied: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py (from r57748, pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py) ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/profiling_cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py Tue Sep 2 12:09:35 2008 @@ -4,7 +4,7 @@ from pypy.lang.gameboy.debug import debug -class ProfilingCPU(CPU): +class EvaluationCPU(CPU): def __init__(self, interrupt, memory): Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py Tue Sep 2 12:09:35 2008 @@ -1,9 +1,10 @@ #!/usr/bin/env python -from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * -from pypy.lang.gameboy.profiling.evaluation.evaluation_cpu import EvaluationCPU from pypy.lang.gameboy.debug import debug +from pypy.lang.gameboy.gameboy_implementation import * from pypy.lang.gameboy.debug.debug_socket_memory import * +from pypy.lang.gameboy.profiling.evaluation.evaluation_cpu import EvaluationCPU +from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * # GAMEBOY ---------------------------------------------------------------------- Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Tue Sep 2 12:09:35 2008 @@ -1,6 +1,6 @@ -from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * from pypy.lang.gameboy.debug import debug +from pypy.lang.gameboy.profiling.evaluation.gameboy_evaluation_implementation import * import py import sys Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/run.sh Tue Sep 2 12:09:35 2008 @@ -3,7 +3,7 @@ rm -rf logs/*.txt romPath=~/Ausbildung/08_UNIBE_FS/bachelor/docs/roms -executable=/pypy-dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py +executable=gameboy_evaluation_target.py python2.5 $executable $romPath/Megaman.gb >> logs/megaman.txt python2.5 $executable $romPath/KirbysDreamLand.gb >> logs/kirbysDreamland.txt From cami at codespeak.net Tue Sep 2 15:22:26 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 15:22:26 +0200 (CEST) Subject: [pypy-svn] r57759 - in pypy/dist/pypy/lang/gameboy/profiling: . evaluation evaluation/logs Message-ID: <20080902132226.549E4169DFB@codespeak.net> Author: cami Date: Tue Sep 2 15:22:25 2008 New Revision: 57759 Removed: pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/kirbysDreamland.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/megaman.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/rom9.txt pypy/dist/pypy/lang/gameboy/profiling/evaluation/logs/superMario.txt Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_test_parser.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Log: fixed some bugs for the profiling version. second order op codes are now executed correctly Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_cpu.py Tue Sep 2 15:22:25 2008 @@ -7,9 +7,9 @@ class EvaluationCPU(CPU): - def __init__(self, interrupt, memory): + def __init__(self, interrupt, memory, cycleLimit): CPU.__init__(self, interrupt, memory) - self.cycle_limit = 0 + self.cycle_limit = cycleLimit self.op_code_count = 0 self.fetch_exec_opcode_histo = [0]*(0xFF+1) self.opcode_histo = [0]*(0xFF+1) @@ -26,5 +26,5 @@ debug.log(self.last_op_code) self.op_code_count += 1 self.opcode_histo[self.last_op_code] += 1 - #if self.op_code_count >= self.cycle_limit: - # raise Exception("Maximal Cyclecount reached") \ No newline at end of file + if self.op_code_count >= self.cycle_limit: + raise Exception("Maximal Cyclecount reached") \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_test_parser.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_test_parser.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/evaluation_test_parser.py Tue Sep 2 15:22:25 2008 @@ -1,47 +1,39 @@ #!/bin/python -def add_rank(file): - lines = open(file).readlines() - lastRank = -1; - for line in lines[4:]: - pos = line.find(":") - try: - lastRank = int(line[:pos]) - add_rank_opcodes(lastRank, line[pos+2:]) - except : - add_rank_fetch_opcodes(lastRank, line[pos+2:]) - - - -def add_rank_opcodes(rank, opcodes): - opcodes = opcodes.strip().split(" ") - for opcode in opcodes: - op_codes[int(opcode, 16)] += rank - -def add_rank_fetch_opcodes(rank, opcodes): - opcodes = opcodes.strip().split(" ") - for opcode in opcodes: - fetch_op_codes[int(opcode, 16)] += rank +import operator +def add_rank(file): + lines = open(file).readlines() + lastRank = -1; + for line in lines: + pos = line.find(":") + if pos > 0: + opcode = line[:pos].strip() + add_rank_opcode(line[:pos], int(line[pos+2:])) + + + +def add_rank_opcode(opcode, count): + if not op_codes.has_key(opcode): + op_codes[opcode] = count + else: + op_codes[opcode] += count + def print_sorted(table): - dict = {} - for i in range(0xFF): - dict[table[i]] = i - keys = dict.keys() - keys.sort() - for key in keys: - if key != 0: - print "0x%2x: %5i" % (dict[key], key) - + sorted_op_codes = sorted(op_codes.items(), key=operator.itemgetter(1)) + sorted_op_codes.reverse() + for op_code in sorted_op_codes: + print "%9s : %s" % (op_code[0], op_code[1]) + # -------------------------------------- files = ["superMario.txt", "rom9.txt", "megaman.txt", "kirbysDreamland.txt"] -op_codes = [0] * 0xFF +op_codes = {} fetch_op_codes = [0] * 0xFF for file in files: - add_rank(file) - + add_rank("logs/"+file) + print_sorted(op_codes) - \ No newline at end of file + \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_implementation.py Tue Sep 2 15:22:25 2008 @@ -13,7 +13,7 @@ def __init__(self, cycleLimit=0): GameBoyImplementation.__init__(self) self.cycleLimit = cycleLimit - self.cpu = EvaluationCPU(self.interrupt, self) + self.cpu = EvaluationCPU(self.interrupt, self, cycleLimit) self.cpu.cycle_limit = cycleLimit def handle_execution_error(self): Modified: pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Tue Sep 2 15:22:25 2008 @@ -9,7 +9,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Tue Sep 2 15:22:25 2008 @@ -9,22 +9,23 @@ def __init__(self, interrupt, memory): CPU.__init__(self, interrupt, memory) - self.cycle_limit = 0 - self.op_code_count = 0 - self.fetch_exec_opcode_histo = [0]*(0xFF+1) - self.opcode_histo = [0]*(0xFF+1) - - def fetch_execute(self): - CPU.fetch_execute(self) - self.op_code_count += 1 - self.fetch_exec_opcode_histo[self.last_fetch_execute_op_code] += 1 - debug.log(self.last_fetch_execute_op_code, is_fetch_execute=True) + self.op_codes = [] - - def execute(self, opCode): - CPU.execute(self, opCode) - debug.log(self.last_op_code) - self.op_code_count += 1 - self.opcode_histo[self.last_op_code] += 1 - #if self.op_code_count >= self.cycle_limit: - # raise Exception("Maximal Cyclecount reached") \ No newline at end of file + def run(self, op_codes): + self.op_codes = op_codes + self.pc.set(0) + i = 0 + while i < len(op_codes): + self.execute(op_codes[i]) + i += 1 + if op_codes[i] == 0xCB: + i += 1 + self.pc.set(i) # 2 cycles + + def fetch(self, use_cycles=True): + # Fetching 1 cycle + if use_cycles: + self.cycles += 1 + data = self.op_codes[self.pc.get(use_cycles) % len(self.op_codes)]; + self.pc.inc(use_cycles) # 2 cycles + return data \ No newline at end of file From cami at codespeak.net Tue Sep 2 15:23:08 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 15:23:08 +0200 (CEST) Subject: [pypy-svn] r57760 - pypy/dist/pypy/translator/goal Message-ID: <20080902132308.5126B169DFB@codespeak.net> Author: cami Date: Tue Sep 2 15:23:07 2008 New Revision: 57760 Modified: pypy/dist/pypy/translator/goal/targetgbimplementation.py pypy/dist/pypy/translator/goal/targetgbprofiling.py Log: added profiling target for the gameboy Modified: pypy/dist/pypy/translator/goal/targetgbimplementation.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetgbimplementation.py (original) +++ pypy/dist/pypy/translator/goal/targetgbimplementation.py Tue Sep 2 15:23:07 2008 @@ -35,3 +35,12 @@ def test_target(): entry_point(["b", ROM_PATH+"/rom9/rom9.gb"]) + + +# STARTPOINT =================================================================== + +if __name__ == '__main__': + from AppKit import NSApplication + NSApplication.sharedApplication() + + test_target() Modified: pypy/dist/pypy/translator/goal/targetgbprofiling.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetgbprofiling.py (original) +++ pypy/dist/pypy/translator/goal/targetgbprofiling.py Tue Sep 2 15:23:07 2008 @@ -1,32 +1,90 @@ import os +import sys import py import pdb -from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import GameBoyProfilingImplementation +import time +from pypy.lang.gameboy.cpu import OP_CODES, FETCH_EXECUTE_OP_CODES +from pypy.lang.gameboy.ram import iMemory +from pypy.lang.gameboy.interrupt import Interrupt +from pypy.lang.gameboy.profiling.profiling_cpu import ProfilingCPU -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" def entry_point(argv=None): - if argv is not None and len(argv) > 1: - filename = argv[1] + if argv is not None and len(argv) >= 1: + typical = argv[1] == "1" + print "Running typical set" else: - pos = str(9) - filename = ROM_PATH+"/rom"+pos+"/rom"+pos+".gb" - print "loading rom: ", str(filename) - gameBoy = GameBoyProfilingImplementation() - try: - gameBoy.load_cartridge_file(str(filename)) - except: - print "Corrupt Cartridge" - gameBoy.load_cartridge_file(str(filename), verify=False) - try: - gameBoy.mainLoop() - except: - pass - #pdb.runcall(gameBoy.mainLoop) - return 0 + typical = False + print "running normal set" + + cpu = ProfilingCPU(Interrupt(), iMemory()) + op_codes = FULL_LIST + if typical: + op_codes = TYPICAL_LIST + + start_time = time.time() + for i in range(1): + cpu.run(op_codes) + end_time = time.time() + print end_time - start_time + return 1 + + +def create_all_op_codes(): + list = [] + for i in range(0xFF): + if i != 0xCB and OP_CODES[i] is not None: + list.append(i) + + for i in range(0xFF): + list.append(0xCB) + list.append(i) + + return list + +def create_typical_op_codes(): + list = [] + append_to_opcode_list(list, 0xff, 911896); + append_to_opcode_list(list, 0x20, 260814); + append_to_opcode_list(list, 0xf0, 166327); + append_to_opcode_list(list, 0xfe, 166263); + append_to_opcode_list(list, 0x13, 74595); + append_to_opcode_list(list, 0x12, 74582); + append_to_opcode_list(list, 0x2a, 72546); + append_to_opcode_list(list, 0xb1, 70495); + append_to_opcode_list(list, 0xb, 70487); + append_to_opcode_list(list, 0x78, 70487); + append_to_opcode_list(list, 0x5, 24998); + append_to_opcode_list(list, 0x32, 24962); + append_to_opcode_list(list, 0x38, 4129); + append_to_opcode_list(list, 0xd, 3170); + append_to_opcode_list(list, 0x22, 1034); + append_to_opcode_list(list, 0xcd, 308); + append_to_opcode_list(list, 0x21, 294); + append_to_opcode_list(list, 0xc9, 292); + append_to_opcode_list(list, 0xf5, 284); + append_to_opcode_list(list, 0xf1, 282); + append_to_opcode_list(list, 0xc3, 277); + append_to_opcode_list(list, 0x77, 275); + append_to_opcode_list(list, 0x7e, 261); + append_to_opcode_list(list, 0x3c, 260); + append_to_opcode_list(list, 0xe0, 88); + append_to_opcode_list(list, 0x3e, 55); + append_to_opcode_list(list, 0xea, 47); + append_to_opcode_list(list, 0xaf, 45); + append_to_opcode_list(list, 0x70, 40); + append_to_opcode_list(list, 0x7d, 40); + return list + +def append_to_opcode_list(list, op_code, count): + for i in range(count): + list.append(op_code) + +TYPICAL_LIST = create_typical_op_codes() +FULL_LIST = create_all_op_codes() # _____ Define and setup target ___ @@ -34,4 +92,9 @@ return entry_point, None def test_target(): - entry_point(["b", ROM_PATH+"/rom9/rom9.gb"]) + entry_point() + + +if __name__ == '__main__': + entry_point(sys.argv) + \ No newline at end of file From cami at codespeak.net Tue Sep 2 15:23:38 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 15:23:38 +0200 (CEST) Subject: [pypy-svn] r57761 - pypy/dist/pypy/lang/gameboy Message-ID: <20080902132338.18AAF169DFB@codespeak.net> Author: cami Date: Tue Sep 2 15:23:37 2008 New Revision: 57761 Modified: pypy/dist/pypy/lang/gameboy/cpu.py Log: small fixes Modified: pypy/dist/pypy/lang/gameboy/cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/cpu.py Tue Sep 2 15:23:37 2008 @@ -425,7 +425,8 @@ # ------------------------------------------------------------------- def debug(self): - print "0xDD called" + #print "0xDD called" + pass def read(self, hi, lo=None): # memory Access, 1 cycle From cami at codespeak.net Tue Sep 2 16:43:08 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 16:43:08 +0200 (CEST) Subject: [pypy-svn] r57763 - pypy/dist/pypy/translator/goal Message-ID: <20080902144308.29647169E2A@codespeak.net> Author: cami Date: Tue Sep 2 16:43:03 2008 New Revision: 57763 Modified: pypy/dist/pypy/translator/goal/targetgbprofiling.py Log: changes Modified: pypy/dist/pypy/translator/goal/targetgbprofiling.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetgbprofiling.py (original) +++ pypy/dist/pypy/translator/goal/targetgbprofiling.py Tue Sep 2 16:43:03 2008 @@ -12,21 +12,25 @@ def entry_point(argv=None): + typical = False + count = 100 if argv is not None and len(argv) >= 1: typical = argv[1] == "1" - print "Running typical set" - else: - typical = False - print "running normal set" + if len(argv) >= 2: + count = int(argv[2]) - cpu = ProfilingCPU(Interrupt(), iMemory()) - - op_codes = FULL_LIST if typical: op_codes = TYPICAL_LIST + print "Running Typical Set" + else: + op_codes = FULL_LIST + print "Running Normal Set" + cpu = ProfilingCPU(Interrupt(), iMemory()) + + start_time = time.time() - for i in range(1): + for i in range(count): cpu.run(op_codes) end_time = time.time() print end_time - start_time @@ -35,13 +39,16 @@ def create_all_op_codes(): list = [] + forbidden = [0xCB, 211] for i in range(0xFF): - if i != 0xCB and OP_CODES[i] is not None: + if i not in forbidden and OP_CODES[i] is not None: list.append(i) - + + forbidden = [] for i in range(0xFF): - list.append(0xCB) - list.append(i) + if i not in forbidden: + list.append(0xCB) + list.append(i) return list From cami at codespeak.net Tue Sep 2 16:44:05 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 16:44:05 +0200 (CEST) Subject: [pypy-svn] r57764 - in pypy/dist/pypy/lang/gameboy/profiling: . evaluation Message-ID: <20080902144405.CBF0B169E2A@codespeak.net> Author: cami Date: Tue Sep 2 16:44:01 2008 New Revision: 57764 Added: pypy/dist/pypy/lang/gameboy/profiling/__init__.py pypy/dist/pypy/lang/gameboy/profiling/evaluation/__init__.py Modified: pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Log: added __init__.py files Added: pypy/dist/pypy/lang/gameboy/profiling/__init__.py ============================================================================== Added: pypy/dist/pypy/lang/gameboy/profiling/evaluation/__init__.py ============================================================================== Modified: pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/profiling_cpu.py Tue Sep 2 16:44:01 2008 @@ -13,19 +13,17 @@ def run(self, op_codes): self.op_codes = op_codes - self.pc.set(0) i = 0 while i < len(op_codes): + self.pc.set(i) self.execute(op_codes[i]) - i += 1 if op_codes[i] == 0xCB: i += 1 - self.pc.set(i) # 2 cycles + i += 1 def fetch(self, use_cycles=True): - # Fetching 1 cycle if use_cycles: self.cycles += 1 - data = self.op_codes[self.pc.get(use_cycles) % len(self.op_codes)]; - self.pc.inc(use_cycles) # 2 cycles + data = self.op_codes[self.pc.get() % len(self.op_codes)]; + self.pc.inc(use_cycles) return data \ No newline at end of file From cami at codespeak.net Tue Sep 2 17:10:23 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 2 Sep 2008 17:10:23 +0200 (CEST) Subject: [pypy-svn] r57766 - pypy/dist/pypy/lang/gameboy/debug Message-ID: <20080902151023.8F8391683D5@codespeak.net> Author: cami Date: Tue Sep 2 17:10:20 2008 New Revision: 57766 Added: pypy/dist/pypy/lang/gameboy/debug/ pypy/dist/pypy/lang/gameboy/debug/__init__.py pypy/dist/pypy/lang/gameboy/debug/debug.py pypy/dist/pypy/lang/gameboy/debug/debug_cpu.py pypy/dist/pypy/lang/gameboy/debug/debug_rpc_xml_memory.py pypy/dist/pypy/lang/gameboy/debug/debug_socket_memory.py pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py (contents, props changed) pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_implementation.py Log: added debug dir debug has some special implementation which is able to compare ram and other values with the working java counterpart Added: pypy/dist/pypy/lang/gameboy/debug/__init__.py ============================================================================== Added: pypy/dist/pypy/lang/gameboy/debug/debug.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/debug.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,125 @@ +import operator + +DEBUG = True +DEBUG_PRINT_LOGS = True +# the following list contains all opcode which have been double checked +# meaning that the test have been reviewed and the code likewise. + +op_codes = [0] * (0xFF+1) +fetch_execute_op_codes = [0] * (0xFF+1) +COUNT = [0] +CHECKED_OP_CODES = [0x00] +# load commands are checked + halt 0x76 +CHECKED_OP_CODES += range(0x40, 0x80) +# and A_B to A_A +CHECKED_OP_CODES += range(0xA0, 0xA8) +# xor A_B to A_A +CHECKED_OP_CODES += range(0xA8, 0xB0) +# or A_B to A_A +CHECKED_OP_CODES += range(0xB0, 0xB8) +CHECKED_OP_CODES += [ + # double register fetch_nn load + 0x01, 0x11, 0x21, 0x31, + # register fetch_nn load + 0x06, 0x0E, 0x16, 0x1E, 0x26, 0x2E, 0x36, 0x3E, + # store_fetched_memory_in_a + 0xFA, + # return contditional + 0xC0, 0xC8, 0xD0, 0xD8, + # restart + 0xC7, 0xCF, 0xD7, 0xDF, 0xE7, 0xEF, 0xF7, 0xFF, + # increment double register + 0x03, 0x13, 0x23, 0x33, + # decrement double register + 0x0B, 0x1B, 0x2B, 0x3B, + # decrement register + 0x05, 0x0D, 0x15, 0x1D, 0x25, 0x2D, 0x35, 0x3D, + # increment register + 0x04, 0x0C, 0x14, 0x1C, 0x24, 0x2C, 0x34, 0x3C, + # enable interrupts + 0xFB, + # disable interrupts + 0xF3, + # return from interrupt + 0xD9, + # svn comm a fetch + 0xC6, + # conditional jump + 0xD2, + # add_hl_bc up to add_hl_sp + 0x09, 0x19, 0x29, 0x39, + # add_a_b thru add_a_A + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + # subtract with carry + 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, + # xor a + 0xEE, + # add with carry + 0x88, + # sp = hl + 0xf9, + # subtract a + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + # add with carry + 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + # compare a + 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + # conditional jumps + 0xC2, 0xCA, 0xD2, 0xCA, + # complement a + 0x2F, + # write_a_at_expaded_c_address + 0xe2, + # conditional calls + 0xC4, 0xCC, 0xD4, 0xDC, + # push Double Registers + 0xC5, 0xD5, 0xE5, 0xF5, + # store_memory_at_de_in_a + 0x1A, + # jump double fetch + 0xC3, + # store_a_at_fetched_address + 0xEA, + # relative jump + 0x18, +] + +CHECKED_OP_CODES = [0x00] +CHECKED_FETCH_OP_CODES = [] + +def log(opCode, is_fetch_execute=False): + global COUNT, op_codes, fetch_execute_op_codes + if DEBUG_PRINT_LOGS: + print "="*40 + if is_fetch_execute: + print COUNT[0], " FETCH EXEC: %i | %s" % (opCode, hex(opCode)) + else: + print COUNT[0], " EXEC: %i | %s" % (opCode, hex(opCode)) + print "-"*40 + + if is_fetch_execute: + fetch_execute_op_codes[opCode ]+= 1 + else: + op_codes[opCode] += 1 + COUNT[0] += 1 + #if COUNT % 1000 == 0: + # print "." + + +def print_results(): + global COUNT, op_codes, fetch_execute_op_codes + + codes = zip(map(lambda x: "%4s" % hex(x), range(len(op_codes))), op_codes) + + fetch_exec_keys = map(lambda x: "%4s %4s" % (hex(x[0]), hex(x[1])), + zip([0x83]*len(fetch_execute_op_codes), + range(len(fetch_execute_op_codes)))) + + codes.extend(zip(fetch_exec_keys, fetch_execute_op_codes)) + + codes = sorted(codes, key=operator.itemgetter(1)) + for code in codes: + if code[1] != 0: + print "%8s : %s" % (code[0], code[1]) + + \ No newline at end of file Added: pypy/dist/pypy/lang/gameboy/debug/debug_cpu.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/debug_cpu.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,18 @@ + +from __future__ import generators +from pypy.lang.gameboy.cpu import CPU +from pypy.lang.gameboy.debug import debug + + +class DebugCPU(CPU): + + def fetch_execute(self): + CPU.fetch_execute(self) + debug.log(self.last_fetch_execute_op_code, is_fetch_execute=True) + self.memory.handle_executed_op_code(is_fetch_execute=True) + + + def execute(self, opCode): + CPU.execute(self, opCode) + debug.log(self.last_op_code) + self.memory.handle_executed_op_code(is_fetch_execute=False) \ No newline at end of file Added: pypy/dist/pypy/lang/gameboy/debug/debug_rpc_xml_memory.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/debug_rpc_xml_memory.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,368 @@ + +from socket import * +from pypy.lang.gameboy import cartridge +from pypy.lang.gameboy import constants +from socket import * +from SimpleXMLRPCServer import * +import sys, threading, time, pdb + + +# ----------------------------------------------------------------------------- + +DEBUG = False + +class printframe(object): + open = 0 + def __init__(self, text): + global DEBUG, open + printframe.open = 0 + self.text = text + + def __call__(self, f): + def wrapper(*args, **kwargs): + global DEBUG + shift = " "*printframe.open + if DEBUG: + print "python:", shift, self.text, "..." + printframe.open += 1 + val = f(*args, **kwargs) + if DEBUG: + print "python:", shift, self.text, "DONE" + printframe.open -= 1 + return val + + return wrapper + +# ----------------------------------------------------------------------------- + +class DebugRpcXmlMemory(SimpleXMLRPCServer, threading.Thread): + + + def __init__(self, gameboy_debug, debuggerPort, skipExecs): + threading.Thread.__init__(self) + SimpleXMLRPCServer.__init__(self, ("localhost", debuggerPort)) + print "python: DEBUGGER PORT:", debuggerPort + self.skipExecs = skipExecs; + self.debuggerPort = debuggerPort + self.gameboy_debug = gameboy_debug + self.cpu = self.gameboy_debug.cpu + self.pending = False + self.started = False + self.rom_checked = False + self.pending_steps = 0 + self.showed_skip_message_count = 0 + self.logRequests = False + self.compare_failed = False + self.is_closed = False + #self.rpc_paths.append("/pygirl") + self.register_introspection_functions() + self.register_functions() + self.start() + + def run(self): + self.serve_forever() + + def register_functions(self): + for fn in [(self.start_debug, "start"), + (self.check_rom, "check_rom"), + (self.close, "close"), + (self.compare, "compare"), + (self.has_next, "has_next"), + (self.next, "next")]: + self.register_function(fn[0], fn[1]) + + # RPC =================================================================== + + def close(self): + if not self.is_closed: + print "python: called close" + self.server_close() + self.is_closed = True + raise Exception("CLOSED CONNECTION") + + def start_debug(self): + print "python: called start" + self.started = True + return "started" + + @printframe("checking rom") + def check_rom(self, data): + self.compare_memory("ROM", self.cpu.rom, data["rom"]) + self.compare_memory("registeredBitmap", constants.REGISTERED_BITMAP, \ + data["registeredBitmap"]) + self.compare_cartridge(data) + self.rom_checked = True + return "checkedRom" + + @printframe("compare elements") + def compare(self, last_op_code, last_fetch_exec_op_code, instruction_count, + registers, interrupts, ram, video, timer, cycles): + self.print_check("instruction count", \ + self.cpu.instruction_counter, instruction_count) + self.compare_op_codes(last_op_code, last_fetch_exec_op_code) + self.compare_registers(registers) + self.compare_interrupts(interrupts) + self.compare_ram(ram) + self.compare_video(video) + self.compare_timer(timer) + self.compare_cycles(cycles) + self.pending = False + return "checked" + + @printframe("waiting for next") + def next(self): + self.wait_for_next_op_code() + return "next" + + def has_next(self): + print "python: called has_next" + return self.pending + + # ========================================================================== + + @printframe("checking cartridge data") + def compare_cartridge(self, data): + self.print_check("cartridge ROM size", \ + self.gameboy_debug.cartridge_manager.get_rom_size(), \ + data["ramSize"]) + self.print_check("cartridge RAM size", + self.gameboy_debug.cartridge_manager.get_ram_size(), \ + data["romSize"]) + self.print_check("cartridge Memory Bank Type", \ + self.gameboy_debug.cartridge_manager.get_memory_bank_type(), \ + data["type"]) + self.print_check("cartridge checksum", \ + self.gameboy_debug.cartridge_manager.get_checksum(), \ + data["checksum"]) + self.print_check("cartridge has battery", \ + self.gameboy_debug.cartridge_manager.has_battery(), \ + data["hasBattery"]) + + @printframe("comparing op codes") + def compare_op_codes(self, last_op_code, last_fetch_exec_op_code): + self.print_check("last opCode" , self.cpu.last_op_code, last_op_code) + self.print_check("last opCode" , self.cpu.last_fetch_execute_op_code, \ + last_fetch_exec_op_code) + + @printframe("comparing registers") + def compare_registers(self, registers): + for reg in [("a", self.cpu.a.get()), ("f", self.cpu.f.get()), + ("b", self.cpu.b.get()), ("c", self.cpu.c.get()), + ("d", self.cpu.d.get()), ("e", self.cpu.e.get()), + ("h", self.cpu.h.get()), ("l", self.cpu.l.get()), + ("sp", self.cpu.sp.get()), ("pc", self.cpu.pc.get())]: + self.print_check("register %s" % reg[0], reg[1], registers[reg[0]]) + + @printframe("comparing interrupts") + def compare_interrupts(self, interrupt): + self.print_check("interrupt ime" , \ + self.cpu.ime, interrupt["ime"]) + self.print_check("interrupt halted" , \ + self.cpu.halted, interrupt["halted"]) + self.print_check("interrupt enable" , \ + self.cpu.interrupt.get_enable_mask(), \ + interrupt["enabled"]) + self.print_check("interrupt flag" , \ + self.cpu.interrupt.get_interrupt_flag(), \ + interrupt["flag"]) + + @printframe("comparing ROM") + def compare_ram(self, ram): + self.compare_memory("wram", \ + self.gameboy_debug.ram.work_ram, ram["wram"]) + self.compare_memory("hram", \ + self.gameboy_debug.ram.hi_ram, ram["hram"]) + self.compare_memory("catridge external", \ + self.get_external_cartridge_ram(), ram["ext"]) + + def get_external_cartridge_ram(self): + ram = [0xFF] * (0xBFFF-0xA000+1) + if self.gameboy_debug.cartridge_manager.mbc.ram_enable: + for i in range(len(ram)): + ram[i] = self.gameboy_debug.read(0xA000+i) + return ram + + @printframe("comparing video") + def compare_video(self, video): + self.compare_video_memory(video) + self.compare_video_registers(video) + + @printframe("comparing memory") + def compare_video_memory(self, video): + self.compare_memory("video vram", self.gameboy_debug.video.vram, \ + video["vram"]) + self.compare_memory("video object attribute memory oam", \ + self.gameboy_debug.video.oam, video["oam"]) + self.compare_memory("video line", self.gameboy_debug.video.line, \ + video["line"]) + self.compare_memory("video objects", self.gameboy_debug.video.objects, \ + video["objects"]) + self.compare_memory("video palette", self.gameboy_debug.video.palette, \ + video["palette"]) + + @printframe("comparing registers") + def compare_video_registers(self, video): + self.print_check("video dirty", \ + self.gameboy_debug.video.dirty, video["dirty"]) + self.print_check("video display", \ + self.gameboy_debug.video.display, video["display"]) + self.print_check("video bgp", \ + self.gameboy_debug.video.background_palette, \ + video["bgp"]) + self.print_check("video dma", \ + self.gameboy_debug.video.dma, video["dma"]) + self.print_check("video frames", \ + self.gameboy_debug.video.frames, video["frames"]) + self.print_check("video frameSkip", \ + self.gameboy_debug.video.frame_skip, \ + video["frameSkip"]) + self.print_check("video lcdc", \ + self.gameboy_debug.video.control.read(), video["lcdc"]) + self.print_check("video ly", \ + self.gameboy_debug.video.line_y, video["ly"]) + self.print_check("video obp0", \ + self.gameboy_debug.video.object_palette_0, \ + video["obp0"]) + self.print_check("video obp1", \ + self.gameboy_debug.video.object_palette_1, \ + video["obp1"]) + self.print_check("video scx", \ + self.gameboy_debug.video.scroll_x, video["scx"]) + self.print_check("video scy", \ + self.gameboy_debug.video.scroll_y, video["scy"]) + self.print_check("video stat", \ + self.gameboy_debug.video.status.read(), video["stat"]) + self.print_check("video transfer", \ + self.gameboy_debug.video.transfer, video["transfer"]) + self.print_check("video vblank", \ + self.gameboy_debug.video.vblank, video["vblank"]) + self.print_check("video wly", \ + self.gameboy_debug.video.window_line_y, video["wly"]) + self.print_check("video wx", \ + self.gameboy_debug.video.window_x, video["wx"]) + self.print_check("video wy", \ + self.gameboy_debug.video.window_y, video["wy"]) + + @printframe("comparing timer") + def compare_timer(self, data): + self.print_check("timer div", \ + self.gameboy_debug.timer.divider, \ + data["div"]) + self.print_check("timer dividerCycles", \ + self.gameboy_debug.timer.divider_cycles, \ + data["dividerCycles"]) + self.print_check("timer tac", \ + self.gameboy_debug.timer.timer_control, \ + data["tac"]) + self.print_check("timer tima", \ + self.gameboy_debug.timer.timer_counter, \ + data["tima"]) + self.print_check("timer timerClock", \ + self.gameboy_debug.timer.timer_clock, \ + data["timerClock"]) + self.print_check("timer timerCycles", \ + self.gameboy_debug.timer.timer_cycles, \ + data["timerCycles"]) + self.print_check("timer tma", \ + self.gameboy_debug.timer.timer_modulo, \ + data["tma"]) + + @printframe("comparing cycles") + def compare_cycles(self, data): + self.print_check("cycles video", \ + self.gameboy_debug.video.cycles, data["video"]) + self.print_check("cycles cpu", \ + self.gameboy_debug.cpu.cycles, data["cpu"]) + self.print_check("cycles serial", \ + self.gameboy_debug.serial.cycles, data["serial"]) + self.print_check("cycles joypad", \ + self.gameboy_debug.joypad.cycles, data["joypad"]) + #sound not yet implemented so no use for checking cycles here + #self.print_check("cycles sound", #self.gameboy_debug.sound.cycles, + # 0, data["sound"]) + + # ========================================================================== + + def compare_memory(self, name, expected, new): + self.print_check(name+" length", len(expected), len(new)) + if len(expected) != len(new): return + for address in range(len(expected)): + self.print_check(name+" value at "+hex(address), \ + expected[address], new[address]) + + def print_check(self, msg, expected, got): + if expected != got: + self.compare_failed = True + print "python: !!", msg, "expected:", expected, "got:", got, "!!" + + # ========================================================================== + + def wait_for_client_start(self): + print "python: waiting for client to start" + while not self.started: + self.sleep() + + def wait_for_rom_check(self): + print "python: waiting for client to send rom" + while not self.rom_checked: + self.sleep() + + def wait_until_checked(self): + while self.pending: + self.sleep() + + def wait_for_next_op_code(self): + while not self.pending: + self.sleep() + + def sleep(self): + time.sleep(10/1000) + + def wait_for_user_input(self): + if self.compare_failed: + self.compare_failed = False + self.handle_compare_failed() + if self.pending_steps > 0: + self.pending_steps -= 1 + return + #self.prompt_for_user_input() + + + + def prompt_for_user_input(self): + if self.showed_skip_message_count < 2: + print ">> enter skip, default is 0:" + self.showed_skip_message_count += 1 + read = sys.stdin.readline() + try: + if int(read) > 0: + self.pending_steps = int(read) + except Exception: + if ("stop" in read) or ("exit" in read) or (read is "Q"): + raise Exception("Debug mode Stopped by User") + + def handle_compare_failed(self): + for i in range(3): + time.sleep(1) + print '\a' + self.pending_steps = 0 + + # ========================================================================== + + @printframe("waiting for client to start") + def start_debug_session(self): + self.wait_for_client_start() + self.wait_for_rom_check() + + @printframe("handle_executed_op_code") + def handle_executed_op_code(self, is_fetch_execute=False): + if self.cpu.instruction_counter > self.skipExecs: + self.pending = True + self.wait_for_user_input() + self.wait_until_checked() + #if self.cpu.instruction_counter == 6154: + #pdb.set_trace() + + + def print_mismatch(self, part, expected, got): + print "python:", str(part), "expected:", str(expected), "got:", str(got) Added: pypy/dist/pypy/lang/gameboy/debug/debug_socket_memory.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/debug_socket_memory.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,33 @@ + +from socket import * +from pypy.lang.gameboy import cartridge +import socket +import sys +import threading + + +def map_binary_to_int(binary): + return ord(binary) + +def map_double_binary_to_int(code): + #maps a double value to a long + return ord(code[0]) + ord(code[1]) >> 8 + +class DebugSocketMemory(object): + + + def __init__(self, gameboy_debug, debuggerPort): + self.debuggerPort = debuggerPort + self.user_wait_skip_count = 0 + self.gameboy_debug = gameboy_debug + self.cpu = self.gameboy_debug.cpu + + def close(self): + pass + + def start_debug_session(self): + self.compare_rom() + + def compare_rom(self): + pass + Added: pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,79 @@ +from pypy.lang.gameboy.debug.gameboy_debug_implementation import * +from pypy.lang.gameboy.debug.debug_rpc_xml_memory import * +import py +import sys +import os +import threading +import pdb + +# ------------------------------------------------------------------------------ + +from AppKit import NSApplication +NSApplication.sharedApplication() +# ------------------------------------------------------------------------------ + +ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +filename = "/Users/cami/Ausbildung/08_UNIBE_FS/bachelor/docs/roms/DieMaus.gb" +filename = ROM_PATH + "/rom9/rom9.gb" +SOCKET_PORT = 55686 +skipExecs = 22545 +skipExecs = 0 + +# ------------------------------------------------------------------------------ + +def parse_file_name(): + global filename + if len(filename) == 0: + pos = str(9) + filename = ROM_PATH+"/rom"+pos+"/rom"+pos+".gb" + print "loading rom: ", str(filename) + +# ------------------------------------------------------------------------------ + + +def start_python_version(): + global filename, skipExecs + gameBoy = GameBoyDebugImplementation(SOCKET_PORT, skipExecs, DebugRpcXmlMemory) + try: + gameBoy.load_cartridge_file(str(filename)) + except: + gameBoy.load_cartridge_file(str(filename), verify=False) + print "Cartridge is Corrupted!" + try: + gameBoy.mainLoop() + except: + print "stopped" + + + +JMARIO_DIR = str(py.magic.autopath().dirpath().dirpath()\ + .dirpath().dirpath()\ + .dirpath().dirpath()) + "/jmario" +JAVA_CLASSPATH =[JMARIO_DIR+"/bin/", + JMARIO_DIR+"/lib/xmlrpc-client-3.1.jar", + JMARIO_DIR+"/lib/xmlrpc-common-3.1.jar", + JMARIO_DIR+"/lib/ws-commons-util-1.0.2.jar", + JMARIO_DIR+"/lib/commons-logging-1.1.jar"]; + +def start_java_version(): + global filename + command = "java" + \ + " -classpath "+ (':'.join(JAVA_CLASSPATH)) +\ + " gameboy.platform.debug.MainDebug " + \ + filename + " " + \ + str(SOCKET_PORT) + " " + \ + str(skipExecs) + #command = "java" + \ + # " -classpath "+ (':'.join(JAVA_CLASSPATH)) +\ + # " gameboy.platform.j2se.Main " + \ + # filename + " " + os.system(command) + + + +# START ======================================================================== +parse_file_name() +threading.Timer(1, start_java_version).start() +start_python_version() + + Added: pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_implementation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/debug/gameboy_debug_implementation.py Tue Sep 2 17:10:20 2008 @@ -0,0 +1,73 @@ +#!/usr/bin/env python +from __future__ import generators + +from pypy.lang.gameboy.gameboy_implementation import * +from pypy.lang.gameboy.debug.debug_cpu import DebugCPU +from pypy.lang.gameboy.debug import debug +from pypy.lang.gameboy.debug.debug_socket_memory import * +import time + +# GAMEBOY ---------------------------------------------------------------------- + +class GameBoyDebugImplementation(GameBoyImplementation): + + def __init__(self, debuggerPort, skipExecs=0, memory_class=DebugSocketMemory): + GameBoy.__init__(self) + self.cpu = DebugCPU(self.interrupt, self) + self.init_sdl() + self.memory = memory_class(self, debuggerPort, skipExecs) + + + def create_drivers(self): + # make sure only the debug drivers are implemented + self.clock = Clock() + self.joypad_driver = JoypadDriverDebugImplementation() + self.video_driver = VideoDriverDebugImplementation() + self.sound_driver = SoundDriverImplementation() + + + def handle_execution_error(self): + print "closing socket connections" + self.is_running = False + debug.print_results() + self.memory.close() + + def handle_executed_op_code(self, is_fetch_execute=True): + self.memory.handle_executed_op_code(is_fetch_execute) + + def mainLoop(self): + self.memory.start_debug_session() + GameBoyImplementation.mainLoop(self) + + + + +# VIDEO DRIVER ----------------------------------------------------------------- + +class VideoDriverDebugImplementation(VideoDriver): + + + def __init__(self): + # do not initialize any libsdl stuff + VideoDriver.__init__(self) + + def update_display(self): + # dont update the display, we're here only for testing + pass + + +# JOYPAD DRIVER ---------------------------------------------------------------- + +class JoypadDriverDebugImplementation(JoypadDriver): + + def __init__(self): + JoypadDriver.__init__(self) + + +# SOUND DRIVER ----------------------------------------------------------------- + +class SoundDriverDebugImplementation(SoundDriverImplementation): + pass + + +# ============================================================================== From pedronis at codespeak.net Tue Sep 2 18:30:21 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 2 Sep 2008 18:30:21 +0200 (CEST) Subject: [pypy-svn] r57767 - pypy/extradoc/planning/1.1 Message-ID: <20080902163021.D5B6D169F83@codespeak.net> Author: pedronis Date: Tue Sep 2 18:30:19 2008 New Revision: 57767 Added: pypy/extradoc/planning/1.1/testing_infrastructure.txt Log: summary of friday's meeting about the testing infrastructure Added: pypy/extradoc/planning/1.1/testing_infrastructure.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/1.1/testing_infrastructure.txt Tue Sep 2 18:30:19 2008 @@ -0,0 +1,50 @@ +This is a summary of the meeting held Fri 29 Aug 2008 about improving +our testing infrastructure. + +People present (irc nicks): +arigato, fijal, hpk, ondrej, sanxiyn, pedronis, mon-mon, getxsick, iko, +bgola, exarkun, jacob22 + +We discussed the current available machines, the state of buildbot work and requirements for out testing infrastructure. + +We listed the following requirements: + +- running pypy own tests + +- running py.test -A tests with translated pypy-cs + +- running CPython test suite with translated pypy-cs + +- running tests on branches (possibly also from a tarball) + +- nightly runs with the possibility to trigger runs manually + +- covering Linux x86-32 and Windows 32 bits (given the current state + it seems we need two machines for each platform, ones for pypy own + tests and one for translating and running tests with translated + pypyc-s) + +After looking at the state of buildbot work we mainly decided to try +to extend buildbot usage to cover all our nightly testing needs. We +looked at machines availability/assignments at least for the +development phase of switching over to buildbot. + +wyvern.cs.uni-duesseldorf.de: buildmaster at least for the +experimenting phase, buildslave running pypy own tests (as it does +now) + +cobra.cs.uni-duesseldorf.de: buildslave for translations and running +tests with translated pypys + +snake.cs.uni-duesseldorf.de: seems to be an available Windows machine + +The first steps to proceed along this path would be to make pypy trunk +and py.lib trunk work together (this needs probably to happen on a +pypy branch at first), we also need to provide status presentations in +buildbot to fit our needs, using the py.lib trunk py.test seems the +right direction to be able to gather the necessary information. + +From a later separate discussion between pedronis and arigo: a +reasonable status presentation we may want to implement first would be +a generalization of the current output of pypy own tests nightly run. + From pedronis at codespeak.net Tue Sep 2 18:37:19 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 2 Sep 2008 18:37:19 +0200 (CEST) Subject: [pypy-svn] r57768 - pypy/extradoc/planning/1.1 Message-ID: <20080902163719.92EFF16855F@codespeak.net> Author: pedronis Date: Tue Sep 2 18:37:18 2008 New Revision: 57768 Modified: pypy/extradoc/planning/1.1/testing_infrastructure.txt Log: mention current buildbot state Modified: pypy/extradoc/planning/1.1/testing_infrastructure.txt ============================================================================== --- pypy/extradoc/planning/1.1/testing_infrastructure.txt (original) +++ pypy/extradoc/planning/1.1/testing_infrastructure.txt Tue Sep 2 18:37:18 2008 @@ -25,8 +25,11 @@ pypyc-s) After looking at the state of buildbot work we mainly decided to try -to extend buildbot usage to cover all our nightly testing needs. We -looked at machines availability/assignments at least for the +to extend buildbot usage to cover all our nightly testing +needs. exarkun and bigdog have already a buildbot based setup that can +do nightly translations: http://codespeak.net/svn/pypy/build/bot/ . + +We then looked at machines availability/assignments at least for the development phase of switching over to buildbot. wyvern.cs.uni-duesseldorf.de: buildmaster at least for the From arigo at codespeak.net Tue Sep 2 19:50:39 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 2 Sep 2008 19:50:39 +0200 (CEST) Subject: [pypy-svn] r57769 - pypy/dist/pypy/objspace Message-ID: <20080902175039.74542169E0D@codespeak.net> Author: arigo Date: Tue Sep 2 19:50:35 2008 New Revision: 57769 Modified: pypy/dist/pypy/objspace/descroperation.py Log: A small XXX. Modified: pypy/dist/pypy/objspace/descroperation.py ============================================================================== --- pypy/dist/pypy/objspace/descroperation.py (original) +++ pypy/dist/pypy/objspace/descroperation.py Tue Sep 2 19:50:35 2008 @@ -290,6 +290,8 @@ raise OperationError(space.w_TypeError, space.wrap("unhashable type")) return default_identity_hash(space, w_obj) + # XXX CPython has a special case for types with "__hash__ = None" + # to produce a nicer error message, namely "unhashable type: 'X'". w_result = space.get_and_call_function(w_hash, w_obj) if space.is_true(space.isinstance(w_result, space.w_int)): return w_result From antocuni at codespeak.net Tue Sep 2 20:07:23 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Sep 2008 20:07:23 +0200 (CEST) Subject: [pypy-svn] r57770 - pypy/extradoc/planning/1.1 Message-ID: <20080902180723.13F9C168567@codespeak.net> Author: antocuni Date: Tue Sep 2 20:07:20 2008 New Revision: 57770 Modified: pypy/extradoc/planning/1.1/testing_infrastructure.txt Log: I was present as well Modified: pypy/extradoc/planning/1.1/testing_infrastructure.txt ============================================================================== --- pypy/extradoc/planning/1.1/testing_infrastructure.txt (original) +++ pypy/extradoc/planning/1.1/testing_infrastructure.txt Tue Sep 2 20:07:20 2008 @@ -3,7 +3,7 @@ People present (irc nicks): arigato, fijal, hpk, ondrej, sanxiyn, pedronis, mon-mon, getxsick, iko, -bgola, exarkun, jacob22 +bgola, exarkun, jacob22, antocuni We discussed the current available machines, the state of buildbot work and requirements for out testing infrastructure. From hpk at codespeak.net Tue Sep 2 21:46:26 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 2 Sep 2008 21:46:26 +0200 (CEST) Subject: [pypy-svn] r57774 - pypy/branch/pypy-pytrunk Message-ID: <20080902194626.67D2A169F13@codespeak.net> Author: hpk Date: Tue Sep 2 21:46:25 2008 New Revision: 57774 Added: pypy/branch/pypy-pytrunk/ (props changed) - copied from r57773, pypy/dist/ Log: adding a branch that uses py/trunk instead of dist. py/trunk has enough legacy support that basic tests e.g. in pypy/interpreter work fine without modifications to conftests. From antocuni at codespeak.net Tue Sep 2 21:57:30 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Sep 2008 21:57:30 +0200 (CEST) Subject: [pypy-svn] r57775 - in pypy/branch/oo-jit/pypy: annotation jit/codegen/cli jit/codegen/cli/test translator/cli Message-ID: <20080902195730.CDB01169EE6@codespeak.net> Author: antocuni Date: Tue Sep 2 21:57:23 2008 New Revision: 57775 Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/translator/cli/database.py Log: a bit of refactoring of {Field,Method}Token and ObjectConst; fix some weird bugs; skip a failing test Modified: pypy/branch/oo-jit/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/binaryop.py (original) +++ pypy/branch/oo-jit/pypy/annotation/binaryop.py Tue Sep 2 21:57:23 2008 @@ -951,14 +951,12 @@ common = r2.ootype elif r2.ootype is None: common = r1.ootype - elif r1.ootype is ootype.Object or r2.ootype is ootype.Object: - common = ootype.Object - elif isinstance(r1.ootype, ootype.BuiltinType) or isinstance(r2.ootype, ootype.BuiltinType): - common = ootype.Object - else: + elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): common = ootype.commonBaseclass(r1.ootype, r2.ootype) assert common is not None, ('Mixing of incompatible classes %r, %r' % (r1.ootype, r2.ootype)) + else: + common = ootype.Object return SomeOOClass(common) class __extend__(pairtype(SomeOOInstance, SomeObject)): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Tue Sep 2 21:57:23 2008 @@ -6,7 +6,7 @@ System = CLR.System OpCodes = System.Reflection.Emit.OpCodes -class Operation: +class Operation(object): _gv_res = None def restype(self): @@ -177,11 +177,9 @@ class GetField(Operation): def __init__(self, meth, gv_obj, fieldtoken): - from pypy.jit.codegen.cli.rgenop import class2type self.meth = meth self.gv_obj = gv_obj - clitype = class2type(fieldtoken.ooclass) - self.fieldinfo = clitype.GetField(str(fieldtoken.name)) + self.fieldinfo = fieldtoken.getFieldInfo() def restype(self): return self.fieldinfo.get_FieldType() @@ -195,12 +193,10 @@ class SetField(Operation): def __init__(self, meth, gv_obj, gv_value, fieldtoken): - from pypy.jit.codegen.cli.rgenop import class2type self.meth = meth self.gv_obj = gv_obj self.gv_value = gv_value - clitype = class2type(fieldtoken.ooclass) - self.fieldinfo = clitype.GetField(str(fieldtoken.name)) + self.fieldinfo = fieldtoken.getFieldInfo() def restype(self): return None @@ -224,6 +220,28 @@ self.meth.il.Emit(OpCodes.Newobj, ctor) self.storeResult() +class OOSend(Operation): + + def __init__(self, meth, gv_self, args_gv, methtoken): + self.meth = meth + self.gv_self = gv_self + self.args_gv = args_gv + self.methodinfo = methtoken.getMethodInfo() + + def restype(self): + clitype = self.methodinfo.get_ReturnType() + if clitype is typeof(System.Void): + return None + return clitype + + def emit(self): + self.gv_self.load(self.meth) + for gv_arg in self.args_gv: + gv_arg.load(self.meth) + self.meth.il.Emit(OpCodes.Callvirt, self.methodinfo) + if self.restype() is not None: + self.storeResult() + def mark(il, s): il.Emit(OpCodes.Ldstr, s) il.Emit(OpCodes.Pop) @@ -308,7 +326,9 @@ code = source.compile() exec code in globals(), out elif value is cli_opcodes.DoNothing: - out[opname] = SameAs + # create a new subclass of SameAs; we can't use SameAs directly + # because its restype could be patched later + out[opname] = type(opname, (SameAs,), {}) else: renderCustomOp(opname, baseclass, value, out) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Tue Sep 2 21:57:23 2008 @@ -39,6 +39,19 @@ self.ooclass = ooclass self.name = name + def getFieldInfo(self): + clitype = class2type(self.ooclass) + return clitype.GetField(str(self.name)) + +class MethToken: + def __init__(self, ooclass, name): + self.ooclass = ooclass + self.name = name + + def getMethodInfo(self): + clitype = class2type(self.ooclass) + return clitype.GetMethod(str(self.name)) + class AllocToken: def __init__(self, ooclass): self.ooclass = ooclass @@ -171,14 +184,12 @@ class ObjectConst(BaseConst): - def __init__(self, obj): + def __init__(self, ooclass, obj): + self.ooclass = ooclass self.obj = obj def getCliType(self): - if self.obj == ootype.NULL: - return class2type(cObject) - cliobj = dotnet.cast_to_native_object(self.obj) - return cliobj.GetType() + return class2type(self.ooclass) def getobj(self): return dotnet.cast_to_native_object(self.obj) @@ -246,7 +257,6 @@ self.inputargs_gv = inputargs_gv def emit_trampoline(self, meth): - from pypy.jit.codegen.cli.operation import mark il = meth.il args_manager = meth.graphinfo.args_manager il.MarkLabel(self.il_trampoline_label) @@ -272,8 +282,9 @@ elif T is ootype.Float: return FloatConst(llvalue) elif isinstance(T, ootype.OOType): + ooclass = ootype.runtimeClass(T) obj = ootype.cast_to_object(llvalue) - return ObjectConst(obj) + return ObjectConst(ooclass, obj) else: assert False, "XXX not implemented" @@ -305,7 +316,8 @@ @staticmethod @specialize.memo() def methToken(TYPE, methname): - return methname #XXX + ooclass = ootype.runtimeClass(TYPE) + return MethToken(ooclass, methname) @staticmethod @specialize.memo() @@ -632,7 +644,9 @@ self.appendop(op) def genop_oosend(self, methtoken, gv_self, args_gv): - raise NotImplementedError + op = ops.OOSend(self.meth, gv_self, args_gv, methtoken) + self.appendop(op) + return op.gv_res() def genop_oononnull(self, gv_obj): raise NotImplementedError @@ -724,4 +738,4 @@ global_rgenop = RCliGenOp() RCliGenOp.constPrebuiltGlobal = global_rgenop.genconst -zero_const = ObjectConst(ootype.NULL) +zero_const = ObjectConst(cObject, ootype.NULL) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Tue Sep 2 21:57:23 2008 @@ -80,7 +80,9 @@ def test_arith_plus_minus(self): py.test.skip("Cannot work unless we add support for constant arguments in compiled tests") - test_compile_time_const_tuple = skip + def test_compile_time_const_tuple(self): + py.test.skip("Fails, and it seems to be related to missing support for constant arguments") + test_green_deepfrozen_oosend = skip test_direct_oosend_with_green_self = skip test_builtin_oosend_with_green_args = skip Modified: pypy/branch/oo-jit/pypy/translator/cli/database.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/database.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/database.py Tue Sep 2 21:57:23 2008 @@ -134,6 +134,8 @@ return self.class_name(TYPE) elif isinstance(TYPE, ootype.Record): return self.get_record_name(TYPE) + elif isinstance(TYPE, ootype.OOType): + return self.cts.lltype_to_cts(TYPE) else: assert False From hpk at codespeak.net Tue Sep 2 22:10:33 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 2 Sep 2008 22:10:33 +0200 (CEST) Subject: [pypy-svn] r57777 - pypy/branch/pypy-pytrunk/pypy/doc Message-ID: <20080902201033.2199B1683E6@codespeak.net> Author: hpk Date: Tue Sep 2 22:10:32 2008 New Revision: 57777 Modified: pypy/branch/pypy-pytrunk/pypy/doc/conftest.py Log: fixing conftest so that pypy/doc tests pass again Modified: pypy/branch/pypy-pytrunk/pypy/doc/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/doc/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/doc/conftest.py Tue Sep 2 22:10:32 2008 @@ -1,5 +1,5 @@ import py -from py.__.doc.conftest import Directory, DoctestText, ReSTChecker +from py.__.doc.conftest import Directory, DoctestText, DocfileTests from py.__.rest.directive import register_linkrole thisdir = py.magic.autopath().dirpath() @@ -15,40 +15,30 @@ ) class PyPyDoctestText(DoctestText): - - def run(self): + def runtest(self): if not option.doctests: py.test.skip("specify --enable-doctests to run doctests") - # XXX refine doctest support with respect to scoping - return super(PyPyDoctestText, self).run() - - def execute(self, module, docstring): + super(PyPyDoctestText, self).runtest() + + def getcontent(self): # XXX execute PyPy prompts as well + # but for now just get rid of those lines l = [] - for line in docstring.split('\n'): + for line in super(PyPyDoctestText, self).getcontent().split('\n'): if line.find('>>>>') != -1: line = "" l.append(line) - text = "\n".join(l) - super(PyPyDoctestText, self).execute(module, text) - - #mod = py.std.types.ModuleType(self.fspath.basename, text) - #self.mergescopes(mod, scopes) - #failed, tot = py.std.doctest.testmod(mod, verbose=1) - #if failed: - # py.test.fail("doctest %s: %s failed out of %s" %( - # self.fspath, failed, tot)) + return "\n".join(l) -class PyPyReSTChecker(ReSTChecker): +class PyPyDocfileTests(DocfileTests): DoctestText = PyPyDoctestText class Directory(Directory): - ReSTChecker = PyPyReSTChecker - def run(self): - l = super(Directory, self).run() - if 'statistic' in l: - l.remove('statistic') - return l + DocfileTests = PyPyDocfileTests + def recfilter(self, path): + if path.basename == "statistic": + return False + return super(Directory, self).recfilter(path) try: from docutils.parsers.rst import directives, states, roles From hpk at codespeak.net Tue Sep 2 22:21:33 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 2 Sep 2008 22:21:33 +0200 (CEST) Subject: [pypy-svn] r57782 - pypy/branch/pypy-pytrunk/pypy/doc Message-ID: <20080902202133.95B62169F99@codespeak.net> Author: hpk Date: Tue Sep 2 22:21:29 2008 New Revision: 57782 Modified: pypy/branch/pypy-pytrunk/pypy/doc/configuration.txt pypy/branch/pypy-pytrunk/pypy/doc/getting-started.txt Log: fixing failing doctests (haven't been run for a long time, i guess) Modified: pypy/branch/pypy-pytrunk/pypy/doc/configuration.txt ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/doc/configuration.txt (original) +++ pypy/branch/pypy-pytrunk/pypy/doc/configuration.txt Tue Sep 2 22:21:29 2008 @@ -132,6 +132,7 @@ ``OptionDescription`` instance. The attributes of the ``Config`` objects are the names of the children of the ``OptionDescription``. Example:: + >>> from pypy.config.config import OptionDescription, Config, BoolOption >>> descr = OptionDescription("options", "", [ ... BoolOption("bool", "", default=False)]) >>> Modified: pypy/branch/pypy-pytrunk/pypy/doc/getting-started.txt ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/doc/getting-started.txt (original) +++ pypy/branch/pypy-pytrunk/pypy/doc/getting-started.txt Tue Sep 2 22:21:29 2008 @@ -198,6 +198,7 @@ (e.g. the object space) and any variables you have created on the PyPy prompt with the prefix ``w_``:: + .. >>> import py ; py.test.skip("skipchunk") >>>> a = 123 >>>> *** Entering interpreter-level console *** From hpk at codespeak.net Tue Sep 2 22:47:50 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 2 Sep 2008 22:47:50 +0200 (CEST) Subject: [pypy-svn] r57783 - in pypy/branch/pypy-pytrunk/pypy: doc tool Message-ID: <20080902204750.7F837169E37@codespeak.net> Author: hpk Date: Tue Sep 2 22:47:48 2008 New Revision: 57783 Modified: pypy/branch/pypy-pytrunk/pypy/doc/conftest.py pypy/branch/pypy-pytrunk/pypy/tool/ansi_mandelbrot.py pypy/branch/pypy-pytrunk/pypy/tool/ansi_print.py Log: fix a few deprecated usages, don't depend on docutils being installed. Modified: pypy/branch/pypy-pytrunk/pypy/doc/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/doc/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/doc/conftest.py Tue Sep 2 22:47:48 2008 @@ -1,6 +1,5 @@ import py from py.__.doc.conftest import Directory, DoctestText, DocfileTests -from py.__.rest.directive import register_linkrole thisdir = py.magic.autopath().dirpath() @@ -42,6 +41,7 @@ try: from docutils.parsers.rst import directives, states, roles + from py.__.rest.directive import register_linkrole except ImportError: pass else: Modified: pypy/branch/pypy-pytrunk/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/ansi_mandelbrot.py Tue Sep 2 22:47:48 2008 @@ -1,6 +1,6 @@ import sys -from py.__.misc.terminal_helper import ansi_print, get_terminal_width +from py.__.io.terminalwriter import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/branch/pypy-pytrunk/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/ansi_print.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/ansi_print.py Tue Sep 2 22:47:48 2008 @@ -4,7 +4,7 @@ import sys -from py.__.misc.terminal_helper import ansi_print +from py.__.io.terminalwriter import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: From antocuni at codespeak.net Wed Sep 3 09:49:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 3 Sep 2008 09:49:14 +0200 (CEST) Subject: [pypy-svn] r57787 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080903074914.0EF69169FC5@codespeak.net> Author: antocuni Date: Wed Sep 3 09:49:11 2008 New Revision: 57787 Added: pypy/extradoc/talk/pycon-uk-2008/author.latex - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/author.latex pypy/extradoc/talk/pycon-uk-2008/beamerdefs.txt - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/beamerdefs.txt pypy/extradoc/talk/pycon-uk-2008/flowgraph.png - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/flowgraph.png pypy/extradoc/talk/pycon-uk-2008/makepdf - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/makepdf pypy/extradoc/talk/pycon-uk-2008/motivation.txt - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/motivation.txt pypy/extradoc/talk/pycon-uk-2008/overview1.png - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/overview1.png pypy/extradoc/talk/pycon-uk-2008/overview2.png - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/overview2.png pypy/extradoc/talk/pycon-uk-2008/pypy-vm.pdf.info - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/pypy-vm.pdf.info pypy/extradoc/talk/pycon-uk-2008/pypy-vm.txt - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/pypy-vm.txt pypy/extradoc/talk/pycon-uk-2008/rainbow.png - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/rainbow.png pypy/extradoc/talk/pycon-uk-2008/stylesheet.latex - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/stylesheet.latex pypy/extradoc/talk/pycon-uk-2008/technical.txt - copied unchanged from r57786, pypy/extradoc/talk/pycon-italy-2008/technical.txt Log: copy slides from pycon-italy-2008 From antocuni at codespeak.net Wed Sep 3 10:04:13 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 3 Sep 2008 10:04:13 +0200 (CEST) Subject: [pypy-svn] r57788 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080903080413.E4BA3169F75@codespeak.net> Author: antocuni Date: Wed Sep 3 10:04:05 2008 New Revision: 57788 Modified: pypy/extradoc/talk/pycon-uk-2008/author.latex Log: update dates & authors Modified: pypy/extradoc/talk/pycon-uk-2008/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/author.latex Wed Sep 3 10:04:05 2008 @@ -1,6 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy and The Art of Generating VMs]{PyPy and The Art of Generating Virtual Machines} -\author[Antonio Cuni]{Antonio Cuni\\DISI, Universit? degli Studi di Genova} -\institute[PyCon Due 2008]{PyCon Due 2008 - Firenze} -\date{May 10, 2008} +\author[A. Cuni, M. Fijalkowski]{Antonio Cuni \and Maciek Fijalkowski\\Merlinux GmbH} + +\institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} +\date{September 13 2008} From antocuni at codespeak.net Wed Sep 3 10:09:05 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 3 Sep 2008 10:09:05 +0200 (CEST) Subject: [pypy-svn] r57789 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080903080905.247A1169EAD@codespeak.net> Author: antocuni Date: Wed Sep 3 10:08:53 2008 New Revision: 57789 Modified: pypy/extradoc/talk/pycon-uk-2008/makepdf Log: add a note where you can find rst2beamer Modified: pypy/extradoc/talk/pycon-uk-2008/makepdf ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/makepdf (original) +++ pypy/extradoc/talk/pycon-uk-2008/makepdf Wed Sep 3 10:08:53 2008 @@ -1,5 +1,8 @@ #!/bin/bash +# you can find rst2beamer.py here: +# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py + # WARNING: to work, it needs this patch for docutils # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 From hpk at codespeak.net Wed Sep 3 10:45:11 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 3 Sep 2008 10:45:11 +0200 (CEST) Subject: [pypy-svn] r57791 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080903084511.5D9781684CD@codespeak.net> Author: hpk Date: Wed Sep 3 10:45:10 2008 New Revision: 57791 Added: pypy/extradoc/sprintinfo/october-2008/announcement.txt (contents, props changed) pypy/extradoc/sprintinfo/october-2008/people.txt (contents, props changed) Log: skeleton draft for sprint annoucnement Added: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Wed Sep 3 10:45:10 2008 @@ -0,0 +1,71 @@ +======================================================== +(XXX) PyPy sprint 7-15th October, 2008 +======================================================== + +arrival day: 7th October +departure day: 15th October +Location XXX Czech Republic or Duesseldorf, GER + +------------------------------ +Goals and topics of the sprint +------------------------------ + +There are many possible and interesting sprint topics - +here are the pre-announced topics that some of us +are set to work on: + +* work towards PyPy 1.1 release +* make pypy-c fit for small devices like mobile phones +* try out Python programs and fix them or fix PyPy or fix performance bottlenecks +* other interesting stuff that you would like to work on ...;-) + +------------------------------ +Getting there +------------------------------ +XXX + +------------------------------ +Accomodation +------------------------------ + + +------------ +Registration +------------ + +If you'd like to come, please subscribe to the `pypy-sprint mailing list`_ +and drop a note about your interests and post any questions. More +organisational information will be sent to that list. + +Please register by adding yourself on the following list (via svn): + + http://codespeak.net/svn/pypy/extradoc/sprintinfo/post-ep2008/people.txt + +or on the pypy-sprint mailing list if you do not yet have check-in rights: + + http://codespeak.net/mailman/listinfo/pypy-sprint + +--------------------------------------- +Preparation (if you feel it is needed): +--------------------------------------- + +* read the `getting-started`_ pages on http://codespeak.net/pypy + +* for inspiration, overview and technical status you are welcome to + read `the technical reports available and other relevant documentation`_ + +* please direct any technical and/or development oriented questions to + pypy-dev at codespeak.net and any sprint organizing/logistical + questions to pypy-sprint at codespeak.net + +We are looking forward to meet you at the Vilnius Post EuroPython +PyPy sprint! + +The PyPy team + +.. See also .. + +.. _getting-started: http://codespeak.net/pypy/dist/pypy/doc/getting-started.html +.. _`pypy-sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint +.. _`the technical reports available and other relevant documentation`: http://codespeak.net/pypy/dist/pypy/doc/index.html +.. _`EuroPython schedule`: http://europython.org/timetable Added: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Wed Sep 3 10:45:10 2008 @@ -0,0 +1,62 @@ + +People coming to the PyPy sprint 7-14th September +===================================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + +==================== ============== ============================ + Name Arrive/Depart Accomodation +==================== ============== ============================ +Antonio Cuni +Holger Krekel +Maciej Fijalkowski +Armin Rigo +Samuele Pedroni +Carl Friedrich Bolz +Alexander Schremmer +==================== ============== ============================ + +People on the following list were present at previous sprints: + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Antonio Cuni +Holger Krekel +Maciej Fijalkowski +Jakub Gustak +Stephan Diehl +Armin Rigo +Samuele Pedroni +Carl Friedrich Bolz ? ? +Alexander Schremmer ? ? +Holger Krekel ? ? +Toon Verwaest ? ? +Camillo Bruni ? ? +Christian Tismer ? ? +Michael Hudson ? ? +Anders Lehmann ? ? +Niklaus Haldimann ? ? +Lene Wagner ? ? +Amaury Forgeot d'Arc ? ? +Valentino Volonghi ? ? +Boris Feigin ? ? +Andrew Thompson ? ? +Bert Freudenberg ? ? +Beatrice Duering ? ? +Richard Emslie ? ? +Johan Hahn ? ? +Niko Matsakis ? ? +Anders Chrigstroem ? ? +Eric van Riet Paap ? ? +Jacob Hallen ? ? +Laura Creighton ? ? +Guido Wesdorp ? ? +Leonardo Santagada ? ? +Alexandre Fayolle ? ? +Sylvain Th?nault ? ? +Toby Watson ? ? +Paul deGrandis ? ? +==================== ============== ===================== From fijal at codespeak.net Wed Sep 3 16:38:27 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 3 Sep 2008 16:38:27 +0200 (CEST) Subject: [pypy-svn] r57796 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080903143827.496CC16A033@codespeak.net> Author: fijal Date: Wed Sep 3 16:38:25 2008 New Revision: 57796 Modified: pypy/dist/pypy/lib/_ctypes/dll.py Log: update the comment Modified: pypy/dist/pypy/lib/_ctypes/dll.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/dll.py (original) +++ pypy/dist/pypy/lib/_ctypes/dll.py Wed Sep 3 16:38:25 2008 @@ -3,5 +3,5 @@ def dlopen(name, mode): # XXX mode is ignored if name is None: - return None # XXX seems to mean the cpython lib + return None # XXX this should return *all* loaded libs, dlopen(NULL) return _rawffi.CDLL(name) From antocuni at codespeak.net Wed Sep 3 18:19:28 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 3 Sep 2008 18:19:28 +0200 (CEST) Subject: [pypy-svn] r57797 - in pypy/branch/oo-jit/pypy: jit/codegen/cli/test jit/rainbow rpython/ootypesystem Message-ID: <20080903161928.CE385169FFB@codespeak.net> Author: antocuni Date: Wed Sep 3 18:19:25 2008 New Revision: 57797 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Log: the translator does not support calling ootype._meth objects; instead, we get the _bound_meth object and call it. Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Wed Sep 3 18:19:25 2008 @@ -83,8 +83,6 @@ def test_compile_time_const_tuple(self): py.test.skip("Fails, and it seems to be related to missing support for constant arguments") - test_green_deepfrozen_oosend = skip - test_direct_oosend_with_green_self = skip test_builtin_oosend_with_green_args = skip test_residual_red_call = skip test_residual_red_call_with_exc = skip Modified: pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py Wed Sep 3 18:19:25 2008 @@ -56,6 +56,23 @@ colororder = None return colororder +def make_revealargs(argiter): + def revealargs(args_gv): + args = () + j = 0 + for ARG in argiter: + if ARG == lltype.Void: + args += (None, ) + else: + genconst = args_gv[j] + arg = genconst.revealconst(ARG) + args += (arg, ) + j += 1 + return args + + return revealargs + + class BaseCallDesc(object): __metaclass__ = cachedtype @@ -76,19 +93,11 @@ if ARG == lltype.Void: voidargcount += 1 argiter = unrolling_iterable(ARGS) + revealargs = make_revealargs(argiter) def do_perform_call(rgenop, fn_or_meth, args_gv): assert len(args_gv) + voidargcount == numargs - args = () - j = 0 - for ARG in argiter: - if ARG == lltype.Void: - args += (None, ) - else: - genconst = args_gv[j] - arg = genconst.revealconst(ARG) - args += (arg, ) - j += 1 + args = revealargs(args_gv) result = maybe_on_top_of_llinterp(self.exceptiondesc, fn_or_meth)(*args) if RESULT is lltype.Void: return None @@ -195,15 +204,49 @@ self.methtoken = RGenOp.methToken(SELFTYPE, methname) self.redboxbuilder = rvalue.ll_redboxbuilder(METH.RESULT) self.colororder = None - ARGS = (SELFTYPE,) + METH.ARGS - do_perform_call = self._define_do_perform_call(ARGS, METH.RESULT) + do_perform_call = self._define_do_perform_call(SELFTYPE, METH, methname) self._define_return_value(RGenOp, METH) def perform_call(rgenop, gv_fnptr, args_gv): assert gv_fnptr is None - return do_perform_call(rgenop, meth, args_gv) + return do_perform_call(rgenop, args_gv) self.perform_call = perform_call + def _define_do_perform_call(self, SELFTYPE, METH, methname): + from pypy.rpython.lltypesystem import lltype + numargs = len(METH.ARGS)+1 + voidargcount = 0 + for ARG in METH.ARGS: + if ARG == lltype.Void: + voidargcount += 1 + argiter = unrolling_iterable(METH.ARGS) + revealargs = make_revealargs(argiter) + + # we can't call bound_meth(*args) because SomeOOBoundMeth + # does not support call_args, so we have to generate + # bound_meth(args[0], args[1], ...) and use exec + bm_args = ['args[%d]' % i for i in range(numargs-1)] + call_bm = 'result = bound_meth(%s)' % ', '.join(bm_args) + src = py.code.Source(""" + def do_perform_call(rgenop, args_gv): + try: + assert len(args_gv) + voidargcount == numargs + this = args_gv[0].revealconst(SELFTYPE) + args = revealargs(args_gv[1:]) + bound_meth = getattr(this, methname) + %s + if METH.RESULT is lltype.Void: + return None + else: + return rgenop.genconst(result) + except Exception, e: + import pdb;pdb.xpm() + """ % call_bm) + exec src.compile() in locals() + self.do_perform_call = do_perform_call + return do_perform_call + + class BytecodeWriter(object): StructTypeDesc = None Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Wed Sep 3 18:19:25 2008 @@ -854,7 +854,8 @@ elif other.obj is None: return self.obj is None else: - return self.obj == other.obj + return self.obj.__class__ == other.obj.__class__ and \ + self.obj == other.obj def __ne__(self, other): return not (self == other) From antocuni at codespeak.net Thu Sep 4 10:17:00 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 10:17:00 +0200 (CEST) Subject: [pypy-svn] r57804 - in pypy/branch/oo-jit/pypy/translator/oosupport: . test_template Message-ID: <20080904081700.094101683FA@codespeak.net> Author: antocuni Date: Thu Sep 4 10:16:55 2008 New Revision: 57804 Modified: pypy/branch/oo-jit/pypy/translator/oosupport/constant.py pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py Log: add support for pbc of type ootype.Object which contains a string Modified: pypy/branch/oo-jit/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/oosupport/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/oosupport/constant.py Thu Sep 4 10:16:55 2008 @@ -54,6 +54,12 @@ if is_primitive(TYPE): return constgen.push_primitive_constant(gen, TYPE, value) + if TYPE is ootype.Object: + obj = value.obj + T2 = ootype.typeOf(obj) + if is_primitive(T2): + return constgen.push_primitive_constant(gen, T2, obj) + const = constgen.record_const(value) if const.is_inline(): const.push_inline(gen, TYPE) Modified: pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py Thu Sep 4 10:16:55 2008 @@ -1,3 +1,5 @@ +from pypy.rpython.ootypesystem import ootype + # used in tests below class A: pass @@ -143,3 +145,12 @@ return mylist[x] res = self.interpret(fn, [0]) assert self.class_name(res) == 'A' + + def test_convert_string_to_object(self): + s = self.string_to_ll("hello world") + obj = ootype.cast_to_object(s) + def fn(): + s1 = ootype.cast_from_object(ootype.String, obj) + return s1 + res = self.interpret(fn, [], backendopt=False) + assert res == 'hello world' From antocuni at codespeak.net Thu Sep 4 11:59:38 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 11:59:38 +0200 (CEST) Subject: [pypy-svn] r57805 - in pypy/branch/oo-jit/pypy: jit/codegen/cli jit/codegen/cli/test jit/rainbow translator/oosupport Message-ID: <20080904095938.B5733169ED6@codespeak.net> Author: antocuni Date: Thu Sep 4 11:59:37 2008 New Revision: 57805 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py pypy/branch/oo-jit/pypy/translator/oosupport/constant.py Log: fix a bug in gencli when handling null ootype._object constants; add the logic to codegen/cli to turn the oosends to String objects into call to the static method helpers in pypy.runtime.String Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Thu Sep 4 11:59:37 2008 @@ -226,10 +226,10 @@ self.meth = meth self.gv_self = gv_self self.args_gv = args_gv - self.methodinfo = methtoken.getMethodInfo() + self.methtoken = methtoken def restype(self): - clitype = self.methodinfo.get_ReturnType() + clitype = self.methtoken.getReturnType() if clitype is typeof(System.Void): return None return clitype @@ -238,7 +238,7 @@ self.gv_self.load(self.meth) for gv_arg in self.args_gv: gv_arg.load(self.meth) - self.meth.il.Emit(OpCodes.Callvirt, self.methodinfo) + self.methtoken.emit_call(self.meth.il) if self.restype() is not None: self.storeResult() Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Thu Sep 4 11:59:37 2008 @@ -27,6 +27,7 @@ cChar = classof(System.Char) cUtils = classof(CLR.pypy.runtime.Utils) cFlexSwitchCase = classof(FlexSwitchCase) +cpypyString = classof(CLR.pypy.runtime.String) class SigToken: def __init__(self, args, res, funcclass): @@ -44,6 +45,7 @@ return clitype.GetField(str(self.name)) class MethToken: + def __init__(self, ooclass, name): self.ooclass = ooclass self.name = name @@ -52,6 +54,31 @@ clitype = class2type(self.ooclass) return clitype.GetMethod(str(self.name)) + def getReturnType(self): + methodinfo = self.getMethodInfo() + return methodinfo.get_ReturnType() + + def emit_call(self, il): + raise NotImplementedError + + +class VirtualMethToken(MethToken): + + def emit_call(self, il): + methodinfo = self.getMethodInfo() + il.Emit(OpCodes.Callvirt, methodinfo) + + +class StaticMethodToken(MethToken): + def __init__(self, ooclass, name): + self.ooclass = ooclass + self.name = name + + def emit_call(self, il): + methodinfo = self.getMethodInfo() + il.Emit(OpCodes.Call, methodinfo) + + class AllocToken: def __init__(self, ooclass): self.ooclass = ooclass @@ -316,8 +343,11 @@ @staticmethod @specialize.memo() def methToken(TYPE, methname): - ooclass = ootype.runtimeClass(TYPE) - return MethToken(ooclass, methname) + if TYPE is ootype.String: + return StaticMethodToken(cpypyString, methname) + else: + ooclass = ootype.runtimeClass(TYPE) + return VirtualMethToken(ooclass, methname) @staticmethod @specialize.memo() Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 11:59:37 2008 @@ -83,7 +83,6 @@ def test_compile_time_const_tuple(self): py.test.skip("Fails, and it seems to be related to missing support for constant arguments") - test_builtin_oosend_with_green_args = skip test_residual_red_call = skip test_residual_red_call_with_exc = skip test_simple_meth = skip Modified: pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py Thu Sep 4 11:59:37 2008 @@ -229,7 +229,6 @@ call_bm = 'result = bound_meth(%s)' % ', '.join(bm_args) src = py.code.Source(""" def do_perform_call(rgenop, args_gv): - try: assert len(args_gv) + voidargcount == numargs this = args_gv[0].revealconst(SELFTYPE) args = revealargs(args_gv[1:]) @@ -239,8 +238,6 @@ return None else: return rgenop.genconst(result) - except Exception, e: - import pdb;pdb.xpm() """ % call_bm) exec src.compile() in locals() self.do_perform_call = do_perform_call Modified: pypy/branch/oo-jit/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/oosupport/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/oosupport/constant.py Thu Sep 4 11:59:37 2008 @@ -38,6 +38,16 @@ def is_primitive(TYPE): return TYPE in PRIMITIVE_TYPES +def get_primitive_constant(TYPE, value): + if is_primitive(TYPE): + return TYPE, value + if TYPE is ootype.Object: + obj = value.obj + T2 = ootype.typeOf(obj) + if obj is not None and is_primitive(T2): + return T2, obj + return None, None + def push_constant(db, TYPE, value, gen): """ General method that pushes the value of the specified constant onto the stack. Use this when you want to load a constant value. @@ -50,15 +60,10 @@ """ constgen = db.constant_generator - - if is_primitive(TYPE): - return constgen.push_primitive_constant(gen, TYPE, value) - if TYPE is ootype.Object: - obj = value.obj - T2 = ootype.typeOf(obj) - if is_primitive(T2): - return constgen.push_primitive_constant(gen, T2, obj) + TYPE2, value2 = get_primitive_constant(TYPE, value) + if TYPE2 is not None: + return constgen.push_primitive_constant(gen, TYPE2, value2) const = constgen.record_const(value) if const.is_inline(): @@ -435,7 +440,8 @@ # Internal helpers def _record_const_if_complex(self, TYPE, value): - if not is_primitive(TYPE): + TYPE2, value2 = get_primitive_constant(TYPE, value) + if not TYPE2: self.db.constant_generator.record_const(value) From antocuni at codespeak.net Thu Sep 4 12:16:07 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 12:16:07 +0200 (CEST) Subject: [pypy-svn] r57806 - in pypy/branch/oo-jit/pypy: jit/codegen/cli/test translator/cli translator/cli/src Message-ID: <20080904101607.53E25169FDC@codespeak.net> Author: antocuni Date: Thu Sep 4 12:16:06 2008 New Revision: 57806 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/translator/cli/opcodes.py pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs Log: implement debug_fatalerror in gencli, and test_residual_red_call passes out of the box Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 12:16:06 2008 @@ -83,7 +83,6 @@ def test_compile_time_const_tuple(self): py.test.skip("Fails, and it seems to be related to missing support for constant arguments") - test_residual_red_call = skip test_residual_red_call_with_exc = skip test_simple_meth = skip test_simple_red_meth = skip Modified: pypy/branch/oo-jit/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/opcodes.py Thu Sep 4 12:16:06 2008 @@ -73,6 +73,7 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'debug_print': Ignore, + 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, 'is_early_constant': [PushPrimitive(ootype.Bool, False)], } Modified: pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs Thu Sep 4 12:16:06 2008 @@ -238,6 +238,12 @@ public class Utils { + + public static void debug_fatalerror(string msg) + { + throw new Exception("debug_fatalerror: " + msg); + } + public static DynamicMethod CreateDynamicMethod(string name, Type res, Type[] args) { return new DynamicMethod(name, res, args, typeof(Utils).Module); From antocuni at codespeak.net Thu Sep 4 12:21:07 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 12:21:07 +0200 (CEST) Subject: [pypy-svn] r57807 - pypy/branch/oo-jit/pypy/jit/codegen/cli/test Message-ID: <20080904102107.6B5D416A01E@codespeak.net> Author: antocuni Date: Thu Sep 4 12:21:06 2008 New Revision: 57807 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: three more tests pass out of the box Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 12:21:06 2008 @@ -83,10 +83,9 @@ def test_compile_time_const_tuple(self): py.test.skip("Fails, and it seems to be related to missing support for constant arguments") - test_residual_red_call_with_exc = skip - test_simple_meth = skip - test_simple_red_meth = skip - test_simple_red_meth_vars_around = skip + def test_residual_red_call_with_exc(self): + py.test.skip("Exceptions not yet supported") + test_yellow_meth_with_green_result = skip test_simple_indirect_call = skip test_normalize_indirect_call = skip From antocuni at codespeak.net Thu Sep 4 15:07:22 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 15:07:22 +0200 (CEST) Subject: [pypy-svn] r57808 - in pypy/branch/oo-jit/pypy/rpython: . test Message-ID: <20080904130722.7E326169F88@codespeak.net> Author: antocuni Date: Thu Sep 4 15:07:19 2008 New Revision: 57808 Modified: pypy/branch/oo-jit/pypy/rpython/rpbc.py pypy/branch/oo-jit/pypy/rpython/test/test_rdict.py Log: always return None when convert_const() a SingleFrozenPBCRepr; this fixes the test Modified: pypy/branch/oo-jit/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/rpbc.py (original) +++ pypy/branch/oo-jit/pypy/rpython/rpbc.py Thu Sep 4 15:07:19 2008 @@ -403,6 +403,9 @@ raise TyperError("getattr on a constant PBC returns a non-constant") return hop.inputconst(hop.r_result, hop.s_result.const) + def convert_const(self, value): + return None + def convert_desc(self, frozendesc): assert frozendesc is self.frozendesc return object() # lowleveltype is Void Modified: pypy/branch/oo-jit/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_rdict.py Thu Sep 4 15:07:19 2008 @@ -496,6 +496,17 @@ res = self.interpret(g, [3]) assert res == 77 + def test_singlefrozenpbc_as_value(self): + class A: + def _freeze_(self): + return True + a = A() + d = {42: a} + def fn(key): + return d[key] + res = self.interpret(fn, [42]) + assert res is None + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): From antocuni at codespeak.net Thu Sep 4 15:23:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 15:23:17 +0200 (CEST) Subject: [pypy-svn] r57809 - in pypy/branch/oo-jit/pypy/jit: codegen/cli/test rainbow/test Message-ID: <20080904132317.C7467169FB4@codespeak.net> Author: antocuni Date: Thu Sep 4 15:23:15 2008 New Revision: 57809 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py Log: a bunch of tests that passes out of the box :-) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 15:23:15 2008 @@ -86,12 +86,8 @@ def test_residual_red_call_with_exc(self): py.test.skip("Exceptions not yet supported") - test_yellow_meth_with_green_result = skip - test_simple_indirect_call = skip - test_normalize_indirect_call = skip - test_normalize_indirect_call_more = skip - test_green_char_at_merge = skip - test_self_referential_structures = skip + check_count_depth = False # see test_self_referential_structures + test_known_nonzero = skip test_debug_assert_ptr_nonzero = skip test_indirect_red_call = skip Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py Thu Sep 4 15:23:15 2008 @@ -1170,6 +1170,7 @@ S.become(lltype.GcStruct('s', ('ps', lltype.Ptr(S)))) return S + check_count_depth = True def test_self_referential_structures(self): S = self._make_self_referential_type() malloc = self.malloc @@ -1187,7 +1188,8 @@ return x res = self.interpret(f, [3], [], policy=P_NOVIRTUAL) - assert count_depth(res) == 2 + if self.check_count_depth: + assert count_depth(res) == 2 def test_known_nonzero(self): S = self.GcStruct('s', ('x', lltype.Signed)) From arigo at codespeak.net Thu Sep 4 16:18:11 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 4 Sep 2008 16:18:11 +0200 (CEST) Subject: [pypy-svn] r57810 - in pypy/branch/2.5-features/pypy/interpreter: . test Message-ID: <20080904141811.67CE3169F84@codespeak.net> Author: arigo Date: Thu Sep 4 16:18:08 2008 New Revision: 57810 Modified: pypy/branch/2.5-features/pypy/interpreter/error.py pypy/branch/2.5-features/pypy/interpreter/test/test_raise.py Log: Fix OperationError.normalize_exception() to handle old-style classes and to do the right thing about space.full_exceptions. Add comments about all cases and about space.full_exceptions, and factor out a helper because that starts to look much too long. This fixes the failure in objspace/flow/. Modified: pypy/branch/2.5-features/pypy/interpreter/error.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/error.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/error.py Thu Sep 4 16:18:08 2008 @@ -145,14 +145,39 @@ """Normalize the OperationError. In other words, fix w_type and/or w_value to make sure that the __class__ of w_value is exactly w_type. """ + # + # This method covers all ways in which the Python statement + # "raise X, Y" can produce a valid exception type and instance. + # + # In the following table, 'Class' means a subclass of BaseException + # and 'inst' is an instance of either 'Class' or a subclass of it. + # Or 'Class' can also be an old-style class and 'inst' an old-style + # instance of it. + # + # Note that 'space.full_exceptions' is set to False by the flow + # object space; in this case we must assume that we are in a + # non-advanced case, and ignore the advanced cases. Old-style + # classes and instances *are* advanced. + # + # input (w_type, w_value)... becomes... advanced case? + # --------------------------------------------------------------------- + # (tuple, w_value) (tuple[0], w_value) yes + # (Class, None) (Class, Class()) no + # (Class, inst) (inst.__class__, inst) no + # (Class, tuple) (Class, Class(*tuple)) yes + # (Class, x) (Class, Class(x)) no + # ("string", ...) ("string", ...) deprecated + # (inst, None) (inst.__class__, inst) no + # w_type = self.w_type w_value = self.w_value if space.full_exceptions: while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if space.full_exceptions and ( - space.is_true(space.abstract_isclass(w_type)) and - space.is_true(space.issubtype(w_type, space.w_BaseException))): + + if (space.is_true(space.abstract_isclass(w_type)) and + is_valid_exception_class(space, w_type)): + # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): # raise Type: we assume we have to instantiate Type w_value = space.call_function(w_type) @@ -179,20 +204,22 @@ space.warn("raising a string exception is deprecated", space.w_DeprecationWarning) - elif space.full_exceptions and space.is_true(space.isinstance(w_type, - space.w_BaseException)): + else: + # the only case left here is (inst, None), from a 'raise inst'. + w_inst = w_type + w_instclass = space.abstract_getclass(w_inst) + if not is_valid_exception_class(space, w_instclass): + instclassname = w_instclass.getname(space, '?') + msg = ("exceptions must be classes, or instances," + "or strings (deprecated) not %s" % (instclassname,)) + raise OperationError(space.w_TypeError, space.wrap(msg)) + if not space.is_w(w_value, space.w_None): raise OperationError(space.w_TypeError, space.wrap("instance exception may not " "have a separate value")) - w_value = w_type - w_type = space.abstract_getclass(w_value) - - else: - if space.full_exceptions: - msg = ("exceptions must be classes, or instances," - "or strings (deprecated) not %s" % (w_type.typedef.name)) - raise OperationError(space.w_TypeError, space.wrap(msg)) + w_value = w_inst + w_type = w_instclass self.w_type = w_type self.w_value = w_value @@ -213,6 +240,22 @@ pass # ignored +def is_valid_exception_class(space, w_type): + """Assuming that 'w_type' is a new-style or old-style class, is it + correct to use it as the class of an exception? The answer is no + if it is a new-style class that doesn't inherit from BaseException. + """ + if not space.full_exceptions: + return True # always, for the flow space + try: + return space.is_true( + space.issubtype(w_type, space.w_BaseException)) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + return True # assuming w_type is an old-style class + + # Utilities from pypy.tool.ansi_print import ansi_print Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_raise.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/test/test_raise.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/test/test_raise.py Thu Sep 4 16:18:08 2008 @@ -107,24 +107,19 @@ raises(StopIteration, f) def test_userclass(self): - # PyPy classes are new-style so can't be raised + # new-style classes can't be raised unless they inherit from + # BaseException - class A: + class A(object): def __init__(self, x=None): self.x = x def f(): - try: - raise A - except A, a: - assert a.x == None + raise A raises(TypeError, f) def f(): - try: - raise A(42) - except A, a: - assert a.x == 42 + raise A(42) raises(TypeError, f) def test_it(self): @@ -134,3 +129,49 @@ {}[C] except KeyError: pass + + def test_oldstyle_userclass(self): + class A: + __metaclass__ = _classobj + def __init__(self, val=None): + self.val = val + class Sub(A): + pass + + try: + raise Sub + except IndexError: + assert 0 + except A, a: + assert a.__class__ is Sub + + sub = Sub() + try: + raise sub + except IndexError: + assert 0 + except A, a: + assert a is sub + + try: + raise A, sub + except IndexError: + assert 0 + except A, a: + assert a is sub + assert sub.val is None + + try: + raise Sub, 42 + except IndexError: + assert 0 + except A, a: + assert a.__class__ is Sub + assert a.val == 42 + + try: + {}[5] + except A, a: + assert 0 + except KeyError: + pass From cfbolz at codespeak.net Thu Sep 4 16:58:55 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Sep 2008 16:58:55 +0200 (CEST) Subject: [pypy-svn] r57811 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904145855.B7BBD169F07@codespeak.net> Author: cfbolz Date: Thu Sep 4 16:58:54 2008 New Revision: 57811 Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt Log: write something about how to get there Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Thu Sep 4 16:58:54 2008 @@ -1,10 +1,10 @@ ======================================================== -(XXX) PyPy sprint 7-15th October, 2008 +D?sseldorf PyPy sprint 7-15th October, 2008 ======================================================== -arrival day: 7th October -departure day: 15th October -Location XXX Czech Republic or Duesseldorf, GER +The next PyPy sprint will be held in the Computer Science department of +Heinrich-Heine Universit?t D?sseldorf from the 7th to the 15th of October. + ------------------------------ Goals and topics of the sprint @@ -22,7 +22,16 @@ ------------------------------ Getting there ------------------------------ -XXX + + +The sprint will take place in a seminar room of the computer science +department. It is in the building 25.12 of the university campus, second +floor. For travel instructions see + + http://stups.cs.uni-duesseldorf.de/anreise/esbahn.php + + + ------------------------------ Accomodation @@ -58,8 +67,7 @@ pypy-dev at codespeak.net and any sprint organizing/logistical questions to pypy-sprint at codespeak.net -We are looking forward to meet you at the Vilnius Post EuroPython -PyPy sprint! +We are looking forward to meet you at the D?sseldorf PyPy sprint! The PyPy team From cfbolz at codespeak.net Thu Sep 4 17:08:44 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Sep 2008 17:08:44 +0200 (CEST) Subject: [pypy-svn] r57812 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904150844.F2EED169F88@codespeak.net> Author: cfbolz Date: Thu Sep 4 17:08:41 2008 New Revision: 57812 Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt Log: be more precise Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Thu Sep 4 17:08:41 2008 @@ -3,7 +3,11 @@ ======================================================== The next PyPy sprint will be held in the Computer Science department of -Heinrich-Heine Universit?t D?sseldorf from the 7th to the 15th of October. +Heinrich-Heine Universit?t D?sseldorf from the 8th to the 14th of +October. + +arrival day: 7th October +departure day: 15th October ------------------------------ From antocuni at codespeak.net Thu Sep 4 19:00:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 19:00:25 +0200 (CEST) Subject: [pypy-svn] r57816 - in pypy/branch/oo-jit/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli Message-ID: <20080904170025.6E0EE16A048@codespeak.net> Author: antocuni Date: Thu Sep 4 19:00:20 2008 New Revision: 57816 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/methodfactory.py pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/translator/cli/opcodes.py Log: implement ooisnull and oononnull in the CLI JIT backend; this allows most of test_known_nonzero to pass, but the test still fails because promotion is not supported yet Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/methodfactory.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/methodfactory.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/methodfactory.py Thu Sep 4 19:00:20 2008 @@ -70,6 +70,7 @@ def create_delegate(self, delegatetype, consts): t = self.typeBuilder.CreateType() methinfo = t.GetMethod("invoke") + #assemblyData.auto_save_assembly.Save() return System.Delegate.CreateDelegate(delegatetype, consts, methinfo) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Thu Sep 4 19:00:20 2008 @@ -171,7 +171,8 @@ for gv_arg in self.args_gv: gv_arg.load(self.meth) self.meth.il.EmitCall(OpCodes.Callvirt, meth_invoke, None) - self.storeResult() + if self.restype(): + self.storeResult() class GetField(Operation): @@ -299,6 +300,8 @@ return '_'.join(parts) def is_comparison(opname): + if opname in ('ooisnull', 'oononnull'): + return True suffixes = '_lt _le _eq _ne _gt _ge'.split() for suffix in suffixes: if opname.endswith(suffix): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Thu Sep 4 19:00:20 2008 @@ -679,10 +679,16 @@ return op.gv_res() def genop_oononnull(self, gv_obj): - raise NotImplementedError + OP = ops.getopclass1('oononnull') + op = OP(self.meth, gv_obj) + self.appendop(op) + return op.gv_res() def genop_ooisnull(self, gv_obj): - raise NotImplementedError + OP = ops.getopclass1('ooisnull') + op = OP(self.meth, gv_obj) + self.appendop(op) + return op.gv_res() def genop_new(self, alloctoken): op = ops.New(self.meth, alloctoken) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 19:00:20 2008 @@ -88,7 +88,9 @@ check_count_depth = False # see test_self_referential_structures - test_known_nonzero = skip + def test_known_nonzero(self): + py.test.skip("Involves promotion") + test_debug_assert_ptr_nonzero = skip test_indirect_red_call = skip test_indirect_red_call_with_exc = skip Modified: pypy/branch/oo-jit/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/opcodes.py Thu Sep 4 19:00:20 2008 @@ -47,9 +47,6 @@ 'cli_eventhandler': [EventHandler], 'cli_getstaticfield': [GetStaticField], 'cli_setstaticfield': [SetStaticField], - 'oois': 'ceq', - 'ooisnull': [PushAllArgs, 'ldnull', 'ceq'], - 'oononnull': [PushAllArgs, 'ldnull', 'ceq']+Not, 'instanceof': [CastTo, 'ldnull', 'cgt.un'], 'subclassof': [PushAllArgs, 'call bool [pypylib]pypy.runtime.Utils::SubclassOf(class [mscorlib]System.Type, class[mscorlib]System.Type)'], 'ooidentityhash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], @@ -109,6 +106,10 @@ 'ullong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], 'ullong_invert': 'not', + 'oois': 'ceq', + 'ooisnull': [PushAllArgs, 'nop', 'nop', 'nop', 'ldnull', 'ceq'], + 'oononnull': [PushAllArgs, 'nop', 'nop', 'nop', 'nop', 'ldnull', 'ceq']+Not, + # when casting from bool we want that every truth value is casted # to 1: we can't simply DoNothing, because the CLI stack could # contains a truth value not equal to 1, so we should use the !=0 From cfbolz at codespeak.net Thu Sep 4 19:27:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 4 Sep 2008 19:27:12 +0200 (CEST) Subject: [pypy-svn] r57819 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904172712.67410168407@codespeak.net> Author: cfbolz Date: Thu Sep 4 19:27:11 2008 New Revision: 57819 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: add my "dates" Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Thu Sep 4 19:27:11 2008 @@ -14,7 +14,7 @@ Maciej Fijalkowski Armin Rigo Samuele Pedroni -Carl Friedrich Bolz +Carl Friedrich Bolz 3rd Oct - ??? my flat Alexander Schremmer ==================== ============== ============================ From hpk at codespeak.net Thu Sep 4 19:30:18 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 4 Sep 2008 19:30:18 +0200 (CEST) Subject: [pypy-svn] r57820 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904173018.1DF64169E2D@codespeak.net> Author: hpk Date: Thu Sep 4 19:30:17 2008 New Revision: 57820 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: my possible dates Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Thu Sep 4 19:30:17 2008 @@ -10,7 +10,7 @@ Name Arrive/Depart Accomodation ==================== ============== ============================ Antonio Cuni -Holger Krekel +Holger Krekel 5-15 possible ??? Maciej Fijalkowski Armin Rigo Samuele Pedroni From fijal at codespeak.net Thu Sep 4 19:31:55 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 4 Sep 2008 19:31:55 +0200 (CEST) Subject: [pypy-svn] r57821 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904173155.24F66169E99@codespeak.net> Author: fijal Date: Thu Sep 4 19:31:52 2008 New Revision: 57821 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: my possible dates Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Thu Sep 4 19:31:52 2008 @@ -11,7 +11,7 @@ ==================== ============== ============================ Antonio Cuni Holger Krekel 5-15 possible ??? -Maciej Fijalkowski +Maciej Fijalkowski 5-15 possible ??? Armin Rigo Samuele Pedroni Carl Friedrich Bolz 3rd Oct - ??? my flat From antocuni at codespeak.net Thu Sep 4 19:41:31 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 19:41:31 +0200 (CEST) Subject: [pypy-svn] r57822 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904174131.BD47216A048@codespeak.net> Author: antocuni Date: Thu Sep 4 19:41:31 2008 New Revision: 57822 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: my possible dates Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Thu Sep 4 19:41:31 2008 @@ -9,7 +9,7 @@ ==================== ============== ============================ Name Arrive/Depart Accomodation ==================== ============== ============================ -Antonio Cuni +Antonio Cuni 5-15 possible ??? Holger Krekel 5-15 possible ??? Maciej Fijalkowski 5-15 possible ??? Armin Rigo From xoraxax at codespeak.net Thu Sep 4 21:14:16 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 4 Sep 2008 21:14:16 +0200 (CEST) Subject: [pypy-svn] r57825 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904191416.5C7E4169F5A@codespeak.net> Author: xoraxax Date: Thu Sep 4 21:14:14 2008 New Revision: 57825 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: Added my dates. Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Thu Sep 4 21:14:14 2008 @@ -15,7 +15,7 @@ Armin Rigo Samuele Pedroni Carl Friedrich Bolz 3rd Oct - ??? my flat -Alexander Schremmer +Alexander Schremmer 5-15 possible ==================== ============== ============================ People on the following list were present at previous sprints: From hpk at codespeak.net Thu Sep 4 21:20:38 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 4 Sep 2008 21:20:38 +0200 (CEST) Subject: [pypy-svn] r57826 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080904192038.213AA16A065@codespeak.net> Author: hpk Date: Thu Sep 4 21:20:28 2008 New Revision: 57826 Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt Log: drafting accomodation section, tweaking to get it more send-out ready. Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Thu Sep 4 21:20:28 2008 @@ -2,21 +2,23 @@ D?sseldorf PyPy sprint 7-15th October, 2008 ======================================================== -The next PyPy sprint will be held in the Computer Science department of -Heinrich-Heine Universit?t D?sseldorf from the 8th to the 14th of -October. +The pypy team is heading for a new sprint meetup, this time +aiming at a 1.1 release and to tweak pypy to work better with +small devices. The sprint is going to take place at the +Computer Science department of Heinrich-Heine Universit?t +D?sseldorf (Germany). + +XXX times may change to become 5-13th october respectively arrival day: 7th October departure day: 15th October - ------------------------------ Goals and topics of the sprint ------------------------------ There are many possible and interesting sprint topics - -here are the pre-announced topics that some of us -are set to work on: +here are some goals and ideas: * work towards PyPy 1.1 release * make pypy-c fit for small devices like mobile phones @@ -34,13 +36,15 @@ http://stups.cs.uni-duesseldorf.de/anreise/esbahn.php - - - ------------------------------ Accomodation ------------------------------ +We may be able to fit some people into private flats +and otherwise there are several hotels, one recommended +one is the http://www.hotel-hillesheim.de/ where we are +investigating regarding a group rate - if you are interested +please register and tell ASAP. ------------ Registration From antocuni at codespeak.net Thu Sep 4 21:48:08 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Sep 2008 21:48:08 +0200 (CEST) Subject: [pypy-svn] r57827 - in pypy/branch/oo-jit/pypy/jit/codegen/cli: . test Message-ID: <20080904194808.08C3116A068@codespeak.net> Author: antocuni Date: Thu Sep 4 21:48:06 2008 New Revision: 57827 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: add support for arrays, some more tests pass Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Thu Sep 4 21:48:06 2008 @@ -221,6 +221,23 @@ self.meth.il.Emit(OpCodes.Newobj, ctor) self.storeResult() +class OONewArray(Operation): + + def __init__(self, meth, alloctoken, gv_length): + self.meth = meth + self.gv_length = gv_length + self.elemtype = alloctoken.getCliType() + self.clitype = self.elemtype.MakeArrayType() + + def restype(self): + return self.clitype + + def emit(self): + self.gv_length.load(self.meth) + self.meth.il.Emit(OpCodes.Newarr, self.elemtype) + self.storeResult() + + class OOSend(Operation): def __init__(self, meth, gv_self, args_gv, methtoken): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Thu Sep 4 21:48:06 2008 @@ -52,7 +52,12 @@ def getMethodInfo(self): clitype = class2type(self.ooclass) - return clitype.GetMethod(str(self.name)) + res = clitype.GetMethod(str(self.name)) + if res is None: + print 'getMethodInfo --> None' + print 'clitype =', clitype.get_FullName() + print 'name =', self.name + return res def getReturnType(self): methodinfo = self.getMethodInfo() @@ -70,14 +75,37 @@ class StaticMethodToken(MethToken): - def __init__(self, ooclass, name): - self.ooclass = ooclass - self.name = name def emit_call(self, il): methodinfo = self.getMethodInfo() il.Emit(OpCodes.Call, methodinfo) +class ArrayMethodToken(MethToken): + + def __init__(self, name, itemclass): + self.name = name + self.itemclass = itemclass + + def getReturnType(self): + if self.name == 'll_getitem_fast': + return class2type(self.itemclass) + elif self.name == 'll_setitem_fast': + return class2type(cVoid) # ??? + elif self.name == 'll_length': + return class2type(cInt32) + else: + assert False + + def emit_call(self, il): + itemtype = class2type(self.itemclass) + if self.name == 'll_getitem_fast': + il.Emit(OpCodes.Ldelem, itemtype) + elif self.name == 'll_setitem_fast': + il.Emit(OpCodes.Stelem, itemtype) + elif self.name == 'll_length': + il.Emit(OpCodes.Ldlen) + else: + assert False class AllocToken: def __init__(self, ooclass): @@ -345,6 +373,9 @@ def methToken(TYPE, methname): if TYPE is ootype.String: return StaticMethodToken(cpypyString, methname) + elif isinstance(TYPE, ootype.Array): + itemclass = RCliGenOp.kindToken(TYPE.ITEM) + return ArrayMethodToken(methname, itemclass) else: ooclass = ootype.runtimeClass(TYPE) return VirtualMethToken(ooclass, methname) @@ -695,6 +726,11 @@ self.appendop(op) return op.gv_res() + def genop_oonewarray(self, alloctoken, gv_length): + op = ops.OONewArray(self.meth, alloctoken, gv_length) + self.appendop(op) + return op.gv_res() + def enter_next_block(self, args_gv): seen = {} for i in range(len(args_gv)): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 4 21:48:06 2008 @@ -89,13 +89,17 @@ check_count_depth = False # see test_self_referential_structures def test_known_nonzero(self): - py.test.skip("Involves promotion") + py.test.skip("replay: NotImplementedError") - test_debug_assert_ptr_nonzero = skip - test_indirect_red_call = skip - test_indirect_red_call_with_exc = skip - test_indirect_gray_call = skip - test_indirect_residual_red_call = skip + def test_debug_assert_ptr_nonzero(self): + py.test.skip("replay: NotImplementedError") + + def test_indirect_red_call_with_exc(self): + py.test.skip("replay: NotImplementedError") + + def test_indirect_gray_call(self): + py.test.skip('mono 1.2 crashes, try again with a newer version') + test_constant_indirect_red_call = skip test_constant_indirect_red_call_no_result = skip test_indirect_sometimes_residual_pure_red_call = skip From antocuni at codespeak.net Fri Sep 5 09:54:58 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 5 Sep 2008 09:54:58 +0200 (CEST) Subject: [pypy-svn] r57829 - pypy/branch/oo-jit/pypy/doc/config Message-ID: <20080905075458.7AC1A169F10@codespeak.net> Author: antocuni Date: Fri Sep 5 09:54:56 2008 New Revision: 57829 Added: pypy/branch/oo-jit/pypy/doc/config/translation.jitbackend.txt (contents, props changed) Log: a doc for the new option Added: pypy/branch/oo-jit/pypy/doc/config/translation.jitbackend.txt ============================================================================== --- (empty file) +++ pypy/branch/oo-jit/pypy/doc/config/translation.jitbackend.txt Fri Sep 5 09:54:56 2008 @@ -0,0 +1 @@ +Which JIT backend to use From cfbolz at codespeak.net Fri Sep 5 10:13:00 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 5 Sep 2008 10:13:00 +0200 (CEST) Subject: [pypy-svn] r57830 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080905081300.9F0E6169F46@codespeak.net> Author: cfbolz Date: Fri Sep 5 10:12:58 2008 New Revision: 57830 Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt Log: some hotels that Claudia (Michael's secretary) recommended. Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Fri Sep 5 10:12:58 2008 @@ -46,6 +46,26 @@ investigating regarding a group rate - if you are interested please register and tell ASAP. +There are also some hotels that are used by visitors to the university and are +quite close to the uni. None of us have been at them, so we have no experience +of how good they are. + +1. Hotel Bl?ttler +http://www.hotel-blaettler.com/ + +2. Hotel Mooren +http://www.hotel-haus-mooren.de/ + +3. Hotel Europa +can't seem to find a website + +4. Schloss Mickeln +This one is the guest house of the university in a very beautiful castle. +Supposed to be extremely nice, but also expensive. +http://www.uni-duesseldorf.de/home/gaeste/schlossmickeln + + + ------------ Registration ------------ From antocuni at codespeak.net Fri Sep 5 12:09:04 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 5 Sep 2008 12:09:04 +0200 (CEST) Subject: [pypy-svn] r57836 - pypy/branch/oo-jit/pypy/rpython/test Message-ID: <20080905100904.EB62F169FA4@codespeak.net> Author: antocuni Date: Fri Sep 5 12:09:03 2008 New Revision: 57836 Modified: pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py Log: a test that shows a problem with r57808. Not sure if we should revert the revision or fix the places that rely on the old behaviour Modified: pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py Fri Sep 5 12:09:03 2008 @@ -1594,6 +1594,17 @@ self.interpret(f, [int]) + def test_specialize_singlefrozenpbc(self): + py.test.skip("r57808 makes this test fail, and llgraph tests rely on this") + from pypy.rlib.objectmodel import specialize + @specialize.arg(0) + def revealconst(T, x): + return lltype.cast_primitive(T, value) + def fn(x): + return revealconst(lltype.Signed, x) + res = self.interpret(fn, [42], backendopt=False) + assert res == 42 + class TestLLtype(BaseTestRPBC, LLRtypeMixin): pass From antocuni at codespeak.net Fri Sep 5 14:08:23 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 5 Sep 2008 14:08:23 +0200 (CEST) Subject: [pypy-svn] r57842 - pypy/branch/oo-jit/pypy/translator/jvm Message-ID: <20080905120823.CC4A5169F2F@codespeak.net> Author: antocuni Date: Fri Sep 5 14:08:22 2008 New Revision: 57842 Modified: pypy/branch/oo-jit/pypy/translator/jvm/cmpopcodes.py pypy/branch/oo-jit/pypy/translator/jvm/opcodes.py Log: ooisnull, oononnull and oois are handled by cmpopcodes.py, so they are not needed in opcodes.py Modified: pypy/branch/oo-jit/pypy/translator/jvm/cmpopcodes.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/jvm/cmpopcodes.py (original) +++ pypy/branch/oo-jit/pypy/translator/jvm/cmpopcodes.py Fri Sep 5 14:08:22 2008 @@ -1,6 +1,6 @@ from pypy.translator.jvm.typesystem import \ IFLT, IFLE, IFEQ, IFNE, IFGT, IFGE, \ - IFNONNULL, IF_ACMPEQ, GOTO, ICONST, \ + IFNULL, IFNONNULL, IF_ACMPEQ, GOTO, ICONST, \ DCONST_0, DCMPG, LCONST_0, LCMP, \ IF_ICMPLT, IF_ICMPLE, IF_ICMPEQ, IF_ICMPNE, IF_ICMPGT, IF_ICMPGE, \ PYPYUINTCMP, PYPYULONGCMP @@ -52,6 +52,7 @@ # Double operand entries: 'oononnull': [IFNONNULL], + 'ooisnull': [IFNULL], 'oois': [IF_ACMPEQ], 'unichar_eq': [IF_ICMPEQ], Modified: pypy/branch/oo-jit/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/oo-jit/pypy/translator/jvm/opcodes.py Fri Sep 5 14:08:22 2008 @@ -77,9 +77,6 @@ 'oosend': [JvmCallMethod, StoreResult], 'ooupcast': DoNothing, 'oodowncast': [DownCast, StoreResult], - 'oois': 'ref_is_eq', - 'ooisnull': 'is_null', - 'oononnull': 'is_not_null', 'instanceof': [CastTo, StoreResult], 'subclassof': [PushAllArgs, jvm.SWAP, jvm.CLASSISASSIGNABLEFROM, StoreResult], 'ooidentityhash': [PushAllArgs, jvm.OBJHASHCODE, StoreResult], From hpk at codespeak.net Fri Sep 5 14:12:29 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 5 Sep 2008 14:12:29 +0200 (CEST) Subject: [pypy-svn] r57843 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080905121229.42BA916A060@codespeak.net> Author: hpk Date: Fri Sep 5 14:12:27 2008 New Revision: 57843 Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt pypy/extradoc/sprintinfo/october-2008/people.txt Log: * trying to finalize announcement. * fixing my date for now Modified: pypy/extradoc/sprintinfo/october-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/announcement.txt Fri Sep 5 14:12:27 2008 @@ -1,40 +1,26 @@ ======================================================== -D?sseldorf PyPy sprint 7-15th October, 2008 +D?sseldorf PyPy sprint 5-13th October, 2008 ======================================================== -The pypy team is heading for a new sprint meetup, this time -aiming at a 1.1 release and to tweak pypy to work better with -small devices. The sprint is going to take place at the -Computer Science department of Heinrich-Heine Universit?t -D?sseldorf (Germany). +The PyPy team is heading for a new sprint meetup, this time +aiming at a 1.1 release and to work on integrating PyPy better +with small devices. We are also open to other sprint topics! -XXX times may change to become 5-13th october respectively +The sprint is going to take place at the Computer Science +department of Heinrich-Heine Universit?t D?sseldorf (Germany). -arrival day: 7th October -departure day: 15th October +Infos in a nutshell: ------------------------------- -Goals and topics of the sprint ------------------------------- - -There are many possible and interesting sprint topics - -here are some goals and ideas: - -* work towards PyPy 1.1 release -* make pypy-c fit for small devices like mobile phones -* try out Python programs and fix them or fix PyPy or fix performance bottlenecks -* other interesting stuff that you would like to work on ...;-) - ------------------------------- -Getting there ------------------------------- +arrival day: Sunday, 5th October +departure day: Monday, 13th October -The sprint will take place in a seminar room of the computer science -department. It is in the building 25.12 of the university campus, second -floor. For travel instructions see +sprint days: 6-12th october with breaks - http://stups.cs.uni-duesseldorf.de/anreise/esbahn.php +Location: Seminar room of the computer science department, situated +in the building 25.12 of the university campus, second +floor. For travel instructions see +http://stups.cs.uni-duesseldorf.de/anreise/esbahn.php ------------------------------ Accomodation @@ -43,8 +29,8 @@ We may be able to fit some people into private flats and otherwise there are several hotels, one recommended one is the http://www.hotel-hillesheim.de/ where we are -investigating regarding a group rate - if you are interested -please register and tell ASAP. +investigating regarding a cheaper group rate. +Please register ASAP if you want to benefit. There are also some hotels that are used by visitors to the university and are quite close to the uni. None of us have been at them, so we have no experience @@ -65,6 +51,14 @@ http://www.uni-duesseldorf.de/home/gaeste/schlossmickeln +----------------------- +Funding your travel +----------------------- + +We have a limited budget for funding travels. If you want to +participate but need travel funding please register and send +a note to the pypy-sprint mailing list. We'll see what we can +do to help. ------------ Registration @@ -76,9 +70,10 @@ Please register by adding yourself on the following list (via svn): - http://codespeak.net/svn/pypy/extradoc/sprintinfo/post-ep2008/people.txt + http://codespeak.net/svn/pypy/extradoc/sprintinfo/october-2008/people.txt -or on the pypy-sprint mailing list if you do not yet have check-in rights: +or announceyourself on the pypy-sprint mailing list if you do not +yet have check-in rights: http://codespeak.net/mailman/listinfo/pypy-sprint Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Fri Sep 5 14:12:27 2008 @@ -1,5 +1,5 @@ -People coming to the PyPy sprint 7-14th September +People coming to the PyPy sprint 5-13th September ===================================================================== People who have a ``?`` in their arrive/depart or accomodation @@ -10,7 +10,7 @@ Name Arrive/Depart Accomodation ==================== ============== ============================ Antonio Cuni 5-15 possible ??? -Holger Krekel 5-15 possible ??? +Holger Krekel 5-13th interested in sharing room Maciej Fijalkowski 5-15 possible ??? Armin Rigo Samuele Pedroni From fijal at codespeak.net Fri Sep 5 15:19:44 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 5 Sep 2008 15:19:44 +0200 (CEST) Subject: [pypy-svn] r57858 - pypy/branch/cross-compilation/pypy/doc/discussion Message-ID: <20080905131944.296E316A0A8@codespeak.net> Author: fijal Date: Fri Sep 5 15:19:42 2008 New Revision: 57858 Added: pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt (contents, props changed) Log: a bit random notes how to benchmark memory usage Added: pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt ============================================================================== --- (empty file) +++ pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt Fri Sep 5 15:19:42 2008 @@ -0,0 +1,30 @@ +Some random notes about memory-saving GCs +========================================= + +Possibilities: + +* squeak-like mark-compact collector + +* deferred refcounting strategy + +* deferred refcounting with a nursery + +Tests: + +We need to decide size of tests. ie how much interpreter size +matters. + +Also we need to decide what exactly we mean by "memory footprint". +avg, min or max size? + +Also, suggestions include: + +* allocate objects all the time, but total number of objects stay constant + +* allocate objects in bursts and immediately throw them away + +* have small number of live objects and from time to time burst allocate + ton of them and throw them away + +I think we need some kind of graph which shows how exactly this grows with +number of objects. From arigo at codespeak.net Fri Sep 5 17:29:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Sep 2008 17:29:46 +0200 (CEST) Subject: [pypy-svn] r57861 - in pypy/branch/oo-jit/dotviewer: . test Message-ID: <20080905152946.21B8E16A0A7@codespeak.net> Author: arigo Date: Fri Sep 5 17:29:42 2008 New Revision: 57861 Added: pypy/branch/oo-jit/dotviewer/test/test_graphparse.py (contents, props changed) Modified: pypy/branch/oo-jit/dotviewer/graphparse.py Log: A fix in the graph viewer. A link did not work (the word remained non-red) if that word only appeared at the beginning of the lines of a multiline node. Modified: pypy/branch/oo-jit/dotviewer/graphparse.py ============================================================================== --- pypy/branch/oo-jit/dotviewer/graphparse.py (original) +++ pypy/branch/oo-jit/dotviewer/graphparse.py Fri Sep 5 17:29:42 2008 @@ -5,7 +5,9 @@ import os, sys, re import msgstruct -re_nonword = re.compile(r'([^0-9a-zA-Z_.]+)') +# a "nonword" is a separator between words. A sequence of '\' and a letter +# is considered as a separator too. +re_nonword = re.compile(r'((?:[^0-9a-zA-Z_.\\]|\\[a-z])+)') re_plain = re.compile(r'graph [-0-9.]+ [-0-9.]+ [-0-9.]+$', re.MULTILINE) re_digraph = re.compile(r'\b(graph|digraph)\b', re.IGNORECASE) Added: pypy/branch/oo-jit/dotviewer/test/test_graphparse.py ============================================================================== --- (empty file) +++ pypy/branch/oo-jit/dotviewer/test/test_graphparse.py Fri Sep 5 17:29:42 2008 @@ -0,0 +1,7 @@ +from dotviewer.graphparse import * + +# XXX mostly empty + +def test_re_nonword(): + words = [s for s in re_nonword.split('abc() def2 \\lghi jkl\\l') if s] + assert words == ['abc', '() ', 'def2', ' \\l', 'ghi', ' ', 'jkl', '\\l'] From arigo at codespeak.net Fri Sep 5 17:32:00 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 5 Sep 2008 17:32:00 +0200 (CEST) Subject: [pypy-svn] r57862 - in pypy/branch/oo-jit/dotviewer: . test Message-ID: <20080905153200.A5C4E16A0A1@codespeak.net> Author: arigo Date: Fri Sep 5 17:31:59 2008 New Revision: 57862 Modified: pypy/branch/oo-jit/dotviewer/graphparse.py pypy/branch/oo-jit/dotviewer/test/test_graphparse.py Log: Tweak the regexp. Modified: pypy/branch/oo-jit/dotviewer/graphparse.py ============================================================================== --- pypy/branch/oo-jit/dotviewer/graphparse.py (original) +++ pypy/branch/oo-jit/dotviewer/graphparse.py Fri Sep 5 17:31:59 2008 @@ -5,9 +5,9 @@ import os, sys, re import msgstruct -# a "nonword" is a separator between words. A sequence of '\' and a letter -# is considered as a separator too. -re_nonword = re.compile(r'((?:[^0-9a-zA-Z_.\\]|\\[a-z])+)') +# a "nonword" is a separator between words. A sequence of '\' followed by +# any character is considered as a single separator too. +re_nonword = re.compile(r'((?:[^0-9a-zA-Z_.\\]|\\.)+)') re_plain = re.compile(r'graph [-0-9.]+ [-0-9.]+ [-0-9.]+$', re.MULTILINE) re_digraph = re.compile(r'\b(graph|digraph)\b', re.IGNORECASE) Modified: pypy/branch/oo-jit/dotviewer/test/test_graphparse.py ============================================================================== --- pypy/branch/oo-jit/dotviewer/test/test_graphparse.py (original) +++ pypy/branch/oo-jit/dotviewer/test/test_graphparse.py Fri Sep 5 17:31:59 2008 @@ -5,3 +5,5 @@ def test_re_nonword(): words = [s for s in re_nonword.split('abc() def2 \\lghi jkl\\l') if s] assert words == ['abc', '() ', 'def2', ' \\l', 'ghi', ' ', 'jkl', '\\l'] + words = [s for s in re_nonword.split('abc\\\\def') if s] + assert words == ['abc', '\\\\', 'def'] From mwh at codespeak.net Sat Sep 6 08:47:53 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 6 Sep 2008 08:47:53 +0200 (CEST) Subject: [pypy-svn] r57869 - pypy/extradoc/talk/osdc2008 Message-ID: <20080906064753.6DB8E16A071@codespeak.net> Author: mwh Date: Sat Sep 6 08:47:50 2008 New Revision: 57869 Added: pypy/extradoc/talk/osdc2008/ Log: add a directory for my talk at OSDC 2008 (december 2-5 in Sydney) From mwh at codespeak.net Sat Sep 6 09:06:00 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 6 Sep 2008 09:06:00 +0200 (CEST) Subject: [pypy-svn] r57870 - pypy/extradoc/talk/osdc2008 Message-ID: <20080906070600.4B78716A0D3@codespeak.net> Author: mwh Date: Sat Sep 6 09:05:59 2008 New Revision: 57870 Added: pypy/extradoc/talk/osdc2008/paper.txt Log: a start, i guess Added: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/osdc2008/paper.txt Sat Sep 6 09:05:59 2008 @@ -0,0 +1,45 @@ +The PyPy Project And You +======================== + +Abstract +-------- + + PyPy aims to provide a common translation and support framework for + producing implementations of dynamic languages, emphasising a clean + separation between language specification and implementation + aspects and a compliant, flexible and fast implementation of the + Python Language using the above framework to enable new advanced + features without having to encode low level details into it. + + This paper gives a brief overview of the motivation and status of + PyPy and attempts to explain why anyone who cares about the + implementation of dynamic languages should be interested in it. + +What is PyPy? +------------- + + + +Motivation +---------- + + +PyPy's Big Idea And The PyPy Meta-Platform +------------------------------------------ + + +The LxOxP problem +----------------- + + +Status +------ + + +An Example Translation +---------------------- + + +The Future +---------- + From fijal at codespeak.net Sat Sep 6 09:59:41 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 09:59:41 +0200 (CEST) Subject: [pypy-svn] r57871 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080906075941.E3E6A169F1D@codespeak.net> Author: fijal Date: Sat Sep 6 09:59:40 2008 New Revision: 57871 Modified: pypy/extradoc/talk/pycon-uk-2008/motivation.txt Log: minor update Modified: pypy/extradoc/talk/pycon-uk-2008/motivation.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/motivation.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/motivation.txt Sat Sep 6 09:59:40 2008 @@ -7,7 +7,8 @@ - a framework for writing dynamic languages -Today we will focus on the latter. +Today we will focus on the latter, +former is discussed on another talk. Agenda From fijal at codespeak.net Sat Sep 6 10:05:27 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 10:05:27 +0200 (CEST) Subject: [pypy-svn] r57873 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080906080527.50BC9169EBA@codespeak.net> Author: fijal Date: Sat Sep 6 10:05:26 2008 New Revision: 57873 Modified: pypy/extradoc/talk/pycon-uk-2008/author.latex Log: use official spelling Modified: pypy/extradoc/talk/pycon-uk-2008/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/author.latex Sat Sep 6 10:05:26 2008 @@ -1,7 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy and The Art of Generating VMs]{PyPy and The Art of Generating Virtual Machines} -\author[A. Cuni, M. Fijalkowski]{Antonio Cuni \and Maciek Fijalkowski\\Merlinux GmbH} +\author[A. Cuni, M. Fijalkowski]{Antonio Cuni \and Maciej Fijalkowski\\Merlinux GmbH} \institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} \date{September 13 2008} From fijal at codespeak.net Sat Sep 6 10:09:19 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 10:09:19 +0200 (CEST) Subject: [pypy-svn] r57874 - pypy/extradoc/talk/pycon-uk-2008 Message-ID: <20080906080919.0B87E169ED7@codespeak.net> Author: fijal Date: Sat Sep 6 10:09:18 2008 New Revision: 57874 Modified: pypy/extradoc/talk/pycon-uk-2008/makepdf pypy/extradoc/talk/pycon-uk-2008/motivation.txt Log: remove status update, it's outdated and belongs to the other talk Modified: pypy/extradoc/talk/pycon-uk-2008/makepdf ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/makepdf (original) +++ pypy/extradoc/talk/pycon-uk-2008/makepdf Sat Sep 6 10:09:18 2008 @@ -6,7 +6,7 @@ # WARNING: to work, it needs this patch for docutils # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 -rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt pypy-vm.txt pypy-vm.latex || exit +python rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt pypy-vm.txt pypy-vm.latex || exit sed 's/\\date{}/\\input{author.latex}/' -i pypy-vm.latex || exit pdflatex pypy-vm.latex || exit Modified: pypy/extradoc/talk/pycon-uk-2008/motivation.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/motivation.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/motivation.txt Sat Sep 6 10:09:18 2008 @@ -23,34 +23,6 @@ * (demo) -PyPy status update (1) -====================== - -- JVM backend completed, pypy-jvm - -- some new GCs, much faster than ever - -- ctypes for PyPy - -- JIT refactoring, needed to make the JIT production-ready - -- improved .NET integration for pypy-cli - -- new blog: http://morepypy.blogspot.com - - -PyPy status update (2) -====================== - -- various performance improvements - -- slighly slower than CPython on pystone (10-20%) - -- but faster on richards (20-24%) - -- less than 2x slower on other benchmarks - - Interpreters ================== From hpk at codespeak.net Sat Sep 6 10:25:16 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 6 Sep 2008 10:25:16 +0200 (CEST) Subject: [pypy-svn] r57875 - in pypy/extradoc/talk/pycon-uk-2008: . jit Message-ID: <20080906082516.019B5169F15@codespeak.net> Author: hpk Date: Sat Sep 6 10:25:15 2008 New Revision: 57875 Added: pypy/extradoc/talk/pycon-uk-2008/jit/ pypy/extradoc/talk/pycon-uk-2008/jit/author.latex - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/author.latex pypy/extradoc/talk/pycon-uk-2008/jit/beamerdefs.txt - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/beamerdefs.txt pypy/extradoc/talk/pycon-uk-2008/jit/flowgraph.png - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/flowgraph.png pypy/extradoc/talk/pycon-uk-2008/jit/makepdf - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/makepdf pypy/extradoc/talk/pycon-uk-2008/jit/motivation.txt - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/motivation.txt pypy/extradoc/talk/pycon-uk-2008/jit/overview1.png - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/overview1.png pypy/extradoc/talk/pycon-uk-2008/jit/overview2.png - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/overview2.png pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.pdf.info - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/pypy-vm.pdf.info pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.txt - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/pypy-vm.txt pypy/extradoc/talk/pycon-uk-2008/jit/rainbow.png - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/rainbow.png pypy/extradoc/talk/pycon-uk-2008/jit/stylesheet.latex - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/stylesheet.latex pypy/extradoc/talk/pycon-uk-2008/jit/technical.txt - copied unchanged from r57874, pypy/extradoc/talk/pycon-uk-2008/technical.txt Removed: pypy/extradoc/talk/pycon-uk-2008/author.latex pypy/extradoc/talk/pycon-uk-2008/beamerdefs.txt pypy/extradoc/talk/pycon-uk-2008/flowgraph.png pypy/extradoc/talk/pycon-uk-2008/makepdf pypy/extradoc/talk/pycon-uk-2008/motivation.txt pypy/extradoc/talk/pycon-uk-2008/overview1.png pypy/extradoc/talk/pycon-uk-2008/overview2.png pypy/extradoc/talk/pycon-uk-2008/pypy-vm.pdf.info pypy/extradoc/talk/pycon-uk-2008/pypy-vm.txt pypy/extradoc/talk/pycon-uk-2008/rainbow.png pypy/extradoc/talk/pycon-uk-2008/stylesheet.latex pypy/extradoc/talk/pycon-uk-2008/technical.txt Log: move jit talk into its own dir From mwh at codespeak.net Sat Sep 6 10:32:12 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 6 Sep 2008 10:32:12 +0200 (CEST) Subject: [pypy-svn] r57876 - pypy/extradoc/talk/osdc2008 Message-ID: <20080906083212.2577D169F6F@codespeak.net> Author: mwh Date: Sat Sep 6 10:32:12 2008 New Revision: 57876 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: more words Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sat Sep 6 10:32:12 2008 @@ -1,6 +1,8 @@ The PyPy Project And You ======================== +:author: Michael Hudson + Abstract -------- @@ -15,14 +17,65 @@ PyPy and attempts to explain why anyone who cares about the implementation of dynamic languages should be interested in it. + What is PyPy? ------------- +PyPy is: + + * An implementation of Python in Python + * A very flexible compiler framework (with some features that are + especially useful for implementing dynamic languages) + * An open source project (MIT license) + * A lot of fun! + +PyPy was also: + + * A Structured Targeted REsearch Proposal (STREP), partly funded by + the European Union + * The funding period ended at the end of March 2007 + * In May 2007 we had our final technical review, and "[PyPy] fully + achieved its objectives and tech goals and has even exceeded + expectations" + +It has now gone back to being a project that people mostly work on in +their spare time, with some people making PyPy the topic of Masters' +theses and similar. Google's Open Source Programs Office also +sponsored some work on getting real world applications running PyPy +(more on that later). Motivation ---------- +The beginnings PyPy can be traced to the first EuroPython conference +in 2002, where some of the people who ultimately became involved in +the project met in person for the first time, and realized they had a +common interest: we were all interested in modifying and extending the +CPython *implementation* of the Python language, rather than the +language itself (which was and still is the usual topic of discussion +on the python-dev list). + +These people included: + + * Armin Rigo, the author of Pysco, the well-known Python accelerator. + * Christian Tismer, the author of the "Stackless" variant of CPython, + which adds coroutines and other related forms of non-traditional + control flow to the language. + * Samuele Pedroni, at the time, the maintainer of Jython. + * Myself, at the time one of the more active CPython developers. + +While in many ways there's nothing deeply wrong about CPython, which +is written in a straightforward style in clear C, there are some +inherent issues: + + * Being written in C, it is hard to port to the JVM or CLI. + * Extensions to the language like Stackless have to be painfully kept + up to date with language changes as they are made. + * Some implementation decisions, such as using reference counting for + memory management or a Global Interpreter Lock, are very hard to + change by now. + PyPy's Big Idea And The PyPy Meta-Platform ------------------------------------------ From hpk at codespeak.net Sat Sep 6 10:35:16 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 6 Sep 2008 10:35:16 +0200 (CEST) Subject: [pypy-svn] r57877 - in pypy/extradoc/talk/pycon-uk-2008: jit status Message-ID: <20080906083516.1C86716A022@codespeak.net> Author: hpk Date: Sat Sep 6 10:35:14 2008 New Revision: 57877 Added: pypy/extradoc/talk/pycon-uk-2008/status/ pypy/extradoc/talk/pycon-uk-2008/status/author.latex - copied, changed from r57875, pypy/extradoc/talk/pycon-uk-2008/jit/author.latex pypy/extradoc/talk/pycon-uk-2008/status/makepdf - copied, changed from r57875, pypy/extradoc/talk/pycon-uk-2008/jit/makepdf pypy/extradoc/talk/pycon-uk-2008/status/status.txt (contents, props changed) pypy/extradoc/talk/pycon-uk-2008/status/stylesheet.latex - copied unchanged from r57875, pypy/extradoc/talk/pycon-uk-2008/jit/stylesheet.latex Modified: pypy/extradoc/talk/pycon-uk-2008/jit/makepdf Log: forking a status directory fixing makepdf Modified: pypy/extradoc/talk/pycon-uk-2008/jit/makepdf ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/jit/makepdf (original) +++ pypy/extradoc/talk/pycon-uk-2008/jit/makepdf Sat Sep 6 10:35:14 2008 @@ -6,7 +6,7 @@ # WARNING: to work, it needs this patch for docutils # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 -python rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt pypy-vm.txt pypy-vm.latex || exit +rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt pypy-vm.txt pypy-vm.latex || exit sed 's/\\date{}/\\input{author.latex}/' -i pypy-vm.latex || exit pdflatex pypy-vm.latex || exit Copied: pypy/extradoc/talk/pycon-uk-2008/status/author.latex (from r57875, pypy/extradoc/talk/pycon-uk-2008/jit/author.latex) ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/jit/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/author.latex Sat Sep 6 10:35:14 2008 @@ -1,7 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy and The Art of Generating VMs]{PyPy and The Art of Generating Virtual Machines} -\author[A. Cuni, M. Fijalkowski]{Antonio Cuni \and Maciej Fijalkowski\\Merlinux GmbH} +\author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski\\Merlinux GmbH} \institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} \date{September 13 2008} Copied: pypy/extradoc/talk/pycon-uk-2008/status/makepdf (from r57875, pypy/extradoc/talk/pycon-uk-2008/jit/makepdf) ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/jit/makepdf (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/makepdf Sat Sep 6 10:35:14 2008 @@ -6,7 +6,8 @@ # WARNING: to work, it needs this patch for docutils # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 -python rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt pypy-vm.txt pypy-vm.latex || exit -sed 's/\\date{}/\\input{author.latex}/' -i pypy-vm.latex || exit -pdflatex pypy-vm.latex || exit +BASE=status +rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt $BASE.txt $BASE.latex || exit +sed 's/\\date{}/\\input{author.latex}/' -i $BASE.latex || exit +pdflatex $BASE.latex || exit Added: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 6 10:35:14 2008 @@ -0,0 +1,242 @@ +====================== +PyPy status talk +====================== + +What this talk is about +======================= + +* more details about recent developments + +* our plans for the near future + +* how we're going to achieve them + +* readers of our blog might know many of those things + +Production ready +===================== + +* we worked a lot on running + existing applications on top of PyPy + +* sometimes requiring to change applications slightly + +* especially refcounting details tend to be a problem + +:: + + open('xxx', 'w').write('stuff') + +CTypes +====== + +* official way to have bindings to + external (C) libraries for PyPy + +* slow, but getting better + +* can handle i.e. pysqlite-ctypes, pyglet, pymunk or Sole Scion + +* ctypes is getting better as a side effect: + + * errno handling + + * bugfixes + + * helper libraries + +* part of google sponsoring + +* demo + +CTypes configure +================ + +* our own small addition to general + CTypes usefulness + +* invokes C compiler for small details + +* can handle #defines, types, structure layout + etc. + +Sqlite +====== + +* part of cpython stdlib since 2.5 + +* we use Gerhard Haering's CTypes version + +* works reasonably well after some fixes + +Django +====== + +* we run (almost) unmodified Django + +* only sqlite DB backend for now + +* cooperation with Django people to make sure + that Django 1.0 works with PyPy + +* http://www.djangoproject.com/ + +Pylons +====== + +* worked almost out of the box once eggs + were working (1 day) + +* no SQLAlchemy yet, obscure problems + ahead + +* unmodified passes all tests + +* http://pylonshq.com/ + +Twisted & Nevow +=============== + +* twisted have some glitches, but mostly works + +* nevow works + +* we don't support PyCrypto nor PyOpenSSL and we + won't anytime soon (if nobody contributes CTypes or rpython + versions) + +* http://twistedmatrix.com/ + +Other software +============== + +* BitTorrent + +* PyPy translation toolchain + +* various smaller things, templating engines, + most pure-python software + +Obscure details that people rely on +=================================== + +* non-string keys in __dict__ of types + +* exact naming of a list comprehension variable + +* relying on untested and undocumented private stuff + (zipimport._zip_directory_cache) + +* exact message matching in exception catching + code + +* refcounting details + +Conclusion on Compatibility +============================ + +* lessons learned: There is no feature obscure enough for people + not to rely on it. But PyPy can usually mimick CPython sufficiently. + +* pypy-c probably most compatible to CPython Interpreter + +* main blocker for running apps will be missing external modules + +Speed - comparison with CPython +=============================== + +* we're something between 0.8-2x slower than + CPython on various benchmarks. + +* gcbench - 0.8 (because of our faster GC) + +* steady but slow progress + +* we hope for our JIT to be a huge leap ahead + +Speed - decent GCs +================== + +* faster than refcounting + +* better handling of unusual patterns + +* troubles with for example communication with C + +* details on different talk + +Speed - JIT generator +===================== + +* not ready yet! + +* will be super fast + +* prolog prototype + +* psyco is a nice proof that this approach + would work + +Other backends +============== + +* PyPy-jvm runs! + +* more integration between pypy-cli and .NET + +* general speed improvements + +* both backends are progressing - very slowly though + +* contributors wanted! + +Sandboxing +========== + +* fully sandboxed python interpreter + +* all external calls to C goes via another + python process + +* special library for making custom + policies + +.. image:: sandboxed.png + :scale: 30 + :align: center + +A lot of cleanups +================= + +* got rid of semi-cool semi-working + proofs of concept + +* examples: CPython extension compiler, + rctypes + +* reduced code size + +* allowed us to progress forward into + advanced features + +* included sprint dedicated to cleanup + (on which noone was allowed to add features) + +Plans +===== + +* more JIT - faster Python + +* support full CPython's stdlib + +* aim for more funding in pushing pypy + forward (more at the next talk) + +Links +===== + +* http://codespeak.net/pypy + +* http://morepypy.blogspot.com + +* this talk is already online From mwh at codespeak.net Sat Sep 6 10:56:20 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sat, 6 Sep 2008 10:56:20 +0200 (CEST) Subject: [pypy-svn] r57878 - pypy/extradoc/talk/osdc2008 Message-ID: <20080906085620.AB033169EEB@codespeak.net> Author: mwh Date: Sat Sep 6 10:56:18 2008 New Revision: 57878 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: more copy-paste-adaptation of my slides Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sat Sep 6 10:56:18 2008 @@ -80,10 +80,57 @@ PyPy's Big Idea And The PyPy Meta-Platform ------------------------------------------ + * Take a description of the Python programming language + * Analyze this description, for example: -The LxOxP problem ------------------ + * Decide whether to include stackless- or psyco-like features + * Decide which GC to use + * Decide the target platform + + * Translate to a lower-level, efficient form + +XXX Get copy of that image in here somehow. + +We chose to specify the Python language by writing an implementation +if Python in a restricted subset of Python that is amenable to +analysis. This let us write unit tests for parts of our +specification/implementation before we had the whole thing done, and +also let us test the whole specification/implementation before the code +that analysed the specification/implementation was written. + +The code that does the analysis -- now revealed in its true colours to +basically be a compiler for this restricted subset of Python -- is +generally referred to as the 'translator. It is unlike most compilers +takes as input live Python objects (as opposed to source code). It +abstractly interprets the bytecode of functions to produce flow +graphs. Further layers of abstract interpretation perform more +analysis and gradually reduce the level of abstraction before finally +C or other source code is generated + +The *LxOxP* problem +------------------- + +We'd written this compiler framework, with only one expected +non-trivial input (our Python interpreter). We realized that it would +be suitable for implementations of other dynamically-typed programming +languages. Now have implementations of Prolog, JavaScript and Scheme +(to varying extents) + +This leads to one of PyPy's meta-goals, ameliorating the so-called +LxOxP problem: given + + * L dynamic languages + * O target platforms + * P implementation decisions + +we don't want to have to write LxOxP different interpreters by hand. +PyPy aims to reduce this to an L+O+P problem: + + * Implement L language front-ends + * Write backends for O platforms + * Take P implementation decisions +Then let the magic of PyPy(TM) tie it all together :-) Status ------ From hpk at codespeak.net Sat Sep 6 11:11:12 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 6 Sep 2008 11:11:12 +0200 (CEST) Subject: [pypy-svn] r57879 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080906091112.F39E7169E8F@codespeak.net> Author: hpk Date: Sat Sep 6 11:11:10 2008 New Revision: 57879 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: a first go at updating/changing the status slides inserted some XXX Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 6 11:11:10 2008 @@ -2,19 +2,19 @@ PyPy status talk ====================== +XXX maybe missing: threading improvements? more on GCs? + What this talk is about ======================= -* more details about recent developments - -* our plans for the near future +* tell what Python Interpreter can run today -* how we're going to achieve them +* what we are heading for with the 1.1 release -* readers of our blog might know many of those things +* Questions and Answers -Production ready -===================== +Getting Production ready +========================== * we worked a lot on running existing applications on top of PyPy @@ -114,6 +114,8 @@ * PyPy translation toolchain +* py lib + * various smaller things, templating engines, most pure-python software @@ -132,38 +134,46 @@ * refcounting details +Transition to 2.5 +========================== + +XXX +* SOC project Bruno Gola +* almost complete +* missing more testing, stdlib porting + Conclusion on Compatibility ============================ * lessons learned: There is no feature obscure enough for people - not to rely on it. But PyPy can usually mimick CPython sufficiently. + not to rely on it. -* pypy-c probably most compatible to CPython Interpreter +* pypy-c interpreter probably the most compatible to CPython * main blocker for running apps will be missing external modules Speed - comparison with CPython =============================== -* we're something between 0.8-2x slower than +* we're something between 0.8-4x slower than CPython on various benchmarks. -* gcbench - 0.8 (because of our faster GC) - * steady but slow progress * we hope for our JIT to be a huge leap ahead -Speed - decent GCs -================== +* pypy-c has fastest Interpreter startup -* faster than refcounting +Memory - comparison with CPython +=================================== -* better handling of unusual patterns +* PyPy has pluggable Garbage Collection -* troubles with for example communication with C +* gcbench - 0.8 (because of our faster GCs) -* details on different talk +* better handling of unusual patterns + +* care needed with communication with C Speed - JIT generator ===================== @@ -205,38 +215,34 @@ :scale: 30 :align: center -A lot of cleanups -================= - -* got rid of semi-cool semi-working - proofs of concept - -* examples: CPython extension compiler, - rctypes - -* reduced code size - -* allowed us to progress forward into - advanced features +pypy-c on small devices +=============================== -* included sprint dedicated to cleanup - (on which noone was allowed to add features) +- cross-compilation +- startup time +- security +- RAM usage +- share interpreter state across processes +- pypy approach a very good fit! -Plans -===== +1.1 release goals +=================================== -* more JIT - faster Python +XXX check/review/complete -* support full CPython's stdlib +- compatible to Python 2.5.2 +- well tested on win/linux/osx +- running some major packages unmodified +- easy_install/distutils working +- deliver pypy-c binary installs windows -* aim for more funding in pushing pypy - forward (more at the next talk) +Contact / Q&A +========================== -Links -===== * http://codespeak.net/pypy * http://morepypy.blogspot.com -* this talk is already online +holger krekel, Maciej Fijalkowski + From fijal at codespeak.net Sat Sep 6 11:21:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 11:21:39 +0200 (CEST) Subject: [pypy-svn] r57880 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080906092139.A037716A082@codespeak.net> Author: fijal Date: Sat Sep 6 11:21:37 2008 New Revision: 57880 Added: pypy/extradoc/talk/pycon-uk-2008/status/sandboxed.png (contents, props changed) Log: missing pictutre Added: pypy/extradoc/talk/pycon-uk-2008/status/sandboxed.png ============================================================================== Binary file. No diff available. From pedronis at codespeak.net Sat Sep 6 15:04:14 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 6 Sep 2008 15:04:14 +0200 (CEST) Subject: [pypy-svn] r57882 - in pypy/branch/garden-call-code-2/pypy/interpreter: . test Message-ID: <20080906130414.EDAB4169EB1@codespeak.net> Author: pedronis Date: Sat Sep 6 15:04:07 2008 New Revision: 57882 Modified: pypy/branch/garden-call-code-2/pypy/interpreter/eval.py pypy/branch/garden-call-code-2/pypy/interpreter/function.py pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py Log: Switch away from having fastcall_* methods on PyCode, calling python functions with more than 4 args is now a bit faster. Noisy slow downs in some other corners though. Modified: pypy/branch/garden-call-code-2/pypy/interpreter/eval.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/eval.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/eval.py Sat Sep 6 15:04:07 2008 @@ -62,18 +62,6 @@ def funcrun_obj(self, func, w_obj, args): return self.funcrun(func, args.prepend(w_obj)) - - # a performance hack (see gateway.BuiltinCode1/2/3 and pycode.PyCode) - def fastcall_0(self, space, func): - raise NotImplementedError - def fastcall_1(self, space, func, w1): - raise NotImplementedError - def fastcall_2(self, space, func, w1, w2): - raise NotImplementedError - def fastcall_3(self, space, func, w1, w2, w3): - raise NotImplementedError - def fastcall_4(self, space, func, w1, w2, w3, w4): - raise NotImplementedError class Frame(Wrappable): """A frame is an environment supporting the execution of a code object. Modified: pypy/branch/garden-call-code-2/pypy/interpreter/function.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/function.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/function.py Sat Sep 6 15:04:07 2008 @@ -6,11 +6,14 @@ attribute. """ +from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.eval import Code from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +funccallunrolling = unrolling_iterable(range(4)) + class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, @@ -44,24 +47,40 @@ return self.code def funccall(self, *args_w): # speed hack + from pypy.interpreter import gateway + from pypy.interpreter.pycode import PyCode + code = self.getcode() # hook for the jit nargs = len(args_w) fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: + assert isinstance(code, gateway.BuiltinCode1) return code.fastcall_1(self.space, self, args_w[0]) elif nargs == 2: + assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, args_w[0], args_w[1]) elif nargs == 3: + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, args_w[0], args_w[1], args_w[2]) elif nargs == 4: + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, args_w[0], args_w[1], args_w[2], args_w[3]) + elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: + assert isinstance(code, PyCode) + if nargs < 5: + new_frame = self.space.createframe(code, self.w_func_globals, + self.closure) + for i in funccallunrolling: + if i < nargs: + new_frame.fastlocals_w[i] = args_w[i] + return new_frame.run() elif nargs >= 1 and fast_natural_arity == -1: - from pypy.interpreter import gateway assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], Arguments(self.space, @@ -70,37 +89,33 @@ def funccall_valuestack(self, nargs, frame): # speed hack from pypy.interpreter import gateway + from pypy.interpreter.pycode import PyCode + code = self.getcode() # hook for the jit fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: - from pypy.interpreter.pycode import PyCode - if type(code) is PyCode: - new_frame = self.space.createframe(code, self.w_func_globals, - self.closure) - for i in xrange(nargs): - w_arg = frame.peekvalue(nargs-1-i) - new_frame.fastlocals_w[i] = w_arg - return new_frame.run() - else: - if nargs == 0: - assert isinstance(code, gateway.BuiltinCode0) - return code.fastcall_0(self.space, self) - elif nargs == 1: - assert isinstance(code, gateway.BuiltinCode1) - return code.fastcall_1(self.space, self, frame.peekvalue(0)) - elif nargs == 2: - assert isinstance(code, gateway.BuiltinCode2) - return code.fastcall_2(self.space, self, frame.peekvalue(1), - frame.peekvalue(0)) - elif nargs == 3: - assert isinstance(code, gateway.BuiltinCode3) - return code.fastcall_3(self.space, self, frame.peekvalue(2), - frame.peekvalue(1), frame.peekvalue(0)) - elif nargs == 4: - assert isinstance(code, gateway.BuiltinCode4) - return code.fastcall_4(self.space, self, frame.peekvalue(3), - frame.peekvalue(2), frame.peekvalue(1), - frame.peekvalue(0)) + if nargs == 0: + assert isinstance(code, gateway.BuiltinCode0) + return code.fastcall_0(self.space, self) + elif nargs == 1: + assert isinstance(code, gateway.BuiltinCode1) + return code.fastcall_1(self.space, self, frame.peekvalue(0)) + elif nargs == 2: + assert isinstance(code, gateway.BuiltinCode2) + return code.fastcall_2(self.space, self, frame.peekvalue(1), + frame.peekvalue(0)) + elif nargs == 3: + assert isinstance(code, gateway.BuiltinCode3) + return code.fastcall_3(self.space, self, frame.peekvalue(2), + frame.peekvalue(1), frame.peekvalue(0)) + elif nargs == 4: + assert isinstance(code, gateway.BuiltinCode4) + return code.fastcall_4(self.space, self, frame.peekvalue(3), + frame.peekvalue(2), frame.peekvalue(1), + frame.peekvalue(0)) + elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: + assert isinstance(code, PyCode) + return self._flat_pycall(code, nargs, frame) elif fast_natural_arity == -1 and nargs >= 1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) w_obj = frame.peekvalue(nargs-1) @@ -118,6 +133,15 @@ if isinstance(args, ArgumentsFromValuestack): args.frame = None + def _flat_pycall(self, code, nargs, frame): + # code is a PyCode + new_frame = self.space.createframe(code, self.w_func_globals, + self.closure) + for i in xrange(nargs): + w_arg = frame.peekvalue(nargs-1-i) + new_frame.fastlocals_w[i] = w_arg + return new_frame.run() + def getdict(self): if self.w_func_dict is None: self.w_func_dict = self.space.newdict() Modified: pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/pycode.py Sat Sep 6 15:04:07 2008 @@ -108,7 +108,7 @@ self._args_as_cellvars.append(-1) # pad self._args_as_cellvars[i] = j - self._compute_fastcall() + self._compute_flatcall() co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace @@ -159,51 +159,20 @@ freevars, cellvars, hidden_applevel) _code_new_w = staticmethod(_code_new_w) + + FLATPYCALL = 0x100 - def _compute_fastcall(self): + def _compute_flatcall(self): # Speed hack! self.fast_natural_arity = -99 if self.co_flags & (CO_VARARGS | CO_VARKEYWORDS): return if len(self._args_as_cellvars) > 0: return - - self.fast_natural_arity = self.co_argcount - - def fastcall_0(self, space, w_func): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - return frame.run() - - def fastcall_1(self, space, w_func, w_arg): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg # frame.setfastscope([w_arg]) - return frame.run() - - def fastcall_2(self, space, w_func, w_arg1, w_arg2): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - return frame.run() - - def fastcall_3(self, space, w_func, w_arg1, w_arg2, w_arg3): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - frame.fastlocals_w[2] = w_arg3 - return frame.run() - - def fastcall_4(self, space, w_func, w_arg1, w_arg2, w_arg3, w_arg4): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - frame.fastlocals_w[2] = w_arg3 - frame.fastlocals_w[3] = w_arg4 - return frame.run() + if self.co_argcount > 0xff: + return + + self.fast_natural_arity = PyCode.FLATPYCALL | self.co_argcount def funcrun(self, func, args): frame = self.space.createframe(self, func.w_func_globals, Modified: pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/garden-call-code-2/pypy/interpreter/test/test_function.py Sat Sep 6 15:04:07 2008 @@ -474,7 +474,7 @@ code = PyCode._from_code(self.space, f.func_code) fn = Function(self.space, code, self.space.newdict()) - assert fn.code.fast_natural_arity == i + assert fn.code.fast_natural_arity == i|PyCode.FLATPYCALL if i < 5: def bomb(*args): @@ -488,7 +488,7 @@ check = space.is_true(space.eq(w_res, space.wrap(res))) assert check - def test_fastcall(self): + def test_flatcall(self): space = self.space def f(a): @@ -496,7 +496,7 @@ code = PyCode._from_code(self.space, f.func_code) fn = Function(self.space, code, self.space.newdict()) - assert fn.code.fast_natural_arity == 1 + assert fn.code.fast_natural_arity == 1|PyCode.FLATPYCALL def bomb(*args): assert False, "shortcutting should have avoided this" @@ -515,7 +515,7 @@ assert w_res is w_3 - def test_fastcall_method(self): + def test_flatcall_method(self): space = self.space def f(self, a): @@ -523,7 +523,7 @@ code = PyCode._from_code(self.space, f.func_code) fn = Function(self.space, code, self.space.newdict()) - assert fn.code.fast_natural_arity == 2 + assert fn.code.fast_natural_arity == 2|PyCode.FLATPYCALL def bomb(*args): assert False, "shortcutting should have avoided this" From fijal at codespeak.net Sat Sep 6 17:03:56 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 17:03:56 +0200 (CEST) Subject: [pypy-svn] r57885 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080906150356.1CB2816A0BC@codespeak.net> Author: fijal Date: Sat Sep 6 17:03:53 2008 New Revision: 57885 Added: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (contents, props changed) pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (contents, props changed) Log: Start of a script to measure memory impact of a given process. Added: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- (empty file) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Sat Sep 6 17:03:53 2008 @@ -0,0 +1,14 @@ +#!/usr/bin/env python +import os +import py + +def run_child(name, args): + pid = os.fork() + if not pid: + os.execvp(name, [name] + args) + else: + res = py.process.cmdexec('pmap -d %d' % pid) + return res + +if __name__ == '__main__': + run_child('python', ['-c', 'pass']) Added: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- (empty file) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Sat Sep 6 17:03:53 2008 @@ -0,0 +1,6 @@ + +from pypy.translator.benchmark import bench_mem + +def test_basic(): + res = bench_mem.run_child('python', ['-c', 'pass']) + assert 'python' in res From pedronis at codespeak.net Sat Sep 6 17:51:19 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 6 Sep 2008 17:51:19 +0200 (CEST) Subject: [pypy-svn] r57887 - pypy/branch/garden-call-code-2/pypy/translator/test Message-ID: <20080906155119.0508A169F09@codespeak.net> Author: pedronis Date: Sat Sep 6 17:51:17 2008 New Revision: 57887 Modified: pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py Log: some tests about stack_check and gc transformer root reloading in the stackless and non-stackless case Modified: pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py (original) +++ pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py Sat Sep 6 17:51:17 2008 @@ -1,22 +1,68 @@ from pypy import conftest -from pypy.translator.translator import TranslationContext +from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.transform import insert_ll_stackcheck +from pypy.rpython.memory.gctransform import framework +from pypy.translator.stackless.transform import StacklessTransformer -def test_simple(): - class A(object): - def __init__(self, n): - self.n = n - - def f(a): - x = A(a.n+1) - if x.n == 10: - return - f(x) +def _follow_path_naive(block, cur_path, accum): + cur_path = (cur_path, block) + if not block.exits: + ops = [] + while cur_path: + block = cur_path[1] + ops.extend(reversed(block.operations)) + cur_path = cur_path[0] + accum.append(list(reversed(ops))) + return + for link in block.exits: + _follow_path_naive(link.target, cur_path, accum) + +# explodes on loops! +def paths_naive(g): + accum = [] + _follow_path_naive(g.startblock, None, accum) + return accum - def g(n): - f(A(n)) +def direct_target(spaceop): + return spaceop.args[0].value._obj.graph.name + +def direct_calls(p): + names = [] + for spaceop in p: + if spaceop.opname == 'direct_call': + names.append(direct_target(spaceop)) + return names + + +def check(g, funcname, ignore=None): + paths = paths_naive(g) + relevant = [] + for p in paths: + funcs_called = direct_calls(p) + if funcname in funcs_called and ignore not in funcs_called: + assert 'stack_check___' in funcs_called + assert (funcs_called.index(funcname) > + funcs_called.index('stack_check___')) + relevant.append(p) + return relevant + +class A(object): + def __init__(self, n): + self.n = n + +def f(a): + x = A(a.n+1) + if x.n == 10: + return + f(x) + +def g(n): + f(A(n)) + return 0 + +def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) @@ -29,7 +75,97 @@ assert n == 1 if conftest.option.view: t.view() - + check(graphof(t, f), 'f') - - +def test_gctransformed(): + t = TranslationContext() + a = t.buildannotator() + a.build_types(g, [int]) + a.simplify() + t.buildrtyper().specialize() + backend_optimizations(t) + t.checkgraphs() + n = insert_ll_stackcheck(t) + t.checkgraphs() + assert n == 1 + exctransf = t.getexceptiontransformer() + f_graph = graphof(t, f) + exctransf.create_exception_handling(f_graph) + if conftest.option.view: + f_graph.show() + check(f_graph, 'f') + + class GCTransform(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {} + + gctransf = GCTransform(t) + gctransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + relevant = check(f_graph, 'f') + for p in relevant: + in_between = False + reload = 0 + for spaceop in p: + if spaceop.opname == 'direct_call': + target = direct_target(spaceop) + if target == 'f': + in_between = False + elif target == 'stack_check___': + in_between = True + if in_between and spaceop.opname == 'gc_reload_possibly_moved': + reload += 1 + + assert reload == 1 # we would like this to be zero + +def test_stackless(): + t = TranslationContext() + a = t.buildannotator() + a.build_types(g, [int]) + a.simplify() + t.buildrtyper().specialize() + backend_optimizations(t) + t.checkgraphs() + n = insert_ll_stackcheck(t) + t.checkgraphs() + assert n == 1 + + stacklesstransf = StacklessTransformer(t, g) + + f_graph = graphof(t, f) + stacklesstransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + + exctransf = t.getexceptiontransformer() + exctransf.create_exception_handling(f_graph) + if conftest.option.view: + f_graph.show() + check(f_graph, 'f', 'fetch_retval_void') + + class GCTransform(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {} + + gctransf = GCTransform(t) + gctransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + relevant = check(f_graph, 'f', 'fetch_retval_void') + for p in relevant: + in_between = False + reload = 0 + for spaceop in p: + if spaceop.opname == 'direct_call': + target = direct_target(spaceop) + if target == 'f': + in_between = False + elif target == 'stack_check___': + in_between = True + if in_between and spaceop.opname == 'gc_reload_possibly_moved': + reload += 1 + + assert reload == 1 From pedronis at codespeak.net Sat Sep 6 18:58:14 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 6 Sep 2008 18:58:14 +0200 (CEST) Subject: [pypy-svn] r57896 - in pypy/branch/garden-call-code-2/pypy: rpython/memory/gctransform rpython/memory/gctransform/test translator/test Message-ID: <20080906165814.79219169F34@codespeak.net> Author: pedronis Date: Sat Sep 6 18:58:12 2008 New Revision: 57896 Modified: pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/framework.py pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/test/test_framework.py pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py Log: avoid the root reloading around stack checks in the non-stackless case Modified: pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/framework.py Sat Sep 6 18:58:12 2008 @@ -7,6 +7,7 @@ from pypy.rpython.memory.gc import marksweep from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib import rstack from pypy.rlib.debug import ll_assert from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.support import var_needsgc @@ -25,7 +26,10 @@ def analyze_direct_call(self, graph, seen=None): try: - if graph.func._gctransformer_hint_cannot_collect_: + func = graph.func + if func is rstack.stack_check: + return self.translator.config.translation.stackless + if func._gctransformer_hint_cannot_collect_: return False except AttributeError: pass Modified: pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/garden-call-code-2/pypy/rpython/memory/gctransform/test/test_framework.py Sat Sep 6 18:58:12 2008 @@ -63,6 +63,28 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) + def g(x): + return -x + t = rtype(g, [int]) + gg = graphof(t, g) + assert not CollectAnalyzer(t).analyze_direct_call(gg) + +def test_cancollect_stack_check(): + from pypy.rlib import rstack + + def with_check(): + rstack.stack_check() + + t = rtype(with_check, []) + with_check_graph = graphof(t, with_check) + + assert not t.config.translation.stackless + can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) + assert not can_collect + + t.config.translation.stackless = True + can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) + assert can_collect class WriteBarrierTransformer(FrameworkGCTransformer): initializing_stores = {} Modified: pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py ============================================================================== --- pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py (original) +++ pypy/branch/garden-call-code-2/pypy/translator/test/test_stackcheck.py Sat Sep 6 18:58:12 2008 @@ -118,7 +118,7 @@ if in_between and spaceop.opname == 'gc_reload_possibly_moved': reload += 1 - assert reload == 1 # we would like this to be zero + assert reload == 0 def test_stackless(): t = TranslationContext() @@ -132,6 +132,7 @@ t.checkgraphs() assert n == 1 + t.config.translation.stackless = True stacklesstransf = StacklessTransformer(t, g) f_graph = graphof(t, f) From fijal at codespeak.net Sat Sep 6 19:47:38 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 19:47:38 +0200 (CEST) Subject: [pypy-svn] r57900 - pypy/branch/cross-compilation/pypy/doc/discussion Message-ID: <20080906174738.CE380169EFB@codespeak.net> Author: fijal Date: Sat Sep 6 19:47:37 2008 New Revision: 57900 Modified: pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt Log: Notes how to read smaps Modified: pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt ============================================================================== --- pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt (original) +++ pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt Sat Sep 6 19:47:37 2008 @@ -28,3 +28,21 @@ I think we need some kind of graph which shows how exactly this grows with number of objects. + +Some notes how to read memory footprint on linux: + +all stuff that we want to know is located in /proc/$pid/smaps (beginning +from linux 2.6.16, which is almost any apparently). This contains: +Size, Rss, Shared and Private. Note that all addresses are virtual, which means +that having the same address in two processes doesn't mean it's the same memory. + +Explanation: +Size: total (virtual) size of memory, possibly irrelevant +Rss: real size of memory +Shared: memory shared with other processes. Note that this is simply a counter +how many processes reference it. Memory can move private -> shared in case +some other process will load the same library or so. +Private: private memory owned by a process. + +Distinction clean/dirty is related to swap and probably irrelevant to our +measurments. From pedronis at codespeak.net Sat Sep 6 20:37:33 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 6 Sep 2008 20:37:33 +0200 (CEST) Subject: [pypy-svn] r57901 - pypy/dist/pypy/translator/c/test Message-ID: <20080906183733.932A0169EA5@codespeak.net> Author: pedronis Date: Sat Sep 6 20:37:31 2008 New Revision: 57901 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: move the recursion tests together again Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Sat Sep 6 20:37:31 2008 @@ -656,18 +656,6 @@ for args in [2, 7, 0], [7, 2, 0], [10, 50, 7], [50, -10, -3]: assert f(*args) == intmask(fn(*args)) - def test_recursion_detection(self): - def f(n, accum): - if n == 0: - return accum - else: - return f(n-1, accum*n) - fn = self.getcompiled(f, [int, int]) - assert fn(7, 1) == 5040 - assert fn(7, 1) == 5040 # detection must work several times, too - assert fn(7, 1) == 5040 - py.test.raises(RuntimeError, fn, -1, 0) - def test_list_len_is_true(self): class X(object): @@ -695,6 +683,18 @@ fn = self.getcompiled(f) assert fn() == 1 + def test_recursion_detection(self): + def f(n, accum): + if n == 0: + return accum + else: + return f(n-1, accum*n) + fn = self.getcompiled(f, [int, int]) + assert fn(7, 1) == 5040 + assert fn(7, 1) == 5040 # detection must work several times, too + assert fn(7, 1) == 5040 + py.test.raises(RuntimeError, fn, -1, 0) + def test_infinite_recursion(self): def f(x): if x: From fijal at codespeak.net Sat Sep 6 20:38:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 20:38:20 +0200 (CEST) Subject: [pypy-svn] r57902 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080906183820.66AEF169EB1@codespeak.net> Author: fijal Date: Sat Sep 6 20:38:19 2008 New Revision: 57902 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: Parse /proc/$pid/smaps output (once) Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Sat Sep 6 20:38:19 2008 @@ -1,13 +1,54 @@ #!/usr/bin/env python import os import py +import time +import re + +class Result(object): + def __init__(self, priv_map, shared_map, starttime=None): + if starttime is None: + starttime = time.time() + self._compute_total(priv_map, shared_map) + self.priv_map = priv_map + self.shared_map = shared_map + + def _compute_total(self, priv_map, shared_map): + self.private = sum(priv_map.values()) + self.shared = sum(shared_map.values()) + +def parse_pmap_output(raw_data): + def number(line): + m = re.search('(\d+) kB', line) + if not m: + raise ValueError("Wrong line: %s" % (line,)) + return int(m.group(1)) + + lines = raw_data.split('\n')[1:-1] + priv_map = {} + shared_map = {} + num = 0 + while num < len(lines): + m = re.search('(\S+)\s*$', lines[num]) + if not m: + raise ValueError("Wrong line " + lines[num]) + name = m.group(1) + priv = number(lines[num + 5]) + number(lines[num + 6]) + shared = number(lines[num + 3]) + number(lines[num + 4]) + if priv: + assert not shared + priv_map[name] = priv + priv_map.get(name, 0) + else: + assert shared + shared_map[name] = shared + shared_map.get(name, 0) + num += 8 + return Result(priv_map, shared_map) def run_child(name, args): pid = os.fork() if not pid: os.execvp(name, [name] + args) else: - res = py.process.cmdexec('pmap -d %d' % pid) + res = py.process.cmdexec('pmap -x %d' % pid) return res if __name__ == '__main__': Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Sat Sep 6 20:38:19 2008 @@ -4,3 +4,50 @@ def test_basic(): res = bench_mem.run_child('python', ['-c', 'pass']) assert 'python' in res + +def test_parse(): + res = bench_mem.parse_pmap_output(example_data) + assert res.private == 796 + 120 + 924 + assert res.shared == 60 + assert res.priv_map == { + '/usr/bin/python2.5': 796 + 120, + '[heap]' : 924, + } + assert res.shared_map == { + '/lib/libncurses.so.5.6' : 60, + } + +example_data = ''' +08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 +Size: 988 kB +Rss: 796 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 796 kB +Private_Dirty: 0 kB +Referenced: 796 kB +0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 +Size: 148 kB +Rss: 120 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 12 kB +Private_Dirty: 108 kB +Referenced: 120 kB +08164000-0825c000 rw-p 08164000 00:00 0 [heap] +Size: 992 kB +Rss: 924 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 924 kB +Referenced: 924 kB +b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 +Size: 240 kB +Rss: 60 kB +Shared_Clean: 60 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 0 kB +Referenced: 60 kB +''' From pedronis at codespeak.net Sat Sep 6 20:49:43 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 6 Sep 2008 20:49:43 +0200 (CEST) Subject: [pypy-svn] r57903 - pypy/dist/pypy/translator/c/test Message-ID: <20080906184943.DB2D2169EC5@codespeak.net> Author: pedronis Date: Sat Sep 6 20:49:41 2008 New Revision: 57903 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: a recent gcc was happily tail optimizing these into failure, make them non tail-recursive Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Sat Sep 6 20:49:41 2008 @@ -684,21 +684,21 @@ assert fn() == 1 def test_recursion_detection(self): - def f(n, accum): + def f(n): if n == 0: - return accum + return 1 else: - return f(n-1, accum*n) - fn = self.getcompiled(f, [int, int]) - assert fn(7, 1) == 5040 - assert fn(7, 1) == 5040 # detection must work several times, too - assert fn(7, 1) == 5040 - py.test.raises(RuntimeError, fn, -1, 0) + return n*f(n-1) + fn = self.getcompiled(f, [int]) + assert fn(7) == 5040 + assert fn(7) == 5040 # detection must work several times, too + assert fn(7) == 5040 + py.test.raises(RuntimeError, fn, -1) def test_infinite_recursion(self): def f(x): if x: - return f(x) + return 1+f(x) return 1 def g(x): try: From fijal at codespeak.net Sat Sep 6 21:04:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 21:04:33 +0200 (CEST) Subject: [pypy-svn] r57904 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080906190433.13B04169EF9@codespeak.net> Author: fijal Date: Sat Sep 6 21:04:29 2008 New Revision: 57904 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: * Make tests more robust agains timing * Adapt a script, it's almost usable by now Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Sat Sep 6 21:04:29 2008 @@ -1,8 +1,12 @@ #!/usr/bin/env python +""" Usage: bench_mem.py program_name [arg0] [arg1] ... +""" import os import py import time import re +import signal +import sys class Result(object): def __init__(self, priv_map, shared_map, starttime=None): @@ -16,14 +20,14 @@ self.private = sum(priv_map.values()) self.shared = sum(shared_map.values()) -def parse_pmap_output(raw_data): +def parse_smaps_output(raw_data): def number(line): m = re.search('(\d+) kB', line) if not m: raise ValueError("Wrong line: %s" % (line,)) return int(m.group(1)) - lines = raw_data.split('\n')[1:-1] + lines = [i for i in raw_data.split('\n')[:-1] if i] priv_map = {} shared_map = {} num = 0 @@ -35,21 +39,48 @@ priv = number(lines[num + 5]) + number(lines[num + 6]) shared = number(lines[num + 3]) + number(lines[num + 4]) if priv: - assert not shared priv_map[name] = priv + priv_map.get(name, 0) - else: - assert shared + if shared: shared_map[name] = shared + shared_map.get(name, 0) num += 8 return Result(priv_map, shared_map) +class ChildProcess(object): + realos = os + + def __init__(self, name, args): + self.pid = run_child(name, args) + + def loop(self, logfile, interval): + if isinstance(logfile, basestring): + logfile = open(logfile, 'w') + counter = 0 + while 1: + try: + res = parse_smaps_output(open('/proc/%d/smaps' % self.pid).read()) + print >>logfile, counter, ' ', res.private + except IOError: + self.close() + return + counter += 1 + time.sleep(interval) + + def close(self): + if self.pid: + self.realos.waitpid(self.pid, 0) + + def __del__(self): + self.close() + def run_child(name, args): pid = os.fork() if not pid: os.execvp(name, [name] + args) - else: - res = py.process.cmdexec('pmap -x %d' % pid) - return res + return pid if __name__ == '__main__': - run_child('python', ['-c', 'pass']) + if len(sys.argv) < 2: + print __doc__ + sys.exit(1) + cp = ChildProcess(sys.argv[1], sys.argv[2:]) + cp.loop(sys.stdout, 0.1) Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Sat Sep 6 21:04:29 2008 @@ -1,12 +1,18 @@ from pypy.translator.benchmark import bench_mem +import time -def test_basic(): - res = bench_mem.run_child('python', ['-c', 'pass']) - assert 'python' in res +def test_compute_memory_usage(): + pid = bench_mem.run_child('python', ['-c', 'import time;time.sleep(1)']) + time.sleep(.3) + assert pid + res = open('/proc/%d/smaps' % pid).read() + parsed = bench_mem.parse_smaps_output(res) + assert parsed.shared + assert parsed.private def test_parse(): - res = bench_mem.parse_pmap_output(example_data) + res = bench_mem.parse_smaps_output(example_data) assert res.private == 796 + 120 + 924 assert res.shared == 60 assert res.priv_map == { From fijal at codespeak.net Sat Sep 6 21:26:38 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 21:26:38 +0200 (CEST) Subject: [pypy-svn] r57906 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes_tests Message-ID: <20080906192638.3DFAC169F23@codespeak.net> Author: fijal Date: Sat Sep 6 21:26:37 2008 New Revision: 57906 Modified: pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/app_test/ctypes_tests/test_guess_argtypes.py Log: Minor fix for guessing types Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Sat Sep 6 21:26:37 2008 @@ -150,7 +150,7 @@ for arg in args: if hasattr(arg, '_as_parameter_'): arg = arg._as_parameter_ - if isinstance(arg, str): + if isinstance(arg, basestring): res.append(c_char_p) elif isinstance(arg, _CData): res.append(type(arg)) Modified: pypy/dist/pypy/lib/app_test/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes_tests/test_guess_argtypes.py Sat Sep 6 21:26:37 2008 @@ -18,6 +18,7 @@ assert guess(['xca']) == [c_char_p] assert guess([None]) == [c_void_p] assert guess([c_int(3)]) == [c_int] + assert guess([u'xca']) == [c_char_p] class Stuff: pass From fijal at codespeak.net Sat Sep 6 21:42:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 21:42:43 +0200 (CEST) Subject: [pypy-svn] r57908 - pypy/branch/cross-compilation/pypy/translator/benchmark/memory Message-ID: <20080906194243.9F84416A091@codespeak.net> Author: fijal Date: Sat Sep 6 21:42:38 2008 New Revision: 57908 Added: pypy/branch/cross-compilation/pypy/translator/benchmark/memory/ pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py (contents, props changed) Log: some benchmark Added: pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py ============================================================================== --- (empty file) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py Sat Sep 6 21:42:38 2008 @@ -0,0 +1,14 @@ + +class A(object): + pass + +class B(object): + def __init__(self): + self.a = [1,2,3] + self.b = "xyz" + +j = 0 +while j < 20: + x = [(A(), B()) for i in range(100000)] + del x + j += 1 From fijal at codespeak.net Sat Sep 6 22:19:05 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 22:19:05 +0200 (CEST) Subject: [pypy-svn] r57909 - in pypy/dist/pypy/rpython/module: . test Message-ID: <20080906201905.EF897169F54@codespeak.net> Author: fijal Date: Sat Sep 6 22:19:02 2008 New Revision: 57909 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_posix.py Log: os.getegid, os.chroot Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Sep 6 22:19:02 2008 @@ -394,6 +394,17 @@ return extdef([], int, export_name="ll_os.ll_os_setsid", llimpl=setsid_llimpl) + @registering_if(os, 'chroot') + def register_os_chroot(self): + os_chroot = self.llexternal('chroot', [rffi.CCHARP], rffi.INT) + def chroot_llimpl(arg): + result = os_chroot(arg) + if result == -1: + raise OSError(rposix.get_errno(), "os_chroot failed") + + return extdef([str], None, export_name="ll_os.ll_os_chroot", + llimpl=chroot_llimpl) + @registering_if(os, 'uname') def register_os_uname(self): CHARARRAY = lltype.FixedSizeArray(lltype.Char, 1) @@ -463,6 +474,11 @@ def register_os_getgid(self): return self.extdef_for_os_function_returning_int('getgid') + @registering_if(os, 'getegid') + def register_os_getegid(self): + return self.extdef_for_os_function_returning_int('getegid') + + @registering(os.open) def register_os_open(self): os_open = self.llexternal(underscore_on_windows+'open', Modified: pypy/dist/pypy/rpython/module/test/test_posix.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_posix.py (original) +++ pypy/dist/pypy/rpython/module/test/test_posix.py Sat Sep 6 22:19:02 2008 @@ -139,6 +139,17 @@ return os.sysconf(i) assert self.interpret(f, [13]) == f(13) + if hasattr(os, 'chroot'): + def test_os_chroot(self): + def f(): + try: + os.chroot('!@$#!#%$#^#@!#!$$#^') + except OSError: + return 1 + return 0 + + assert self.interpret(f, []) == 1 + def test_os_wstar(self): from pypy.rpython.module.ll_os import RegisterOs for name in RegisterOs.w_star: @@ -157,3 +168,6 @@ class TestOOtype(BaseTestPosix, OORtypeMixin): def test_fstat(self): py.test.skip("ootypesystem does not support os.fstat") + + def test_os_chroot(self): + py.test.skip("ootypesystem does not support os.chroot") From fijal at codespeak.net Sat Sep 6 22:43:11 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 22:43:11 +0200 (CEST) Subject: [pypy-svn] r57912 - pypy/dist/pypy/module/posix Message-ID: <20080906204311.8F85F16A0F0@codespeak.net> Author: fijal Date: Sat Sep 6 22:43:09 2008 New Revision: 57912 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py Log: * implement getegid and chroot. * seems that this comment is completely outdated, remove Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sat Sep 6 22:43:09 2008 @@ -93,6 +93,8 @@ interpleveldefs['geteuid'] = 'interp_posix.geteuid' if hasattr(os, 'getgid'): interpleveldefs['getgid'] = 'interp_posix.getgid' + if hasattr(os, 'getegid'): + interpleveldefs['getegid'] = 'interp_posix.getegid' if hasattr(os, 'setuid'): interpleveldefs['setuid'] = 'interp_posix.setuid' if hasattr(os, 'setgid'): @@ -100,6 +102,8 @@ # not visible via os, inconsistency in nt: if hasattr(posix, '_getfullpathname'): interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' + if hasattr(os, 'chroot'): + interpleveldefs['chroot'] = 'interp_posix.chroot' for name in RegisterOs.w_star: if hasattr(os, name): @@ -109,10 +113,6 @@ """NOT_RPYTHON""" space = self.space config = space.config - # XXX execve does not work under ootypesystem yet :-( - # YYY nor does it anywhere else - #if config.translating and config.translation.type_system != "lltype": - # space.delattr(self, space.wrap("execve")) if config.translating and config.translation.backend == "llvm": space.delattr(self, space.wrap("execv")) Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sat Sep 6 22:43:09 2008 @@ -591,6 +591,18 @@ return space.w_None setgid.unwrap_spec = [ObjSpace, int] +def chroot(space, path): + """ chroot(path) + + Change root directory to path. + """ + try: + os.chroot(path) + except OSError, e: + raise wrap_oserror(space, e) + return space.w_None +chroot.unwrap_spec = [ObjSpace, str] + def getgid(space): """ getgid() -> gid @@ -599,6 +611,14 @@ return space.wrap(os.getgid()) getgid.unwrap_spec = [ObjSpace] +def getegid(space): + """ getegid() -> gid + + Return the current process's effective group id. + """ + return space.wrap(os.getegid()) +getgid.unwrap_spec = [ObjSpace] + def geteuid(space): """ geteuid() -> euid From fijal at codespeak.net Sat Sep 6 23:19:27 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 6 Sep 2008 23:19:27 +0200 (CEST) Subject: [pypy-svn] r57913 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080906211927.DEF3E16A117@codespeak.net> Author: fijal Date: Sat Sep 6 23:19:26 2008 New Revision: 57913 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: update status of django Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 6 23:19:26 2008 @@ -72,13 +72,10 @@ Django ====== -* we run (almost) unmodified Django +* we run unmodified Django 1.0 * only sqlite DB backend for now -* cooperation with Django people to make sure - that Django 1.0 works with PyPy - * http://www.djangoproject.com/ Pylons From fijal at codespeak.net Sun Sep 7 09:04:06 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 09:04:06 +0200 (CEST) Subject: [pypy-svn] r57919 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080907070406.BABC116A12F@codespeak.net> Author: fijal Date: Sun Sep 7 09:04:02 2008 New Revision: 57919 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: small glitches Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sun Sep 7 09:04:02 2008 @@ -47,6 +47,8 @@ * part of google sponsoring +* 32bit only by now + * demo CTypes configure @@ -113,6 +115,8 @@ * py lib +* sympy + * various smaller things, templating engines, most pure-python software @@ -134,9 +138,10 @@ Transition to 2.5 ========================== -XXX * SOC project Bruno Gola + * almost complete + * missing more testing, stdlib porting Conclusion on Compatibility @@ -172,6 +177,8 @@ * care needed with communication with C +* GCs are semi-decent + Speed - JIT generator ===================== @@ -179,7 +186,7 @@ * will be super fast -* prolog prototype +* some prototypes, research ongoing * psyco is a nice proof that this approach would work From fijal at codespeak.net Sun Sep 7 09:17:10 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 09:17:10 +0200 (CEST) Subject: [pypy-svn] r57920 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080907071710.F23B6169EE4@codespeak.net> Author: fijal Date: Sun Sep 7 09:17:09 2008 New Revision: 57920 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: twisted update Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sun Sep 7 09:17:09 2008 @@ -96,7 +96,7 @@ Twisted & Nevow =============== -* twisted have some glitches, but mostly works +* twisted works (60/4500 tests failing) * nevow works From fijal at codespeak.net Sun Sep 7 09:19:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 09:19:39 +0200 (CEST) Subject: [pypy-svn] r57921 - in pypy/dist/pypy: module/posix rpython/module Message-ID: <20080907071939.55EE0169F34@codespeak.net> Author: fijal Date: Sun Sep 7 09:19:38 2008 New Revision: 57921 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/rpython/module/ll_os.py Log: os.seteuid Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sun Sep 7 09:19:38 2008 @@ -97,6 +97,8 @@ interpleveldefs['getegid'] = 'interp_posix.getegid' if hasattr(os, 'setuid'): interpleveldefs['setuid'] = 'interp_posix.setuid' + if hasattr(os, 'seteuid'): + interpleveldefs['seteuid'] = 'interp_posix.seteuid' if hasattr(os, 'setgid'): interpleveldefs['setgid'] = 'interp_posix.setgid' # not visible via os, inconsistency in nt: Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Sep 7 09:19:38 2008 @@ -579,6 +579,18 @@ return space.w_None setuid.unwrap_spec = [ObjSpace, int] +def seteuid(space, arg): + """ seteuid(uid) + + Set the current process's effective user id. + """ + try: + os.seteuid(arg) + except OSError, e: + raise wrap_oserror(space, e) + return space.w_None +seteuid.unwrap_spec = [ObjSpace, int] + def setgid(space, arg): """ setgid(gid) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sun Sep 7 09:19:38 2008 @@ -462,6 +462,10 @@ def register_os_setuid(self): return self.extdef_for_function_int_to_int('setuid') + @registering_if(os, 'seteuid') + def register_os_seteuid(self): + return self.extdef_for_function_int_to_int('seteuid') + @registering_if(os, 'setgid') def register_os_setgid(self): return self.extdef_for_function_int_to_int('setgid') From fijal at codespeak.net Sun Sep 7 11:15:28 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 11:15:28 +0200 (CEST) Subject: [pypy-svn] r57927 - pypy/branch/cross-compilation/pypy/translator/benchmark Message-ID: <20080907091528.493C516A12F@codespeak.net> Author: fijal Date: Sun Sep 7 11:15:26 2008 New Revision: 57927 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Log: allow to have -l log option Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Sun Sep 7 11:15:26 2008 @@ -1,5 +1,5 @@ #!/usr/bin/env python -""" Usage: bench_mem.py program_name [arg0] [arg1] ... +""" Usage: bench_mem.py [-l log] program_name [arg0] [arg1] ... """ import os import py @@ -49,25 +49,33 @@ realos = os def __init__(self, name, args): + signal.signal(signal.SIGCHLD, lambda a,b: self.close()) self.pid = run_child(name, args) + self.results = [] def loop(self, logfile, interval): if isinstance(logfile, basestring): logfile = open(logfile, 'w') counter = 0 - while 1: - try: - res = parse_smaps_output(open('/proc/%d/smaps' % self.pid).read()) - print >>logfile, counter, ' ', res.private - except IOError: - self.close() - return - counter += 1 - time.sleep(interval) + try: + while 1: + try: + res = parse_smaps_output(open('/proc/%d/smaps' % self.pid).read()) + self.results.append((counter, res.private)) + if logfile: + print >>logfile, counter, ' ', res.private + except IOError: + return + counter += 1 + time.sleep(interval) + except (KeyboardInterrupt, SystemExit): + os.kill(self.pid, signal.SIGTERM) + raise def close(self): - if self.pid: + if self.pid != -1: self.realos.waitpid(self.pid, 0) + self.pid = -1 def __del__(self): self.close() @@ -78,9 +86,27 @@ os.execvp(name, [name] + args) return pid +def parse_options(argv): + num = 0 + logname = None + while num < len(argv): + arg = argv[num] + if arg == '-l': + logname = argv[num + 1] + num += 1 + else: + name = argv[num] + if logname is None: + logname = py.path.local(name).basename + '.log' + args = argv[num + 1:] + return logname, name, args + num += 1 + raise Exception("Wrong arguments: %s" % (argv,)) + if __name__ == '__main__': if len(sys.argv) < 2: print __doc__ sys.exit(1) - cp = ChildProcess(sys.argv[1], sys.argv[2:]) - cp.loop(sys.stdout, 0.1) + logname, name, args = parse_options(sys.argv[1:]) + cp = ChildProcess(name, args) + cp.loop(logname, 0) From fijal at codespeak.net Sun Sep 7 11:17:54 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 11:17:54 +0200 (CEST) Subject: [pypy-svn] r57928 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . memory Message-ID: <20080907091754.3495E16A155@codespeak.net> Author: fijal Date: Sun Sep 7 11:17:53 2008 New Revision: 57928 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py Log: update docstring Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Sun Sep 7 11:17:53 2008 @@ -1,5 +1,9 @@ #!/usr/bin/env python """ Usage: bench_mem.py [-l log] program_name [arg0] [arg1] ... + +Run a program with given args under constant memory usage monitoring. +Dump output (which is step & private memory used) into a file suitable +for plotting with gnuplot. """ import os import py Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/memory/one.py Sun Sep 7 11:17:53 2008 @@ -1,4 +1,6 @@ +import gc + class A(object): pass @@ -7,8 +9,14 @@ self.a = [1,2,3] self.b = "xyz" -j = 0 -while j < 20: - x = [(A(), B()) for i in range(100000)] - del x - j += 1 +x = [(A(), B()) for i in range(100000)] +gc.collect() + +while 1: + pass +#j = 0 +#while j < 20: +# x = [(A(), B()) for i in range(100000)] +# del x +# gc.collect() +# j += 1 From fijal at codespeak.net Sun Sep 7 11:59:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 11:59:39 +0200 (CEST) Subject: [pypy-svn] r57929 - pypy/branch/tuple-nonresizable-395 Message-ID: <20080907095939.3595516A135@codespeak.net> Author: fijal Date: Sun Sep 7 11:59:37 2008 New Revision: 57929 Added: pypy/branch/tuple-nonresizable-395/ - copied from r57928, pypy/dist/ Log: Create a branch ro resolve #395 From pedronis at codespeak.net Sun Sep 7 12:31:57 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 7 Sep 2008 12:31:57 +0200 (CEST) Subject: [pypy-svn] r57935 - in pypy/dist/pypy: interpreter interpreter/test rpython/memory/gctransform rpython/memory/gctransform/test translator/test Message-ID: <20080907103157.43C2616A135@codespeak.net> Author: pedronis Date: Sun Sep 7 12:31:55 2008 New Revision: 57935 Modified: pypy/dist/pypy/interpreter/eval.py pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/pycode.py pypy/dist/pypy/interpreter/test/test_function.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py pypy/dist/pypy/translator/test/test_stackcheck.py Log: merging garden-call-code-2 - don't use fastcall* shortcuts for PyCode, simply have uniform code for the flat f(a,b,c,d) case. Speeds up calls to python functions with more than 4 arguments - avoid unneeded root reloading around stack_check in non-stackless builds: speeds up quite a bit of things except for the odd one because of our usual translation performance instabilities Modified: pypy/dist/pypy/interpreter/eval.py ============================================================================== --- pypy/dist/pypy/interpreter/eval.py (original) +++ pypy/dist/pypy/interpreter/eval.py Sun Sep 7 12:31:55 2008 @@ -62,18 +62,6 @@ def funcrun_obj(self, func, w_obj, args): return self.funcrun(func, args.prepend(w_obj)) - - # a performance hack (see gateway.BuiltinCode1/2/3 and pycode.PyCode) - def fastcall_0(self, space, func): - raise NotImplementedError - def fastcall_1(self, space, func, w1): - raise NotImplementedError - def fastcall_2(self, space, func, w1, w2): - raise NotImplementedError - def fastcall_3(self, space, func, w1, w2, w3): - raise NotImplementedError - def fastcall_4(self, space, func, w1, w2, w3, w4): - raise NotImplementedError class Frame(Wrappable): """A frame is an environment supporting the execution of a code object. Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Sun Sep 7 12:31:55 2008 @@ -6,11 +6,14 @@ attribute. """ +from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.eval import Code from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +funccallunrolling = unrolling_iterable(range(4)) + class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, @@ -44,24 +47,40 @@ return self.code def funccall(self, *args_w): # speed hack + from pypy.interpreter import gateway + from pypy.interpreter.pycode import PyCode + code = self.getcode() # hook for the jit nargs = len(args_w) fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: + assert isinstance(code, gateway.BuiltinCode1) return code.fastcall_1(self.space, self, args_w[0]) elif nargs == 2: + assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, args_w[0], args_w[1]) elif nargs == 3: + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, args_w[0], args_w[1], args_w[2]) elif nargs == 4: + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, args_w[0], args_w[1], args_w[2], args_w[3]) + elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: + assert isinstance(code, PyCode) + if nargs < 5: + new_frame = self.space.createframe(code, self.w_func_globals, + self.closure) + for i in funccallunrolling: + if i < nargs: + new_frame.fastlocals_w[i] = args_w[i] + return new_frame.run() elif nargs >= 1 and fast_natural_arity == -1: - from pypy.interpreter import gateway assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], Arguments(self.space, @@ -69,25 +88,35 @@ return self.call_args(Arguments(self.space, list(args_w))) def funccall_valuestack(self, nargs, frame): # speed hack + from pypy.interpreter import gateway + from pypy.interpreter.pycode import PyCode + code = self.getcode() # hook for the jit fast_natural_arity = code.fast_natural_arity - if nargs == fast_natural_arity: + if nargs == fast_natural_arity: if nargs == 0: + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: + assert isinstance(code, gateway.BuiltinCode1) return code.fastcall_1(self.space, self, frame.peekvalue(0)) elif nargs == 2: + assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, frame.peekvalue(1), frame.peekvalue(0)) elif nargs == 3: + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, frame.peekvalue(2), frame.peekvalue(1), frame.peekvalue(0)) elif nargs == 4: + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, frame.peekvalue(3), frame.peekvalue(2), frame.peekvalue(1), frame.peekvalue(0)) + elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: + assert isinstance(code, PyCode) + return self._flat_pycall(code, nargs, frame) elif fast_natural_arity == -1 and nargs >= 1: - from pypy.interpreter import gateway assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) @@ -104,6 +133,15 @@ if isinstance(args, ArgumentsFromValuestack): args.frame = None + def _flat_pycall(self, code, nargs, frame): + # code is a PyCode + new_frame = self.space.createframe(code, self.w_func_globals, + self.closure) + for i in xrange(nargs): + w_arg = frame.peekvalue(nargs-1-i) + new_frame.fastlocals_w[i] = w_arg + return new_frame.run() + def getdict(self): if self.w_func_dict is None: self.w_func_dict = self.space.newdict() Modified: pypy/dist/pypy/interpreter/pycode.py ============================================================================== --- pypy/dist/pypy/interpreter/pycode.py (original) +++ pypy/dist/pypy/interpreter/pycode.py Sun Sep 7 12:31:55 2008 @@ -108,7 +108,7 @@ self._args_as_cellvars.append(-1) # pad self._args_as_cellvars[i] = j - self._compute_fastcall() + self._compute_flatcall() co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace @@ -159,53 +159,20 @@ freevars, cellvars, hidden_applevel) _code_new_w = staticmethod(_code_new_w) + + FLATPYCALL = 0x100 - def _compute_fastcall(self): + def _compute_flatcall(self): # Speed hack! self.fast_natural_arity = -99 - if not (0 <= self.co_argcount <= 4): - return if self.co_flags & (CO_VARARGS | CO_VARKEYWORDS): return if len(self._args_as_cellvars) > 0: return - - self.fast_natural_arity = self.co_argcount - - def fastcall_0(self, space, w_func): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - return frame.run() - - def fastcall_1(self, space, w_func, w_arg): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg # frame.setfastscope([w_arg]) - return frame.run() - - def fastcall_2(self, space, w_func, w_arg1, w_arg2): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - return frame.run() - - def fastcall_3(self, space, w_func, w_arg1, w_arg2, w_arg3): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - frame.fastlocals_w[2] = w_arg3 - return frame.run() - - def fastcall_4(self, space, w_func, w_arg1, w_arg2, w_arg3, w_arg4): - frame = space.createframe(self, w_func.w_func_globals, - w_func.closure) - frame.fastlocals_w[0] = w_arg1 # frame.setfastscope([w_arg]) - frame.fastlocals_w[1] = w_arg2 - frame.fastlocals_w[2] = w_arg3 - frame.fastlocals_w[3] = w_arg4 - return frame.run() + if self.co_argcount > 0xff: + return + + self.fast_natural_arity = PyCode.FLATPYCALL | self.co_argcount def funcrun(self, func, args): frame = self.space.createframe(self, func.w_func_globals, Modified: pypy/dist/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_function.py (original) +++ pypy/dist/pypy/interpreter/test/test_function.py Sun Sep 7 12:31:55 2008 @@ -457,9 +457,38 @@ w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str) assert space.is_w(w_meth5, w_meth3) -class TestShortcuts(object): +class TestShortcuts(object): - def test_fastcall(self): + def test_call_function(self): + space = self.space + + d = {} + for i in range(10): + args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" + exec """ +def f%s: + return %s +""" % (args, args) in d + f = d['f'] + res = f(*range(i)) + code = PyCode._from_code(self.space, f.func_code) + fn = Function(self.space, code, self.space.newdict()) + + assert fn.code.fast_natural_arity == i|PyCode.FLATPYCALL + if i < 5: + + def bomb(*args): + assert False, "shortcutting should have avoided this" + + code.funcrun = bomb + code.funcrun_obj = bomb + + args_w = map(space.wrap, range(i)) + w_res = space.call_function(fn, *args_w) + check = space.is_true(space.eq(w_res, space.wrap(res))) + assert check + + def test_flatcall(self): space = self.space def f(a): @@ -467,32 +496,26 @@ code = PyCode._from_code(self.space, f.func_code) fn = Function(self.space, code, self.space.newdict()) - assert fn.code.fast_natural_arity == 1 + assert fn.code.fast_natural_arity == 1|PyCode.FLATPYCALL - called = [] - fastcall_1 = fn.code.fastcall_1 - def witness_fastcall_1(space, w_func, w_arg): - called.append(w_func) - return fastcall_1(space, w_func, w_arg) + def bomb(*args): + assert False, "shortcutting should have avoided this" - fn.code.fastcall_1 = witness_fastcall_1 + code.funcrun = bomb + code.funcrun_obj = bomb w_3 = space.newint(3) w_res = space.call_function(fn, w_3) assert w_res is w_3 - assert called == [fn] - - called = [] w_res = space.appexec([fn, w_3], """(f, x): return f(x) """) assert w_res is w_3 - assert called == [fn] - def test_fastcall_method(self): + def test_flatcall_method(self): space = self.space def f(self, a): @@ -500,15 +523,13 @@ code = PyCode._from_code(self.space, f.func_code) fn = Function(self.space, code, self.space.newdict()) - assert fn.code.fast_natural_arity == 2 + assert fn.code.fast_natural_arity == 2|PyCode.FLATPYCALL - called = [] - fastcall_2 = fn.code.fastcall_2 - def witness_fastcall_2(space, w_func, w_arg1, w_arg2): - called.append(w_func) - return fastcall_2(space, w_func, w_arg1, w_arg2) + def bomb(*args): + assert False, "shortcutting should have avoided this" - fn.code.fastcall_2 = witness_fastcall_2 + code.funcrun = bomb + code.funcrun_obj = bomb w_3 = space.newint(3) w_res = space.appexec([fn, w_3], """(f, x): @@ -521,7 +542,6 @@ """) assert space.is_true(w_res) - assert called == [fn, fn] Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Sep 7 12:31:55 2008 @@ -7,6 +7,7 @@ from pypy.rpython.memory.gc import marksweep from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib import rstack from pypy.rlib.debug import ll_assert from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.support import var_needsgc @@ -25,7 +26,10 @@ def analyze_direct_call(self, graph, seen=None): try: - if graph.func._gctransformer_hint_cannot_collect_: + func = graph.func + if func is rstack.stack_check: + return self.translator.config.translation.stackless + if func._gctransformer_hint_cannot_collect_: return False except AttributeError: pass Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_framework.py Sun Sep 7 12:31:55 2008 @@ -63,6 +63,28 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) + def g(x): + return -x + t = rtype(g, [int]) + gg = graphof(t, g) + assert not CollectAnalyzer(t).analyze_direct_call(gg) + +def test_cancollect_stack_check(): + from pypy.rlib import rstack + + def with_check(): + rstack.stack_check() + + t = rtype(with_check, []) + with_check_graph = graphof(t, with_check) + + assert not t.config.translation.stackless + can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) + assert not can_collect + + t.config.translation.stackless = True + can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) + assert can_collect class WriteBarrierTransformer(FrameworkGCTransformer): initializing_stores = {} Modified: pypy/dist/pypy/translator/test/test_stackcheck.py ============================================================================== --- pypy/dist/pypy/translator/test/test_stackcheck.py (original) +++ pypy/dist/pypy/translator/test/test_stackcheck.py Sun Sep 7 12:31:55 2008 @@ -1,22 +1,68 @@ from pypy import conftest -from pypy.translator.translator import TranslationContext +from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.transform import insert_ll_stackcheck +from pypy.rpython.memory.gctransform import framework +from pypy.translator.stackless.transform import StacklessTransformer -def test_simple(): - class A(object): - def __init__(self, n): - self.n = n - - def f(a): - x = A(a.n+1) - if x.n == 10: - return - f(x) +def _follow_path_naive(block, cur_path, accum): + cur_path = (cur_path, block) + if not block.exits: + ops = [] + while cur_path: + block = cur_path[1] + ops.extend(reversed(block.operations)) + cur_path = cur_path[0] + accum.append(list(reversed(ops))) + return + for link in block.exits: + _follow_path_naive(link.target, cur_path, accum) + +# explodes on loops! +def paths_naive(g): + accum = [] + _follow_path_naive(g.startblock, None, accum) + return accum - def g(n): - f(A(n)) +def direct_target(spaceop): + return spaceop.args[0].value._obj.graph.name + +def direct_calls(p): + names = [] + for spaceop in p: + if spaceop.opname == 'direct_call': + names.append(direct_target(spaceop)) + return names + + +def check(g, funcname, ignore=None): + paths = paths_naive(g) + relevant = [] + for p in paths: + funcs_called = direct_calls(p) + if funcname in funcs_called and ignore not in funcs_called: + assert 'stack_check___' in funcs_called + assert (funcs_called.index(funcname) > + funcs_called.index('stack_check___')) + relevant.append(p) + return relevant + +class A(object): + def __init__(self, n): + self.n = n + +def f(a): + x = A(a.n+1) + if x.n == 10: + return + f(x) + +def g(n): + f(A(n)) + return 0 + +def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) @@ -29,7 +75,98 @@ assert n == 1 if conftest.option.view: t.view() - + check(graphof(t, f), 'f') - - +def test_gctransformed(): + t = TranslationContext() + a = t.buildannotator() + a.build_types(g, [int]) + a.simplify() + t.buildrtyper().specialize() + backend_optimizations(t) + t.checkgraphs() + n = insert_ll_stackcheck(t) + t.checkgraphs() + assert n == 1 + exctransf = t.getexceptiontransformer() + f_graph = graphof(t, f) + exctransf.create_exception_handling(f_graph) + if conftest.option.view: + f_graph.show() + check(f_graph, 'f') + + class GCTransform(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {} + + gctransf = GCTransform(t) + gctransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + relevant = check(f_graph, 'f') + for p in relevant: + in_between = False + reload = 0 + for spaceop in p: + if spaceop.opname == 'direct_call': + target = direct_target(spaceop) + if target == 'f': + in_between = False + elif target == 'stack_check___': + in_between = True + if in_between and spaceop.opname == 'gc_reload_possibly_moved': + reload += 1 + + assert reload == 0 + +def test_stackless(): + t = TranslationContext() + a = t.buildannotator() + a.build_types(g, [int]) + a.simplify() + t.buildrtyper().specialize() + backend_optimizations(t) + t.checkgraphs() + n = insert_ll_stackcheck(t) + t.checkgraphs() + assert n == 1 + + t.config.translation.stackless = True + stacklesstransf = StacklessTransformer(t, g) + + f_graph = graphof(t, f) + stacklesstransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + + exctransf = t.getexceptiontransformer() + exctransf.create_exception_handling(f_graph) + if conftest.option.view: + f_graph.show() + check(f_graph, 'f', 'fetch_retval_void') + + class GCTransform(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {} + + gctransf = GCTransform(t) + gctransf.transform_graph(f_graph) + if conftest.option.view: + f_graph.show() + relevant = check(f_graph, 'f', 'fetch_retval_void') + for p in relevant: + in_between = False + reload = 0 + for spaceop in p: + if spaceop.opname == 'direct_call': + target = direct_target(spaceop) + if target == 'f': + in_between = False + elif target == 'stack_check___': + in_between = True + if in_between and spaceop.opname == 'gc_reload_possibly_moved': + reload += 1 + + assert reload == 1 From pedronis at codespeak.net Sun Sep 7 12:32:52 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 7 Sep 2008 12:32:52 +0200 (CEST) Subject: [pypy-svn] r57936 - pypy/branch/garden-call-code-2 Message-ID: <20080907103252.41F0616A14A@codespeak.net> Author: pedronis Date: Sun Sep 7 12:32:50 2008 New Revision: 57936 Removed: pypy/branch/garden-call-code-2/ Log: remove merged branch From fijal at codespeak.net Sun Sep 7 12:39:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 12:39:58 +0200 (CEST) Subject: [pypy-svn] r57937 - pypy/branch/tuple-nonresizable-395 Message-ID: <20080907103958.84EEA16A152@codespeak.net> Author: fijal Date: Sun Sep 7 12:39:57 2008 New Revision: 57937 Removed: pypy/branch/tuple-nonresizable-395/ Log: Remove the branch From fijal at codespeak.net Sun Sep 7 12:40:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 12:40:47 +0200 (CEST) Subject: [pypy-svn] r57938 - pypy/branch/tuple-nonresizable-395 Message-ID: <20080907104047.9120116A158@codespeak.net> Author: fijal Date: Sun Sep 7 12:40:46 2008 New Revision: 57938 Added: pypy/branch/tuple-nonresizable-395/ - copied from r57937, pypy/dist/ Log: Branch again, from fresh trunk From fijal at codespeak.net Sun Sep 7 12:56:09 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 12:56:09 +0200 (CEST) Subject: [pypy-svn] r57939 - in pypy/branch/tuple-nonresizable-395/pypy/annotation: . test Message-ID: <20080907105609.CF97916A169@codespeak.net> Author: fijal Date: Sun Sep 7 12:56:07 2008 New Revision: 57939 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Log: add dont_resize hint to listitem, useful for debugging Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py Sun Sep 7 12:56:07 2008 @@ -26,6 +26,7 @@ self.bookkeeper = bookkeeper self.itemof = {} # set of all ListDefs using this ListItem self.read_locations = {} + self.dont_resize = False if bookkeeper is None: self.dont_change_any_more = True @@ -37,7 +38,7 @@ def resize(self): if not self.resized: - if self.dont_change_any_more: + if self.dont_change_any_more or self.dont_resize: raise TooLateForChange self.resized = True @@ -181,6 +182,9 @@ self.listitem.mutate() self.listitem.resize() + def dont_resize_any_more(self): + self.listitem.dont_resize = True + MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sun Sep 7 12:56:07 2008 @@ -10,7 +10,7 @@ from pypy.translator.translator import graphof as tgraphof from pypy.annotation import policy from pypy.annotation import specialize -from pypy.annotation.listdef import ListDef +from pypy.annotation.listdef import ListDef, TooLateForChange from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong @@ -3040,6 +3040,28 @@ a.build_types(f, [str]) + def test_listitem_no_mutating(self): + from pypy.rlib.debug import check_annotation + called = [] + + def checker(ann, bk): + called.append(True) + assert not ann.listdef.listitem.mutated + ann.listdef.dont_resize_any_more() + + def f(): + l = [1,2,3] + check_annotation(l, checker) + return l + + def g(): + l = f() + l.append(4) + + a = self.RPythonAnnotator() + py.test.raises(TooLateForChange, a.build_types, g, []) + assert called + def g(n): return [0,1,2,n] From fijal at codespeak.net Sun Sep 7 13:14:42 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 13:14:42 +0200 (CEST) Subject: [pypy-svn] r57941 - in pypy/branch/tuple-nonresizable-395/pypy/annotation: . test Message-ID: <20080907111442.1A15D16A16E@codespeak.net> Author: fijal Date: Sun Sep 7 13:14:40 2008 New Revision: 57941 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Log: Rename dont_resize_any_more to never_resize and also check for the fact that list should not be resizable at all Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py Sun Sep 7 13:14:40 2008 @@ -182,7 +182,9 @@ self.listitem.mutate() self.listitem.resize() - def dont_resize_any_more(self): + def never_resize(self): + if self.listitem.resized: + raise TooLateForChange() self.listitem.dont_resize = True Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sun Sep 7 13:14:40 2008 @@ -3047,7 +3047,7 @@ def checker(ann, bk): called.append(True) assert not ann.listdef.listitem.mutated - ann.listdef.dont_resize_any_more() + ann.listdef.never_resize() def f(): l = [1,2,3] From fijal at codespeak.net Sun Sep 7 13:24:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 13:24:39 +0200 (CEST) Subject: [pypy-svn] r57943 - pypy/branch/tuple-nonresizable-395/pypy/annotation/test Message-ID: <20080907112439.96F1516A11D@codespeak.net> Author: fijal Date: Sun Sep 7 13:24:37 2008 New Revision: 57943 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Log: another test for never_resize Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sun Sep 7 13:24:37 2008 @@ -3062,6 +3062,21 @@ py.test.raises(TooLateForChange, a.build_types, g, []) assert called + def test_listitem_never_resize(self): + from pypy.rlib.debug import check_annotation + + def checker(ann, bk): + ann.listdef.never_resize() + + def f(): + l = [1,2,3] + l.append(4) + check_annotation(l, checker) + + a = self.RPythonAnnotator() + py.test.raises(TooLateForChange, a.build_types, f, []) + + def g(n): return [0,1,2,n] From fijal at codespeak.net Sun Sep 7 14:08:02 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 14:08:02 +0200 (CEST) Subject: [pypy-svn] r57944 - in pypy/branch/tuple-nonresizable-395/pypy/rlib: . test Message-ID: <20080907120802.7C2DF169F13@codespeak.net> Author: fijal Date: Sun Sep 7 14:07:59 2008 New Revision: 57944 Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py pypy/branch/tuple-nonresizable-395/pypy/rlib/test/test_debug.py Log: Add make_sure_not_resized Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py Sun Sep 7 14:07:59 2008 @@ -80,3 +80,21 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) +def make_sure_not_resized(arg): + """ Function checking whether annotation of SomeList is never resized, + useful for debugging. Does nothing when run directly + """ + pass + +class Entry(ExtRegistryEntry): + _about_ = make_sure_not_resized + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeList + assert isinstance(s_arg, SomeList) + s_arg.listdef.never_resize() + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/rlib/test/test_debug.py Sun Sep 7 14:07:59 2008 @@ -1,6 +1,6 @@ import py -from pypy.rlib.debug import check_annotation +from pypy.rlib.debug import check_annotation, make_sure_not_resized from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -27,3 +27,12 @@ py.test.raises(Error, "interpret(g, [3])") +def test_make_sure_not_resized(): + from pypy.annotation.listdef import TooLateForChange + def f(): + result = [1,2,3] + make_sure_not_resized(result) + result.append(4) + return len(result) + + py.test.raises(TooLateForChange, interpret, f, []) From fijal at codespeak.net Sun Sep 7 15:44:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 15:44:50 +0200 (CEST) Subject: [pypy-svn] r57945 - in pypy/branch/tuple-nonresizable-395/pypy/annotation: . test Message-ID: <20080907134450.2EAA416A183@codespeak.net> Author: fijal Date: Sun Sep 7 15:44:46 2008 New Revision: 57945 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Log: A logic that keeps track when merging resizable self with non-resizable other. fragile Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py Sun Sep 7 15:44:46 2008 @@ -63,8 +63,11 @@ # things more general self, other = other, self + if self.resized and other.dont_resize: + raise TooLateForChange() if other.mutated: self.mutate() - if other.resized: self.resize() + if other.resized: + self.resize() if other.range_step != self.range_step: self.setrangestep(self._step_map[type(self.range_step), type(other.range_step)]) Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sun Sep 7 15:44:46 2008 @@ -3062,6 +3062,28 @@ py.test.raises(TooLateForChange, a.build_types, g, []) assert called + def test_listitem_no_mutating2(self): + from pypy.rlib.debug import make_sure_not_resized + + def f(): + return make_sure_not_resized([1,2,3]) + + def g(): + l = [1,2,3] + l.append(4) + return l + + def fn(i): + if i: + func = f + else: + func = g + return func() + + a = self.RPythonAnnotator() + py.test.raises(TooLateForChange, a.build_types, fn, [int]) + + def test_listitem_never_resize(self): from pypy.rlib.debug import check_annotation From fijal at codespeak.net Sun Sep 7 16:31:35 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 16:31:35 +0200 (CEST) Subject: [pypy-svn] r57950 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080907143135.7B77116A1A8@codespeak.net> Author: fijal Date: Sun Sep 7 16:31:33 2008 New Revision: 57950 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: A checks for non-resizable list Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py Sun Sep 7 16:31:33 2008 @@ -8,6 +8,7 @@ from pypy.interpreter.pyopcode import unrolling_compare_dispatch_table, \ BytecodeCorruption from pypy.rlib.objectmodel import instantiate +from pypy.rlib.debug import make_sure_not_resized from pypy.interpreter.gateway import PyPyCacheDir from pypy.tool.cache import Cache from pypy.tool.sourcetools import func_with_new_name @@ -563,6 +564,7 @@ def newtuple(self, list_w): from pypy.objspace.std.tupletype import wraptuple assert isinstance(list_w, list) + make_sure_not_resized(list_w) return wraptuple(self, list_w) def newlist(self, list_w): Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Sun Sep 7 16:31:33 2008 @@ -3,11 +3,14 @@ from pypy.rlib.rarithmetic import intmask from pypy.objspace.std.sliceobject import W_SliceObject from pypy.interpreter import gateway +from pypy.rlib.debug import make_sure_not_resized +from pypy.annotation import model as annmodel class W_TupleObject(W_Object): from pypy.objspace.std.tupletype import tuple_typedef as typedef def __init__(w_self, wrappeditems): + make_sure_not_resized(wrappeditems) w_self.wrappeditems = wrappeditems # a list of wrapped values def __repr__(w_self): From witulski at codespeak.net Sun Sep 7 18:58:37 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Sun, 7 Sep 2008 18:58:37 +0200 (CEST) Subject: [pypy-svn] r57954 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080907165837.7C2BE16A18F@codespeak.net> Author: witulski Date: Sun Sep 7 18:58:36 2008 New Revision: 57954 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/objmodel.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: Added MUL and tests FIXED IMM32 Problem ( supports now values grather than 255) Add new Add test for big nums (test is broken) Add CMP and SETG Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Sun Sep 7 18:58:36 2008 @@ -1,4 +1,5 @@ -from pypy.jit.codegen.x86_64.objmodel import Register64, Immediate32 +from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32 + #Mapping from register to coding (Rex.W or Rex.B , ModRM) REGISTER_MAP = { @@ -19,14 +20,23 @@ "r14": (1, 6), "r15": (1, 7), } - + +REGISTER_MAP_8BIT = { + "al":0, + "cl":1, + "dl":2, + } + # This method wirtes the bitencodings into # the memory. The parameters are overwritten # if one of the operands is an register -def make_two_operand_instr(W = None, R = None, X = None, B = None, opcode =None, md1 = None, md2 = None): +# tttn codes the flags and is only used by SETcc +# extra is an extra byte for long opcodes like imul +def make_two_operand_instr(W = None, R = None, X = None, B = None, opcode =None, m = None, md1 = None, md2 = None, tttn = None, extra = None): def quadreg_instr(self, arg1, arg2): # move the parameter # to the inner function + mod = m modrm1 = md1 modrm2 = md2 rexW = W @@ -36,37 +46,44 @@ # Todo: other cases e.g memory as operand if isinstance(arg1,Register64): rexR, modrm1 = self.get_register_bits(arg1.reg) - - if isinstance(arg2,Register64): - rexB, modrm2 = self.get_register_bits(arg2.reg) + elif isinstance(arg1,Register8): + modrm1 = self.get_register_bits_8Bit(arg1.reg) # exchange the two arguments (modrm2/modrm1) if isinstance(arg2,Immediate32): + # e.g: IMUL + if(modrm2=="sameReg"): + modrm2 = modrm1 + rexB = rexR self.write_rex_byte(rexW, rexR, rexX, rexB) self.write(opcode) self.write_modRM_byte(3, modrm2, modrm1) - # FIXME: Bad solution - # TODO: support values > 255 - if(arg2.value<256): - self.write(chr(arg2.value)) - self.write(chr(0)) - self.write(chr(0)) - self.write(chr(0)) - else: + self.writeImm32(arg2.value) + elif isinstance(arg2,Immediate8): + self.write_rex_byte(rexW, rexR, rexX, rexB) + self.write(opcode) + self.write_modRM_byte(3, modrm2, modrm1) + self.write(chr(arg2.value)) + elif isinstance(arg2,Register64): + rexB, modrm2 = self.get_register_bits(arg2.reg) # FIXME: exchange the two arguments (rexB/rexR) self.write_rex_byte(rexW, rexB, rexX, rexR) self.write(opcode) - self.write_modRM_byte(3, modrm2, modrm1) + # used in imul + if not extra == None: + self.write(extra) + self.write_modRM_byte(mod, modrm2, modrm1) return quadreg_instr # This method wirtes the bitencodings into # the memory. The parameters are overwritten # if one of the operands is an register -def make_one_operand_instr(W = None, R = None, X = None, B = None, opcode = None, md1 = None, md2 = None): +def make_one_operand_instr(W = None, R = None, X = None, B = None, opcode = None, m = None, md1 = None, md2 = None, tttn=None, extra = None): def quadreg_instr(self, arg1): # move the parameter # to the inner function + mod = m modrm1 = md1 modrm2 = md2 rexW = W @@ -77,13 +94,18 @@ # Todo: other cases e.g memory as operand if isinstance(arg1,Register64): rexB, modrm1 = self.get_register_bits(arg1.reg) + if isinstance(arg1,Register8): + modrm1 = self.get_register_bits_8Bit(arg1.reg) # rexW(1) = 64bitMode self.write_rex_byte(rexW, rexR, rexX, rexB) self.write(opcode) - self.write_modRM_byte(3, modrm2, modrm1) + if not tttn == None: + byte = (9 << 4) | tttn + self.write(chr(byte)) + self.write_modRM_byte(mod, modrm2, modrm1) return quadreg_instr - + class X86_64CodeBuilder(object): """ creats x86_64 opcodes""" def write(self, data): @@ -96,29 +118,47 @@ # The opcodes differs depending on the operands + # Params: + # W, R, X, B, Opcode, mod, modrm1, modrm2 # FIXME: rexX,rexB are set - _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", None, 2) - _ADD_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x00", None, None) + _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 2) + _ADD_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x00", 3, None, None) - _DEC_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", None, 1) - _INC_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", None, 0) + # FIXME: rexB is set + _CMP_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 1, "\x81", 3, None, 7) + _CMP_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x39", 3, None, None) + # FIXME: rex B is set + _CMP_8REG_IMM8 = make_two_operand_instr( 0, 0, 0, 0, "\x82", 3, None, 7) + + _DEC_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 1) + _INC_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 0) - _MOV_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, None, "\xC7", None, 0) - _MOV_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x89", None, None) + _MOV_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, None, "\xC7", 3, None, 0) + _MOV_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x89", 3, None, None) + + _IMUL_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x0F", 3, None, None, None, "\xAF") + _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") # FIXME: rexW is set - _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", None, 0) - _PUSH_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", None, 6) + _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", 3, None, 0) + _PUSH_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 6) - _SUB_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x28", None, None) + _SETG_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,15) + + _SUB_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x28", 3, None, None) + _SUB_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 5) # TODO: maybe a problem with more ore less than two arg. def ADD(self, op1, op2): method = getattr(self, "_ADD"+op1.to_string()+op2.to_string()) method(op1, op2) + def CMP(self, op1, op2): + method = getattr(self, "_CMP"+op1.to_string()+op2.to_string()) + method(op1, op2) + def DEC(self, op1): method = getattr(self, "_DEC"+op1.to_string()) method(op1) @@ -127,6 +167,10 @@ method = getattr(self, "_INC"+op1.to_string()) method(op1) + def JMP(self,displ): + self.write("\xE9") + self.writeImm32(displ) + def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) method(op1) @@ -138,10 +182,23 @@ def MOV(self, op1, op2): method = getattr(self, "_MOV"+op1.to_string()+op2.to_string()) method(op1, op2) + + def IMUL(self, op1, op2): + method = getattr(self, "_IMUL"+op1.to_string()+op2.to_string()) + # exchange the two arguments because + # the result is in the first register + if(op1.to_string()=="_QWREG" and op2.to_string()=="_QWREG"): + method(op2, op1) + else: + method(op1, op2) def RET(self): self.write("\xC3") + def SETG(self, op1): + method = getattr(self, "_SETG"+op1.to_string()) + method(op1) + def SUB(self, op1, op2): method = getattr(self, "_SUB"+op1.to_string()+op2.to_string()) method(op1, op2) @@ -149,6 +206,22 @@ def get_register_bits(self, register): return REGISTER_MAP[register] + def get_register_bits_8Bit(self, register): + return REGISTER_MAP_8BIT[register] + + # Parse the integervalue to an charakter + # and write it + def writeImm32(self, imm32): + x = hex(imm32) + # parse to string and cut "0x" off + # fill with zeros if to short + y = "0"*(10-len(x))+x[2:len(x)] + assert len(y) == 8 + self.write(chr(int(y[6:8],16))) + self.write(chr(int(y[4:6],16))) + self.write(chr(int(y[2:4],16))) + self.write(chr(int(y[0:2],16))) + # Rex-Prefix 4WRXB see AMD vol3 page 45 def write_rex_byte(self, rexW, rexR, rexX, rexB): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/objmodel.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/objmodel.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/objmodel.py Sun Sep 7 18:58:36 2008 @@ -1,7 +1,17 @@ from pypy.jit.codegen import model +from pypy.rpython.lltypesystem import lltype, rffi, llmemory # Wrapper Classes # The opcaodes differ from the type of # the operand. So every wrapper is necessary +# The to string method is used to choose the right +# method inside the assembler +class Register8(model.GenVar): + def __init__(self, reg): + self.reg = reg + + def to_string(self): + return "_8REG" + class Register64(model.GenVar): def __init__(self, reg): self.reg = reg @@ -9,6 +19,13 @@ def to_string(self): return "_QWREG" +class Immediate8(model.GenConst): + def __init__(self, value): + self.value = value + + def to_string(self): + return "_IMM8" + class Immediate32(model.GenConst): def __init__(self, value): self.value = value Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Sun Sep 7 18:58:36 2008 @@ -1,16 +1,17 @@ from pypy.jit.codegen import model from pypy.rlib.objectmodel import specialize -from pypy.jit.codegen.x86_64.objmodel import Register64, Immediate32 +from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32 from pypy.jit.codegen.x86_64.codebuf import InMemoryCodeBuilder #TODO: understand llTypesystem from pypy.rpython.lltypesystem import llmemory, lltype from pypy.jit.codegen.ia32.objmodel import LL_TO_GENVAR +from pypy.jit.codegen.model import GenLabel # TODO: support zero arg. -# This method calles the assembler to generate code. +# This method calls the assembler to generate code. # It saves the operands in the helpregister gv_z # and determine the Type of the operands, # to choose the right method in assembler.py @@ -39,11 +40,18 @@ arg = lltype.Void return LL_TO_GENVAR[arg] +class Label(GenLabel): + def __init__(self, startaddr, arg_positions, stackdepth): + self.startaddr = startaddr + self.arg_positions = arg_positions + self.stackdepth = stackdepth + class Builder(model.GenBuilder): MC_SIZE = 65536 + #FIXME: The MemCodeBuild. is not opend in an _open method def __init__(self): self.mc = InMemoryCodeBuilder(self.MC_SIZE) #callee-saved registers are commented out @@ -51,18 +59,21 @@ "rax":None, "rcx":None, "rdx":None, - # "rbx":None, + # "rbx":None, "rsi":None, "rdi":None, "r8": None, "r9": None, "r10":None, - # "r11":None, - # "r12":None, - # "r13":None, - # "r14":None, - # "r15":None, + # "r11":None, + # "r12":None, + # "r13":None, + # "r14":None, + # "r15":None, } + + def _open(self): + pass @specialize.arg(1) def genop1(self, opname, gv_arg): @@ -75,16 +86,42 @@ return genmethod(gv_arg1, gv_arg2) op_int_add = make_two_argument_method("ADD") - op_int_sub = make_two_argument_method("SUB") - op_int_inc = make_one_argument_method("INC") op_int_dec = make_one_argument_method("DEC") + op_int_inc = make_one_argument_method("INC") + op_int_mul = make_two_argument_method("IMUL") op_int_push = make_one_argument_method("PUSH") op_int_pop = make_one_argument_method("POP") + op_int_sub = make_two_argument_method("SUB") + + def jump_if_true(self, gv_condition, args_for_jump_gv): + targetbuilder = Builder() + self.mc.CMP(gv_condition, Immediate8(0)) + #targetbuilder.come_from(self.mc, 'JNE') + return targetbuilder + + def op_int_gt(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + # You can not use evry register for + # 8 bit operations, so you have to + # choose rax,rcx or rdx + # TODO: rcx rdx + gv_z = self.allocate_register("rax") + self.mc.SETG(Register8("al")) + return Register8("al") def finish_and_return(self, sigtoken, gv_returnvar): #self.mc.write("\xB8\x0F\x00\x00\x00") + self._open() self.mc.MOV(Register64("rax"), gv_returnvar) self.mc.RET() + self._close() + + #TODO: Implementation + def finish_and_goto(self, outputargs_gv, target): + self._open() + #FIXME: startaddr is maybe not 32bit + self.mc.JMP(target.startaddr) + self._close() def allocate_register(self, register=None): if register is None: @@ -97,6 +134,14 @@ def end(self): pass + + #TODO: Implementation + def enter_next_block(self, args_gv): + print "WriteMe: enter_next_block" + return Label(self.mc.tell(), [], 0) + + def _close(self): + pass class RX86_64GenOp(model.AbstractRGenOp): @@ -110,7 +155,7 @@ # wrappes a integer value def genconst(self, llvalue): T = lltype.typeOf(llvalue) - # TODO: other cases(?) + # TODO: other cases(?),imm64 if T is lltype.Signed: return Immediate32(llvalue) @@ -122,6 +167,7 @@ entrypoint = builder.mc.tell() # TODO: support more than two reg register_list = ["rdi","rsi"] + # fill the list with the correct registers inputargs_gv = [builder.allocate_register(register_list[i]) for i in range(len(arg_tokens))] return builder,Immediate32(entrypoint), inputargs_gv Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Sun Sep 7 18:58:36 2008 @@ -10,6 +10,24 @@ def skip(self): py.test.skip("not implemented yet") +def make_mul(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_mul, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") + builder.start_writing() + gv_result = builder.genop2("int_mul", gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_mul + +def make_mul_im32(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_mul, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") + builder.start_writing() + gv_result = builder.genop2("int_mul", gv_x, rgenop.genconst(200)) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_mul + def make_inc(rgenop): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) builder, gv_inc, gv_x = rgenop.newgraph(sigtoken, "inc") @@ -28,29 +46,63 @@ builder.end() return gv_dec +def make_push(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) + builder, gv_push, gv_x = rgenop.newgraph(sigtoken, "push") + builder.start_writing() + gv_result = builder.genop1("int_push", gv_x[0]) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_push + +def make_pop(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) + builder, gv_pop, gv_x = rgenop.newgraph(sigtoken, "pop") + builder.start_writing() + gv_result = builder.genop1("int_pop", gv_x[0]) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_pop + class TestRGenopDirect(AbstractRGenOpTestsDirect): RGenOp = RX86_64GenOp def test_inc(self): rgenop = self.RGenOp() - inc_result = make_inc(rgenop) - fnptr = self.cast(inc_result,1) + inc_function = make_inc(rgenop) + fnptr = self.cast(inc_function,1) res = fnptr(0) assert res == 1 def test_dec(self): rgenop = self.RGenOp() - dec_result = make_dec(rgenop) - fnptr = self.cast(dec_result,1) + dec_function = make_dec(rgenop) + fnptr = self.cast(dec_function,1) res = fnptr(2) assert res == 1 - #def test_push_and_pop(self): - # rgenop = self.RGenOp() - # push_result = make_push(rgenop) - # fnptr = self.cast(push_result,1) - # res = fnptr(2) - # assert res == 1 + def test_mul_im32(self): + rgenop = self.RGenOp() + mul_function = make_mul_im32(rgenop) + fnptr = self.cast(mul_function,1) + res = fnptr(210) + assert res == 42000 + + def test_mul(self): + rgenop = self.RGenOp() + mul_function = make_mul(rgenop) + fnptr = self.cast(mul_function,2) + res = fnptr(1200,300) + assert res == 360000 + + # def test_push_and_pop(self): + # rgenop = self.RGenOp() + # push_result = make_push(rgenop) + # fnptr = self.cast(push_result,1) + # pop_result = make_pop(rgenop) + # fnptr = self.cast(pop_result,1) + # res = fnptr(42) + #assert res == 1 test_directtesthelper_direct = skip test_dummy_compile = skip @@ -61,7 +113,7 @@ test_dummy_direct = skip test_largedummy_direct = skip test_branching_direct = skip - test_goto_direct = skip + ##test_goto_direct = skip## test_if_direct = skip test_switch_direct = skip test_large_switch_direct = skip Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Sun Sep 7 18:58:36 2008 @@ -8,18 +8,28 @@ rgenop = RX86_64GenOp() -def make_testbuilder(): - FUNC = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) #the funtiontype(arguments,returntype) of the graph we will create +def make_testbuilder(num_of_args): + FUNC = lltype.FuncType([lltype.Signed]*num_of_args, lltype.Signed) #the funtiontype(arguments,returntype) of the graph we will create token = rgenop.sigToken(FUNC) builder, entrypoint, inputargs_gv = rgenop.newgraph(token, "test") builder.start_writing() - ctypestypes = [c_long, c_long] + ctypestypes = [c_long]*num_of_args fp = cast(c_void_p(entrypoint.value), CFUNCTYPE(c_long, *ctypestypes)) return builder, fp, inputargs_gv, token +def test_add_big_num(): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "place" + genv1 = inputargs_gv[1] + genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + num = fp(1280, 1000) + assert num == 2280 + print num + def test_add(): - builder, fp, inputargs_gv, token = make_testbuilder() + builder, fp, inputargs_gv, token = make_testbuilder(2) genv0 = inputargs_gv[0] #the first argument "place" genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result @@ -28,20 +38,38 @@ assert ten == 10 print ten +def test_add_imm32(): + builder, fp, inputargs_gv, token = make_testbuilder(1) + genv0 = inputargs_gv[0] #the first argument "place" + genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + num = fp(1111) + assert num == 2111 + print num + def test_ret(): - builder, fp, inputargs_gv, token = make_testbuilder() + builder, fp, inputargs_gv, token = make_testbuilder(1) builder.finish_and_return(token, inputargs_gv[0]) print repr("".join(builder.mc._all)) - four = fp(4, 17) + four = fp(4) assert four == 4 print four def test_sub(): - builder, fp, inputargs_gv, token = make_testbuilder() + builder, fp, inputargs_gv, token = make_testbuilder(2) genv0 = inputargs_gv[0] #the first argument "place" genv1 = inputargs_gv[1] - genv_result = builder.genop2("int_sub", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result + genv_result = builder.genop2("int_sub", genv0, genv1) #creates the subtraction and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) four = fp(10, 6) assert four == 4 - print four \ No newline at end of file + print four + +def test_sub_imm32(): + builder, fp, inputargs_gv, token = make_testbuilder(1) + genv0 = inputargs_gv[0] #the first argument "place" + genv_result = builder.genop2("int_sub", genv0, rgenop.genconst(2)) #creates the subtraction and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + eight = fp(10) + assert eight == 8 + print eight \ No newline at end of file From fijal at codespeak.net Sun Sep 7 20:21:53 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 7 Sep 2008 20:21:53 +0200 (CEST) Subject: [pypy-svn] r57955 - in pypy/branch/tuple-nonresizable-395/pypy/annotation: . test Message-ID: <20080907182153.3003C16A152@codespeak.net> Author: fijal Date: Sun Sep 7 20:21:51 2008 New Revision: 57955 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py Log: Start of a mergeable ListItemIterator. Breaks tons of stuff, but annotation passes Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py Sun Sep 7 20:21:51 2008 @@ -20,6 +20,7 @@ from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import SomeGenericCallable from pypy.annotation.model import SomeExternalInstance, SomeUnicodeString +from pypy.annotation.model import SomeListIterator from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.rlib import rarithmetic @@ -778,6 +779,17 @@ raise UnionError("merging incompatible iterators variants") return SomeIterator(s_cont, *iter1.variant) +class __extend__(pairtype(SomeListIterator, SomeListIterator)): + def union((iter1, iter2)): + # merge here s_value of listdefs, but not listdefs themselves + # (ie resizable and mergeable parameters) + # XXX should we perform some better union??? + for listdef in iter1.listdefs: + listdef.generalize(iter2.listdefs[0].listitem.s_value) + for listdef in iter2.listdefs: + listdef.generalize(iter1.listdefs[0].listitem.s_value) + listdefs = dict.fromkeys(iter1.listdefs + iter2.listdefs).keys() + return SomeListIterator(listdefs) class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/listdef.py Sun Sep 7 20:21:51 2008 @@ -63,8 +63,10 @@ # things more general self, other = other, self - if self.resized and other.dont_resize: - raise TooLateForChange() + if other.dont_resize: + if self.resized: + raise TooLateForChange() + self.dont_resize = True if other.mutated: self.mutate() if other.resized: self.resize() Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py Sun Sep 7 20:21:51 2008 @@ -323,6 +323,15 @@ def can_be_none(self): return False +class SomeListIterator(SomeObject): + def __init__(self, listdefs): + self.listdefs = listdefs + + def __eq__(self, other): + if not isinstance(other, SomeListIterator): + return False + return dict.fromkeys(self.listdefs) == dict.fromkeys(other.listdefs) + class SomeInstance(SomeObject): "Stands for an instance of a (user-defined) class." Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sun Sep 7 20:21:51 2008 @@ -443,7 +443,7 @@ def test_simple_iter_list(self): a = self.RPythonAnnotator() s = a.build_types(snippet.simple_iter, [list]) - assert isinstance(s, annmodel.SomeIterator) + assert isinstance(s, annmodel.SomeListIterator) def test_simple_iter_next(self): def f(x): @@ -3097,7 +3097,19 @@ a = self.RPythonAnnotator() py.test.raises(TooLateForChange, a.build_types, f, []) - + + def test_mixing_iterators(self): + def f(i): + if i: + return iter([1,2,3]) + else: + l = [1,2,3] + l.append(4) + return iter(l) + + a = self.RPythonAnnotator() + s_res = a.build_types(f, [int]) + assert isinstance(s_res, annmodel.SomeListIterator) def g(n): return [0,1,2,n] Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py Sun Sep 7 20:21:51 2008 @@ -9,7 +9,7 @@ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable, SomeWeakRef, SomeUnicodeString + SomeGenericCallable, SomeWeakRef, SomeUnicodeString, SomeListIterator from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -315,7 +315,7 @@ return SomeObject.len(lst) def iter(lst): - return SomeIterator(lst) + return SomeListIterator([lst.listdef]) iter.can_only_throw = [] def getanyitem(lst): @@ -529,6 +529,15 @@ next.can_only_throw = _can_only_throw method_next = next +class __extend__(SomeListIterator): + def iter(itr): + return itr + iter.can_only_throw = [] + + def next(itr): + return itr.listdefs[0].read_item() + next.can_only_throw = [StopIteration] + method_next = next class __extend__(SomeInstance): From fijal at codespeak.net Mon Sep 8 12:11:05 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 12:11:05 +0200 (CEST) Subject: [pypy-svn] r57960 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908101105.58E5916A22C@codespeak.net> Author: fijal Date: Mon Sep 8 12:11:03 2008 New Revision: 57960 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: Add a link to django's wiki Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Mon Sep 8 12:11:03 2008 @@ -80,6 +80,8 @@ * http://www.djangoproject.com/ +* http://code.djangoproject.com/wiki/DjangoAndPyPy + Pylons ====== From hpk at codespeak.net Mon Sep 8 12:14:38 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 8 Sep 2008 12:14:38 +0200 (CEST) Subject: [pypy-svn] r57961 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908101438.6214E16A239@codespeak.net> Author: hpk Date: Mon Sep 8 12:14:36 2008 New Revision: 57961 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: last fixes from my side Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Mon Sep 8 12:14:36 2008 @@ -1,8 +1,6 @@ -====================== -PyPy status talk -====================== - -XXX maybe missing: threading improvements? more on GCs? +================================ +PyPy Python Interpreter status +================================ What this talk is about ======================= @@ -234,10 +232,8 @@ 1.1 release goals =================================== -XXX check/review/complete - - compatible to Python 2.5.2 -- well tested on win/linux/osx +- well tested on win/linux 32 bit - running some major packages unmodified - easy_install/distutils working - deliver pypy-c binary installs windows @@ -245,10 +241,9 @@ Contact / Q&A ========================== +holger krekel, Maciej Fijalkowski +at http://merlinux.eu -* http://codespeak.net/pypy - -* http://morepypy.blogspot.com +PyPy: http://codespeak.net/pypy -holger krekel, Maciej Fijalkowski - +Blog: http://morepypy.blogspot.com From fijal at codespeak.net Mon Sep 8 12:16:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 12:16:50 +0200 (CEST) Subject: [pypy-svn] r57962 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908101650.EBF7C16A23D@codespeak.net> Author: fijal Date: Mon Sep 8 12:16:49 2008 New Revision: 57962 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: * URLs fell of the line, don't use * * add "64bit and os/x likely" Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Mon Sep 8 12:16:49 2008 @@ -76,9 +76,9 @@ * only sqlite DB backend for now -* http://www.djangoproject.com/ +http://www.djangoproject.com -* http://code.djangoproject.com/wiki/DjangoAndPyPy +http://code.djangoproject.com/wiki/DjangoAndPyPy Pylons ====== @@ -234,6 +234,7 @@ - compatible to Python 2.5.2 - well tested on win/linux 32 bit +- os/x and 64 also targets - running some major packages unmodified - easy_install/distutils working - deliver pypy-c binary installs windows From fijal at codespeak.net Mon Sep 8 12:19:44 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 12:19:44 +0200 (CEST) Subject: [pypy-svn] r57963 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908101944.77D0C16A23A@codespeak.net> Author: fijal Date: Mon Sep 8 12:19:40 2008 New Revision: 57963 Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex Log: change author.latex to reflect title Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/author.latex Mon Sep 8 12:19:40 2008 @@ -1,6 +1,6 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy and The Art of Generating VMs]{PyPy and The Art of Generating Virtual Machines} +\title[PyPy status talk]{PyPy status talk} \author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski\\Merlinux GmbH} \institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} From witulski at codespeak.net Mon Sep 8 13:53:33 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Mon, 8 Sep 2008 13:53:33 +0200 (CEST) Subject: [pypy-svn] r57966 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080908115333.81D4C16A20A@codespeak.net> Author: witulski Date: Mon Sep 8 13:53:29 2008 New Revision: 57966 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: added a new compare greater test (int_gt; CMP) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Mon Sep 8 13:53:29 2008 @@ -1,7 +1,7 @@ from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32 -#Mapping from register to coding (Rex.W or Rex.B , ModRM) +#Mapping from 64Bit-Register to coding (Rex.W or Rex.B , ModRM) REGISTER_MAP = { "rax": (0, 0), "rcx": (0, 1), @@ -21,6 +21,7 @@ "r15": (1, 7), } +# Mapping from 8Bit-Register to coding REGISTER_MAP_8BIT = { "al":0, "cl":1, @@ -30,8 +31,8 @@ # This method wirtes the bitencodings into # the memory. The parameters are overwritten # if one of the operands is an register -# tttn codes the flags and is only used by SETcc -# extra is an extra byte for long opcodes like imul +# tttn isn't used yet +# extra is an extra byte for long opcodes like IMUL def make_two_operand_instr(W = None, R = None, X = None, B = None, opcode =None, m = None, md1 = None, md2 = None, tttn = None, extra = None): def quadreg_instr(self, arg1, arg2): # move the parameter @@ -43,7 +44,7 @@ rexR = R rexX = X rexB = B - # Todo: other cases e.g memory as operand + # TODO: other cases e.g memory as operand if isinstance(arg1,Register64): rexR, modrm1 = self.get_register_bits(arg1.reg) elif isinstance(arg1,Register8): @@ -51,7 +52,7 @@ # exchange the two arguments (modrm2/modrm1) if isinstance(arg2,Immediate32): - # e.g: IMUL + # e.g: IMUL (source==dest) if(modrm2=="sameReg"): modrm2 = modrm1 rexB = rexR @@ -69,7 +70,7 @@ # FIXME: exchange the two arguments (rexB/rexR) self.write_rex_byte(rexW, rexB, rexX, rexR) self.write(opcode) - # used in imul + # used in imul (extra long opcode) if not extra == None: self.write(extra) self.write_modRM_byte(mod, modrm2, modrm1) @@ -79,6 +80,7 @@ # This method wirtes the bitencodings into # the memory. The parameters are overwritten # if one of the operands is an register +# tttn codes the flags and is only used by SETcc def make_one_operand_instr(W = None, R = None, X = None, B = None, opcode = None, m = None, md1 = None, md2 = None, tttn=None, extra = None): def quadreg_instr(self, arg1): # move the parameter @@ -91,7 +93,7 @@ rexX = X rexB = B - # Todo: other cases e.g memory as operand + # TODO: other cases e.g memory as operand if isinstance(arg1,Register64): rexB, modrm1 = self.get_register_bits(arg1.reg) if isinstance(arg1,Register8): @@ -119,7 +121,7 @@ # The opcodes differs depending on the operands # Params: - # W, R, X, B, Opcode, mod, modrm1, modrm2 + # W, R, X, B, Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode # FIXME: rexX,rexB are set _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 2) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Mon Sep 8 13:53:29 2008 @@ -33,6 +33,7 @@ # a small helper that provides correct type signature +# used by sigtoken def map_arg(arg): if isinstance(arg, lltype.Ptr): arg = llmemory.Address @@ -101,13 +102,13 @@ def op_int_gt(self, gv_x, gv_y): self.mc.CMP(gv_x, gv_y) - # You can not use evry register for + # You can not use every register for # 8 bit operations, so you have to # choose rax,rcx or rdx # TODO: rcx rdx gv_z = self.allocate_register("rax") self.mc.SETG(Register8("al")) - return Register8("al") + return Register64("rax") def finish_and_return(self, sigtoken, gv_returnvar): #self.mc.write("\xB8\x0F\x00\x00\x00") Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Mon Sep 8 13:53:29 2008 @@ -9,7 +9,17 @@ def skip(self): py.test.skip("not implemented yet") + +def make_cmp_gt(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_cmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") + builder.start_writing() + gv_result = builder.genop2("int_gt", gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_cmp + def make_mul(rgenop): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_mul, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") @@ -63,10 +73,11 @@ builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_pop + class TestRGenopDirect(AbstractRGenOpTestsDirect): RGenOp = RX86_64GenOp - + def test_inc(self): rgenop = self.RGenOp() inc_function = make_inc(rgenop) @@ -95,6 +106,17 @@ res = fnptr(1200,300) assert res == 360000 + def test_greater(self): + rgenop = self.RGenOp() + cmp_function = make_cmp_gt(rgenop) + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3>4? + assert res == 0 # false + res = fnptr(4,3) + assert res == 1 + res = fnptr(4,4) + assert res == 0 + # def test_push_and_pop(self): # rgenop = self.RGenOp() # push_result = make_push(rgenop) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Mon Sep 8 13:53:29 2008 @@ -1,3 +1,5 @@ +# Some opcode simple tests + import py from pypy.jit.codegen.x86_64.rgenop import RX86_64GenOp from pypy.rpython.lltypesystem import lltype @@ -17,59 +19,61 @@ fp = cast(c_void_p(entrypoint.value), CFUNCTYPE(c_long, *ctypestypes)) return builder, fp, inputargs_gv, token - -def test_add_big_num(): - builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" - genv1 = inputargs_gv[1] - genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result - builder.finish_and_return(token, genv_result) - num = fp(1280, 1000) - assert num == 2280 - print num - -def test_add(): - builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" - genv1 = inputargs_gv[1] - genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result - builder.finish_and_return(token, genv_result) - ten = fp(4, 6) - assert ten == 10 - print ten - -def test_add_imm32(): - builder, fp, inputargs_gv, token = make_testbuilder(1) - genv0 = inputargs_gv[0] #the first argument "place" - genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result - builder.finish_and_return(token, genv_result) - num = fp(1111) - assert num == 2111 - print num - -def test_ret(): - builder, fp, inputargs_gv, token = make_testbuilder(1) - builder.finish_and_return(token, inputargs_gv[0]) - print repr("".join(builder.mc._all)) - four = fp(4) - assert four == 4 - print four - -def test_sub(): - builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" - genv1 = inputargs_gv[1] - genv_result = builder.genop2("int_sub", genv0, genv1) #creates the subtraction and returns the place(register) of the result in genv_result - builder.finish_and_return(token, genv_result) - four = fp(10, 6) - assert four == 4 - print four -def test_sub_imm32(): - builder, fp, inputargs_gv, token = make_testbuilder(1) - genv0 = inputargs_gv[0] #the first argument "place" - genv_result = builder.genop2("int_sub", genv0, rgenop.genconst(2)) #creates the subtraction and returns the place(register) of the result in genv_result - builder.finish_and_return(token, genv_result) - eight = fp(10) - assert eight == 8 - print eight \ No newline at end of file +class TestSimple(): + + def test_add_big_num(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "place" + genv1 = inputargs_gv[1] + genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + num = fp(1280, 1000) + assert num == 2280 + print num + + def test_add(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "place" + genv1 = inputargs_gv[1] + genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + ten = fp(4, 6) + assert ten == 10 + print ten + + def test_add_imm32(self): + builder, fp, inputargs_gv, token = make_testbuilder(1) + genv0 = inputargs_gv[0] #the first argument "place" + genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + num = fp(1111) + assert num == 2111 + print num + + def test_ret(self): + builder, fp, inputargs_gv, token = make_testbuilder(1) + builder.finish_and_return(token, inputargs_gv[0]) + print repr("".join(builder.mc._all)) + four = fp(4) + assert four == 4 + print four + + def test_sub(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "place" + genv1 = inputargs_gv[1] + genv_result = builder.genop2("int_sub", genv0, genv1) #creates the subtraction and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + four = fp(10, 6) + assert four == 4 + print four + + def test_sub_imm32(self): + builder, fp, inputargs_gv, token = make_testbuilder(1) + genv0 = inputargs_gv[0] #the first argument "place" + genv_result = builder.genop2("int_sub", genv0, rgenop.genconst(2)) #creates the subtraction and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + eight = fp(10) + assert eight == 8 + print eight \ No newline at end of file From fijal at codespeak.net Mon Sep 8 14:05:37 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 14:05:37 +0200 (CEST) Subject: [pypy-svn] r57967 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908120537.865A116A1A2@codespeak.net> Author: fijal Date: Mon Sep 8 14:05:34 2008 New Revision: 57967 Added: pypy/extradoc/talk/pycon-uk-2008/status/merlinux-logo.jpg (contents, props changed) pypy/extradoc/talk/pycon-uk-2008/status/title.latex Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex pypy/extradoc/talk/pycon-uk-2008/status/makepdf pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: Whack Whack whack until it works - logo insertion for first and last page. I'm not very happy with that though Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/author.latex Mon Sep 8 14:05:34 2008 @@ -1,7 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy status talk]{PyPy status talk} -\author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski\\Merlinux GmbH} +\author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski} \institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} \date{September 13 2008} Modified: pypy/extradoc/talk/pycon-uk-2008/status/makepdf ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/makepdf (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/makepdf Mon Sep 8 14:05:34 2008 @@ -9,5 +9,6 @@ BASE=status rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt $BASE.txt $BASE.latex || exit sed 's/\\date{}/\\input{author.latex}/' -i $BASE.latex || exit +sed 's/\\maketitle/\\input{title.latex}/' -i $BASE.latex || exit pdflatex $BASE.latex || exit Added: pypy/extradoc/talk/pycon-uk-2008/status/merlinux-logo.jpg ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Mon Sep 8 14:05:34 2008 @@ -248,3 +248,11 @@ PyPy: http://codespeak.net/pypy Blog: http://morepypy.blogspot.com + +.. raw:: latex + + \begin{figure} + \includegraphics[width=64px,height=64px]{merlinux-logo.jpg} + \qquad + \includegraphics[width=80px]{../../img/py-web.png} + \end{figure} Added: pypy/extradoc/talk/pycon-uk-2008/status/title.latex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-uk-2008/status/title.latex Mon Sep 8 14:05:34 2008 @@ -0,0 +1,7 @@ +\begin{titlepage} +\begin{figure}[h] +\includegraphics[width=64px,height=64px]{merlinux-logo.jpg} +\qquad +\includegraphics[width=80px]{../../img/py-web.png} +\end{figure} +\end{titlepage} From fijal at codespeak.net Mon Sep 8 14:09:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 14:09:12 +0200 (CEST) Subject: [pypy-svn] r57968 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908120912.5469016A227@codespeak.net> Author: fijal Date: Mon Sep 8 14:09:10 2008 New Revision: 57968 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: make this page a bit shorter Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Mon Sep 8 14:09:10 2008 @@ -47,8 +47,6 @@ * 32bit only by now -* demo - CTypes configure ================ From fijal at codespeak.net Mon Sep 8 14:15:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 14:15:43 +0200 (CEST) Subject: [pypy-svn] r57969 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080908121543.9911F16A230@codespeak.net> Author: fijal Date: Mon Sep 8 14:15:42 2008 New Revision: 57969 Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex Log: oops, forgotten this Modified: pypy/extradoc/talk/pycon-uk-2008/status/author.latex ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/author.latex (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/author.latex Mon Sep 8 14:15:42 2008 @@ -1,7 +1,8 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy status talk]{PyPy status talk} -\author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski} +\author[H. Krekel, M. Fijalkowski]{Holger Krekel \and Maciej Fijalkowski\\ +Merlinux GmbH} \institute[PyCon UK 2008]{PyCon UK 2008 - Birmingham} \date{September 13 2008} From fijal at codespeak.net Mon Sep 8 14:33:31 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 14:33:31 +0200 (CEST) Subject: [pypy-svn] r57970 - pypy/extradoc/talk/pycon-uk-2008/jit Message-ID: <20080908123331.AC4BF16A1FA@codespeak.net> Author: fijal Date: Mon Sep 8 14:33:30 2008 New Revision: 57970 Modified: pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.txt Log: Add Q/A slide Modified: pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.txt Mon Sep 8 14:33:30 2008 @@ -5,3 +5,21 @@ .. include:: motivation.txt .. include:: technical.txt + +Contact / Q&A +========================== + +Antonio Cuni, Maciej Fijalkowski +at http://merlinux.eu + +PyPy: http://codespeak.net/pypy + +Blog: http://morepypy.blogspot.com + +.. raw:: latex + + \begin{figure} + \includegraphics[width=64px,height=64px]{../status/merlinux-logo.jpg} + \qquad + \includegraphics[width=80px]{../../img/py-web.png} + \end{figure} From fijal at codespeak.net Mon Sep 8 16:20:55 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 16:20:55 +0200 (CEST) Subject: [pypy-svn] r57973 - in pypy/dist/pypy: module/posix rpython/module Message-ID: <20080908142055.7A1EB169FB7@codespeak.net> Author: fijal Date: Mon Sep 8 16:20:53 2008 New Revision: 57973 Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/rpython/module/ll_os.py Log: os.setegid Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Mon Sep 8 16:20:53 2008 @@ -101,6 +101,8 @@ interpleveldefs['seteuid'] = 'interp_posix.seteuid' if hasattr(os, 'setgid'): interpleveldefs['setgid'] = 'interp_posix.setgid' + if hasattr(os, 'setegid'): + interpleveldefs['setegid'] = 'interp_posix.setegid' # not visible via os, inconsistency in nt: if hasattr(posix, '_getfullpathname'): interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Sep 8 16:20:53 2008 @@ -603,6 +603,18 @@ return space.w_None setgid.unwrap_spec = [ObjSpace, int] +def setegid(space, arg): + """ setegid(gid) + + Set the current process's effective group id. + """ + try: + os.setegid(arg) + except OSError, e: + raise wrap_oserror(space, e) + return space.w_None +setgid.unwrap_spec = [ObjSpace, int] + def chroot(space, path): """ chroot(path) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Mon Sep 8 16:20:53 2008 @@ -470,6 +470,10 @@ def register_os_setgid(self): return self.extdef_for_function_int_to_int('setgid') + @registering_if(os, 'setegid') + def register_os_setegid(self): + return self.extdef_for_function_int_to_int('setegid') + @registering_if(os, 'getpid') def register_os_getpid(self): return self.extdef_for_os_function_returning_int('getpid') From fijal at codespeak.net Mon Sep 8 16:46:06 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 16:46:06 +0200 (CEST) Subject: [pypy-svn] r57974 - pypy/dist/pypy/module/posix Message-ID: <20080908144606.912A916A198@codespeak.net> Author: fijal Date: Mon Sep 8 16:46:03 2008 New Revision: 57974 Modified: pypy/dist/pypy/module/posix/interp_posix.py Log: typo :-/ Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Mon Sep 8 16:46:03 2008 @@ -613,7 +613,7 @@ except OSError, e: raise wrap_oserror(space, e) return space.w_None -setgid.unwrap_spec = [ObjSpace, int] +setegid.unwrap_spec = [ObjSpace, int] def chroot(space, path): """ chroot(path) From witulski at codespeak.net Mon Sep 8 16:57:41 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Mon, 8 Sep 2008 16:57:41 +0200 (CEST) Subject: [pypy-svn] r57975 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080908145741.7762B16A1C0@codespeak.net> Author: witulski Date: Mon Sep 8 16:57:40 2008 New Revision: 57975 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: MOV and JMP now support IMM64 values, but still crash sometimes with a seg. fault Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Mon Sep 8 16:57:40 2008 @@ -1,4 +1,4 @@ -from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32 +from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32, Immediate64 #Mapping from 64Bit-Register to coding (Rex.W or Rex.B , ModRM) @@ -108,6 +108,29 @@ self.write_modRM_byte(mod, modrm2, modrm1) return quadreg_instr +# TODO: comment +def make_two_operand_instr_with_alternate_encoding(W = None, R = None, X = None, B = None, opcode =None, md1 = None, md2 = None): + def quadreg_instr(self, arg1, arg2): + # move the parameter + # to the inner function + modrm1 = md1 + modrm2 = md2 + rexW = W + rexR = R + rexX = X + rexB = B + # TODO: other cases e.g memory as operand + # FIXME: rexB? + if isinstance(arg1,Register64): + rexB, modrm1 = self.get_register_bits(arg1.reg) + + # exchange the two arguments (modrm2/modrm1) + if isinstance(arg2,Immediate64): + self.write_rex_byte(rexW, rexR, rexX, rexB) + self.write(opcode+chr(modrm1)) + self.writeImm64(arg2.value) + return quadreg_instr + class X86_64CodeBuilder(object): """ creats x86_64 opcodes""" def write(self, data): @@ -139,10 +162,13 @@ _MOV_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, None, "\xC7", 3, None, 0) _MOV_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x89", 3, None, None) - + _MOV_QWREG_IMM64 = make_two_operand_instr_with_alternate_encoding(1,0,0,None,"\xB8",None,None) + _IMUL_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x0F", 3, None, None, None, "\xAF") _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") + _JMP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 4) + # FIXME: rexW is set _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", 3, None, 0) _PUSH_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 6) @@ -169,9 +195,12 @@ method = getattr(self, "_INC"+op1.to_string()) method(op1) - def JMP(self,displ): - self.write("\xE9") - self.writeImm32(displ) + # op1 must be a register + def JMP(self,op1): + method = getattr(self, "_JMP"+op1.to_string()) + method(op1) + #self.write("\xE9") + #self.writeImm32(displ) def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) @@ -223,6 +252,24 @@ self.write(chr(int(y[4:6],16))) self.write(chr(int(y[2:4],16))) self.write(chr(int(y[0:2],16))) + + + # Parse the integervalue to an charakter + # and write it + def writeImm64(self, imm64): + x = hex(imm64) + # parse to string and cut "0x" off + # fill with zeros if to short + y = "0"*(18-len(x))+x[2:len(x)] + assert len(y) == 16 + self.write(chr(int(y[14:16],16))) + self.write(chr(int(y[12:14],16))) + self.write(chr(int(y[10:12],16))) + self.write(chr(int(y[8:10],16))) + self.write(chr(int(y[6:8],16))) + self.write(chr(int(y[4:6],16))) + self.write(chr(int(y[2:4],16))) + self.write(chr(int(y[0:2],16))) # Rex-Prefix 4WRXB see AMD vol3 page 45 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Mon Sep 8 16:57:40 2008 @@ -1,6 +1,6 @@ from pypy.jit.codegen import model from pypy.rlib.objectmodel import specialize -from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32 +from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32, Immediate64 from pypy.jit.codegen.x86_64.codebuf import InMemoryCodeBuilder #TODO: understand llTypesystem from pypy.rpython.lltypesystem import llmemory, lltype @@ -20,7 +20,16 @@ gv_z = self.allocate_register() self.mc.MOV(gv_z, gv_x) method = getattr(self.mc, name) - method(gv_z, gv_y) + + # Many operations don't support + # 64 Bit Immmediates directly + if isinstance(gv_y,Immediate64): + gv_w = self.allocate_register() + self.mc.MOV(gv_w, gv_y) + method(gv_z, gv_w) + else: + method(gv_z, gv_y) + return gv_z return op_int @@ -96,7 +105,7 @@ def jump_if_true(self, gv_condition, args_for_jump_gv): targetbuilder = Builder() - self.mc.CMP(gv_condition, Immediate8(0)) + self.mc.CMP(gv_condition, Immediate32(0)) #targetbuilder.come_from(self.mc, 'JNE') return targetbuilder @@ -117,11 +126,13 @@ self.mc.RET() self._close() - #TODO: Implementation + # if the label is greater than 32bit + # it must be in a register def finish_and_goto(self, outputargs_gv, target): self._open() - #FIXME: startaddr is maybe not 32bit - self.mc.JMP(target.startaddr) + gv_x = self.allocate_register() + self.mc.MOV(gv_x,Immediate64(target.startaddr)) + self.mc.JMP(gv_x) self._close() def allocate_register(self, register=None): @@ -158,7 +169,10 @@ T = lltype.typeOf(llvalue) # TODO: other cases(?),imm64 if T is lltype.Signed: - return Immediate32(llvalue) + if llvalue > int("FFFFFFFF",16): + return Immediate64(llvalue) + else: + return Immediate32(llvalue) def newgraph(self, sigtoken, name): arg_tokens, res_token = sigtoken Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Mon Sep 8 16:57:40 2008 @@ -29,11 +29,11 @@ builder.end() return gv_mul -def make_mul_im32(rgenop): +def make_mul_imm(rgenop, num): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_mul, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") builder.start_writing() - gv_result = builder.genop2("int_mul", gv_x, rgenop.genconst(200)) + gv_result = builder.genop2("int_mul", gv_x, rgenop.genconst(num)) builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_mul @@ -77,7 +77,7 @@ class TestRGenopDirect(AbstractRGenOpTestsDirect): RGenOp = RX86_64GenOp - + def test_inc(self): rgenop = self.RGenOp() inc_function = make_inc(rgenop) @@ -94,11 +94,20 @@ def test_mul_im32(self): rgenop = self.RGenOp() - mul_function = make_mul_im32(rgenop) + mul_function = make_mul_imm(rgenop,200) fnptr = self.cast(mul_function,1) res = fnptr(210) assert res == 42000 + # segmentation fault at mov(qwreg,imm64) + + #def test_mul_im64(self): + # rgenop = self.RGenOp() + # mul_function = make_mul_imm(rgenop,int("123456789",16)) + # fnptr = self.cast(mul_function,1) + # res = fnptr(2) + # assert res == int("123456789",16)*2 + def test_mul(self): rgenop = self.RGenOp() mul_function = make_mul(rgenop) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Mon Sep 8 16:57:40 2008 @@ -21,7 +21,7 @@ return builder, fp, inputargs_gv, token class TestSimple(): - + def test_add_big_num(self): builder, fp, inputargs_gv, token = make_testbuilder(2) genv0 = inputargs_gv[0] #the first argument "place" @@ -76,4 +76,5 @@ builder.finish_and_return(token, genv_result) eight = fp(10) assert eight == 8 - print eight \ No newline at end of file + print eight + \ No newline at end of file From fijal at codespeak.net Mon Sep 8 17:05:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 17:05:58 +0200 (CEST) Subject: [pypy-svn] r57976 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080908150558.00D86169FB0@codespeak.net> Author: fijal Date: Mon Sep 8 17:05:54 2008 New Revision: 57976 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: More support for running cooperative benchmarks Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Mon Sep 8 17:05:54 2008 @@ -53,6 +53,7 @@ realos = os def __init__(self, name, args): + self.pid = -1 signal.signal(signal.SIGCHLD, lambda a,b: self.close()) self.pid = run_child(name, args) self.results = [] @@ -84,6 +85,32 @@ def __del__(self): self.close() +def run_cooperative(func): + """ The idea is that we fork() and execute the function func in one + process. In parent process we measure private memory obtained by a process + when we read anything in pipe. We send back that we did that + """ + parentread, childwrite = os.pipe() + childread, parentwrite = os.pipe() + pid = os.fork() + if not pid: + os.close(parentread) + os.close(parentwrite) + try: + func(lambda : os.read(childread, 1), + lambda x: os.write(childwrite, x)) + except (SystemExit, KeyboardInterrupt): + pass + except: + import traceback + traceback.print_tb(sys.exc_info()[2]) + os._exit(1) + os.close(childread) + os.close(childwrite) + return (lambda : os.read(parentread, 1), + lambda x: os.write(parentwrite, x), + pid) + def run_child(name, args): pid = os.fork() if not pid: Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Mon Sep 8 17:05:54 2008 @@ -23,6 +23,17 @@ '/lib/libncurses.so.5.6' : 60, } +def test_run_cooperative(): + def f(read, write): + x = read() + assert x == '3' + write('4') + + read, write, pid = bench_mem.run_cooperative(f) + assert pid + write('3') + assert read() == '4' + example_data = ''' 08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 Size: 988 kB From pedronis at codespeak.net Mon Sep 8 17:52:50 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 8 Sep 2008 17:52:50 +0200 (CEST) Subject: [pypy-svn] r57978 - in pypy/branch/pypy-pytrunk/pypy: . tool/pytest Message-ID: <20080908155250.AE0BA16A135@codespeak.net> Author: pedronis Date: Mon Sep 8 17:52:49 2008 New Revision: 57978 Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py Log: (iko, pedronis) start at making the app tests print app-level tracebacks on failure Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Mon Sep 8 17:52:49 2008 @@ -267,6 +267,11 @@ return space +class AppError(Exception): + + def __init__(self, excinfo): + self.excinfo = excinfo + class PyPyTestFunction(py.test.collect.Function): # All PyPy test items catch and display OperationErrors specially. # @@ -279,8 +284,13 @@ raise OpErrKeyboardInterrupt, OpErrKeyboardInterrupt(), tb appexcinfo = appsupport.AppExceptionInfo(space, e) if appexcinfo.traceback: - raise Failed(excinfo=appsupport.AppExceptionInfo(space, e)) - raise + raise AppError(appexcinfo) + raise + + def repr_failure(self, excinfo, outerr): + if excinfo.errisinstance(AppError): + excinfo = excinfo.value.excinfo + return super(PyPyTestFunction, self).repr_failure(excinfo, outerr) _pygame_imported = False Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py Mon Sep 8 17:52:49 2008 @@ -29,6 +29,10 @@ return py.code.Source(self.path.read(mode="rU")) fullsource = property(fullsource, None, None, "Full source of AppCode") + def getargs(self): + # xxx WIP + return [] + class AppFrame(py.code.Frame): def __init__(self, space, pyframe): From pedronis at codespeak.net Mon Sep 8 18:25:30 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 8 Sep 2008 18:25:30 +0200 (CEST) Subject: [pypy-svn] r57983 - pypy/branch/pypy-pytrunk/pypy/tool/pytest Message-ID: <20080908162530.3CF7D16A115@codespeak.net> Author: pedronis Date: Mon Sep 8 18:25:29 2008 New Revision: 57983 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/pypy_test_failure_demo.py Log: (iko, pedronis) add a sanity example about app-level skips Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/pypy_test_failure_demo.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/pypy_test_failure_demo.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/pypy_test_failure_demo.py Mon Sep 8 18:25:29 2008 @@ -22,5 +22,9 @@ raises(SyntaxError, int, "hello") def app_test_raises_doesnt(): - raises(ValueError, int, 3) + raises(ValueError, int, 3) + +def app_test_skip(): + skip("skipped test") + From pedronis at codespeak.net Mon Sep 8 18:26:04 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 8 Sep 2008 18:26:04 +0200 (CEST) Subject: [pypy-svn] r57984 - pypy/branch/pypy-pytrunk/pypy Message-ID: <20080908162604.57DEA16A140@codespeak.net> Author: pedronis Date: Mon Sep 8 18:26:03 2008 New Revision: 57984 Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py Log: (iko, pedronis) tweak the exception traceback so that --pdb behaves like before Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Mon Sep 8 18:26:03 2008 @@ -279,12 +279,12 @@ try: target(*args) except OperationError, e: + tb = sys.exc_info()[2] if e.match(space, space.w_KeyboardInterrupt): - tb = sys.exc_info()[2] raise OpErrKeyboardInterrupt, OpErrKeyboardInterrupt(), tb appexcinfo = appsupport.AppExceptionInfo(space, e) if appexcinfo.traceback: - raise AppError(appexcinfo) + raise AppError, AppError(appexcinfo), tb raise def repr_failure(self, excinfo, outerr): From pedronis at codespeak.net Mon Sep 8 18:41:43 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 8 Sep 2008 18:41:43 +0200 (CEST) Subject: [pypy-svn] r57985 - pypy/branch/pypy-pytrunk/pypy/tool/pytest Message-ID: <20080908164143.5ABB816A217@codespeak.net> Author: pedronis Date: Mon Sep 8 18:41:42 2008 New Revision: 57985 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py Log: (iko, pedronis) stab at getargs for AppFrames Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/appsupport.py Mon Sep 8 18:41:42 2008 @@ -10,8 +10,8 @@ class AppCode(object): def __init__(self, space, pycode): - self.code = space.unwrap(space.getattr(pycode, space.wrap('co_code'))) - self.raw = self.code + self.code = pycode + self.raw = pycode self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) self.firstlineno = space.unwrap(space.getattr(pycode, space.wrap('co_firstlineno'))) @@ -30,8 +30,7 @@ fullsource = property(fullsource, None, None, "Full source of AppCode") def getargs(self): - # xxx WIP - return [] + return self.raw.co_varnames[:self.raw.co_argcount] class AppFrame(py.code.Frame): @@ -64,6 +63,15 @@ def is_true(self, w_value): return self.space.is_true(w_value) + def getargs(self): + space = self.space + retval = [] + for arg in self.code.getargs(): + w_val = space.getitem(self.w_locals, space.wrap(arg)) + retval.append((arg, w_val)) + return retval + + class AppExceptionInfo(py.code.ExceptionInfo): """An ExceptionInfo object representing an app-level exception.""" From arigo at codespeak.net Mon Sep 8 18:43:36 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 8 Sep 2008 18:43:36 +0200 (CEST) Subject: [pypy-svn] r57986 - pypy/branch/oo-jit/pypy/translator/tool Message-ID: <20080908164336.9E75116A1FD@codespeak.net> Author: arigo Date: Mon Sep 8 18:43:33 2008 New Revision: 57986 Modified: pypy/branch/oo-jit/pypy/translator/tool/make_dot.py Log: Add support for one more graph attribute. Modified: pypy/branch/oo-jit/pypy/translator/tool/make_dot.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/tool/make_dot.py (original) +++ pypy/branch/oo-jit/pypy/translator/tool/make_dot.py Mon Sep 8 18:43:33 2008 @@ -54,10 +54,11 @@ color="black", dir="forward", weight="5", + constraint="true", ): d = locals() attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) - for x in ['label', 'style', 'color', 'dir', 'weight']] + for x in ['label', 'style', 'color', 'dir', 'weight', 'constraint']] self.emit('edge [%s];' % ", ".join(attrs)) self.emit('%s -> %s' % (safename(name1), safename(name2))) From fijal at codespeak.net Mon Sep 8 20:55:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 8 Sep 2008 20:55:58 +0200 (CEST) Subject: [pypy-svn] r57988 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080908185558.AEFD516A176@codespeak.net> Author: fijal Date: Mon Sep 8 20:55:56 2008 New Revision: 57988 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: A helper to run couple of functions in one block Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Mon Sep 8 20:55:56 2008 @@ -111,6 +111,22 @@ lambda x: os.write(parentwrite, x), pid) +def measure(measure_func, funcs_to_run): + results = [] + for func in funcs_to_run: + read, write, pid = run_cooperative(func) + elem = [] + while 1: + res = read() + if res == 'e': + os.waitpid(pid, 0) + break + else: + elem.append(measure_func(pid)) + write('x') + results.append(elem) + return results + def run_child(name, args): pid = os.fork() if not pid: Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Mon Sep 8 20:55:56 2008 @@ -26,13 +26,27 @@ def test_run_cooperative(): def f(read, write): x = read() - assert x == '3' - write('4') + assert x == 'y' + write('x') read, write, pid = bench_mem.run_cooperative(f) assert pid - write('3') - assert read() == '4' + write('y') + assert read() == 'x' + +def test_measure_cooperative(): + def f1(read, write): + write('g') + read() + write('g') + read() + write('e') + + def measure(arg): + return 42 + + measurments = bench_mem.measure(measure, [f1, f1]) + assert measurments == [[42, 42], [42, 42]] example_data = ''' 08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 From cami at codespeak.net Mon Sep 8 21:30:51 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Mon, 8 Sep 2008 21:30:51 +0200 (CEST) Subject: [pypy-svn] r57989 - in pypy/dist/pypy/lang/gameboy: . profiling test Message-ID: <20080908193051.3E9E7169F61@codespeak.net> Author: cami Date: Mon Sep 8 21:30:47 2008 New Revision: 57989 Modified: pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/video.py Log: refactored video, introduced separated mode objects adapted tests according to the refactoring Modified: pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py (original) +++ pypy/dist/pypy/lang/gameboy/profiling/gameboy_profiling_implementation.py Mon Sep 8 21:30:47 2008 @@ -1,11 +1,13 @@ #!/usr/bin/env python from __future__ import generators +from pypy.lang.gameboy.ram import iMemory from pypy.lang.gameboy.gameboy_implementation import * from pypy.lang.gameboy.profiling.profiling_cpu import ProfilingCPU from pypy.lang.gameboy.debug import debug from pypy.lang.gameboy.debug.debug_socket_memory import * + # GAMEBOY ---------------------------------------------------------------------- class GameBoyProfilingImplementation(GameBoyImplementation): @@ -21,9 +23,10 @@ self.is_running = False debug.print_results() - + # CUSTOM DRIVER IMPLEMENTATIONS currently not used ============================= + # VIDEO DRIVER ----------------------------------------------------------------- class VideoDriverDebugImplementation(VideoDriverImplementation): Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Mon Sep 8 21:30:47 2008 @@ -227,7 +227,7 @@ video.transfer = False video.status.write(0xFE, write_all=True) video.cycles = 0 - video.emulate_oam() + video.status.mode2.emulate_oam() assert video.status.read(extend=True) == 0xFF assert video.cycles == constants.MODE_3_BEGIN_TICKS assert video.transfer == True @@ -238,7 +238,7 @@ video.transfer = False video.cycles = 0 video.status.write(0xF0, write_all=True) - video.emulate_transfer() + video.status.mode3.emulate_transfer() assert video.status.read(extend=True) == 0xF0 assert video.cycles == constants.MODE_0_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -247,7 +247,7 @@ video.cycles = 0 video.status.write(0xF8, write_all=True) assert not video.lcd_interrupt_flag.is_pending() - video.emulate_transfer() + video.status.mode3.emulate_transfer() assert video.status.read(extend=True) == 0xF8 assert video.cycles == constants.MODE_0_TICKS assert video.lcd_interrupt_flag.is_pending() @@ -255,7 +255,7 @@ video.transfer = True video.cycles = 0 video.status.write(0xFC, write_all=True) - video.emulate_transfer() + video.status.mode3.emulate_transfer() assert video.cycles == constants.MODE_3_END_TICKS assert video.transfer == False assert video.status.read(extend=True) == 0xFF @@ -267,7 +267,7 @@ video.line_y_compare = 0x13 video.status.line_y_compare_flag = True video.status.line_y_compare_interrupt = False - video.emulate_hblank_line_y_compare() + video.status.mode0.emulate_hblank_line_y_compare() assert not video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -276,7 +276,7 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = False - video.emulate_hblank_line_y_compare() + video.status.mode0.emulate_hblank_line_y_compare() assert video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -285,7 +285,7 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = True - video.emulate_hblank_line_y_compare() + video.status.mode0.emulate_hblank_line_y_compare() assert video.status.line_y_compare_flag assert video.lcd_interrupt_flag.is_pending() @@ -295,7 +295,7 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = True video.status.line_y_compare_interrupt = True - video.emulate_hblank_line_y_compare(stat_check=True) + video.status.mode0.emulate_hblank_line_y_compare(stat_check=True) assert video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -304,7 +304,7 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = True - video.emulate_hblank_line_y_compare(stat_check=True) + video.status.mode0.emulate_hblank_line_y_compare(stat_check=True) assert video.status.line_y_compare_flag assert video.lcd_interrupt_flag.is_pending() @@ -316,7 +316,7 @@ video.cycles = 0 video.frames = 0 assert not video.lcd_interrupt_flag.is_pending() - video.emulate_hblank() + video.status.mode0.emulate_hblank() assert video.cycles == constants.MODE_2_TICKS assert video.lcd_interrupt_flag.is_pending() assert video.status.get_mode() == 2 @@ -332,7 +332,7 @@ video.status.write(0x0F, write_all=True) video.cycles = 0 video.frames = 0 - video.emulate_hblank() + video.status.mode0.emulate_hblank() assert video.line_y == 2 assert video.cycles == constants.MODE_2_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -349,7 +349,7 @@ video.frame_skip = 20 video.v_blank = False video.display = False - video.emulate_hblank() + video.status.mode0.emulate_hblank() assert video.line_y == 145 assert video.cycles == constants.MODE_1_BEGIN_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -369,7 +369,7 @@ video.frame_skip = 10 video.v_blank = False video.display = False - video.emulate_hblank() + video.status.mode0.emulate_hblank() assert video.line_y == 145 assert video.cycles == constants.MODE_1_BEGIN_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -385,7 +385,7 @@ video.status.write(0xFE, write_all=True) video.v_blank = True video.cycles = 0 - video.emulate_v_blank() + video.status.mode1.emulate_v_blank() assert video.v_blank == False assert video.status.get_mode() == 1 assert video.status.read(extend=True) == 0xFD @@ -399,7 +399,7 @@ video.v_blank = True assert not video.v_blank_interrupt_flag.is_pending() assert not video.lcd_interrupt_flag.is_pending() - video.emulate_v_blank() + video.status.mode1.emulate_v_blank() assert video.status.read(extend=True) == 0x01 assert video.v_blank_interrupt_flag.is_pending() assert not video.lcd_interrupt_flag.is_pending() @@ -414,7 +414,7 @@ video.v_blank = False video.cycles = 0 video.line_y = 0 - video.emulate_v_blank() + video.status.mode1.emulate_v_blank() assert video.v_blank == False assert video.status.read(extend=True) == 0x2E assert video.cycles == constants.MODE_2_TICKS @@ -425,7 +425,7 @@ video.v_blank_interrupt_flag.set_pending(False) video.cycles = 0 video.status.write(0xFD, write_all=True) - video.emulate_v_blank() + video.status.mode1.emulate_v_blank() assert video.v_blank == False assert video.status.read(extend=True) == 0xFE assert video.cycles == constants.MODE_2_TICKS Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Mon Sep 8 21:30:47 2008 @@ -85,32 +85,109 @@ class Mode(object): def id(self): raise Exception() + def __init__(self, video): self.video = video + def emulate_hblank_line_y_compare(self, stat_check=False): + if self.video.line_y == self.video.line_y_compare: + if not (stat_check and self.video.status.line_y_compare_flag): + self.video.line_y_line_y_compare_interrupt_check() + else: + self.video.status.line_y_compare_flag = False + class Mode0(Mode): def id(self): return 0 + def emulate(self): - self.video.emulate_hblank() - + #self.video.emulate_hblank() + self.emulate_hblank() + + def emulate_hblank(self): + self.video.line_y += 1 + self.emulate_hblank_line_y_compare() + if self.video.line_y < 144: + self.video.set_mode_2() + else: + self.emulate_hblank_part_2() + + def emulate_hblank_part_2(self): + if self.video.display: + self.video.draw_frame() + self.video.frames += 1 + if self.video.frames >= self.video.frame_skip: + self.video.display = True + self.video.frames = 0 + else: + self.video.display = False + self.video.set_mode_1_begin() + self.video.v_blank = True + + class Mode1(Mode): def id(self): return 1 + def emulate(self): - self.video.emulate_v_blank() - + self.emulate_v_blank() + + def emulate_v_blank(self): + if self.video.v_blank: + self.emulate_v_blank_v_blank() + elif self.video.line_y == 0: + self.video.set_mode_2() + else: + self.emulate_v_blank_other() + + def emulate_v_blank_v_blank(self): + self.video.v_blank = False + self.video.set_mode_1_between() + self.video.v_blank_interrupt_check() + + def emulate_v_blank_other(self): + if self.video.line_y < 153: + self.emulate_v_blank_mode_1() + else: + self.video.line_y = 0 + self.video.window_line_y = 0 + self.video.set_mode_1_between() + self.emulate_hblank_line_y_compare() + + def emulate_v_blank_mode_1(self): + self.video.line_y += 1 + if self.video.line_y == 153: + self.video.set_mode_1_end() + else: + self.video.set_mode_1() + class Mode2(Mode): def id(self): return 2 + def emulate(self): - self.video.emulate_oam() + self.emulate_oam() + + def emulate_oam(self): + self.video.set_mode_3_begin() class Mode3(Mode): + def id(self): return 3 + def emulate(self): - self.video.emulate_transfer() + self.emulate_transfer() + + def emulate_transfer(self): + if self.video.transfer: + if self.video.display: + self.video.draw_line() + self.video.set_mode_3_end() + else: + self.video.set_mode_0() + +# ----------------------------------------------------------------------------- class StatusRegister(object): """ @@ -126,12 +203,16 @@ 3: During Transfering Data to LCD Driver """ def __init__(self, video): - self.modes = [Mode0(video), - Mode1(video), - Mode2(video), - Mode3(video)] + self.create_modes(video) self.reset() + def create_modes(self, video): + self.mode0 = Mode0(video) + self.mode1 = Mode1(video) + self.mode2 = Mode2(video) + self.mode3 = Mode3(video) + self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] + def reset(self): self.set_mode(0x02) self.line_y_compare_flag = False @@ -433,7 +514,7 @@ """ self.line_y_compare = data if self.control.lcd_enabled: - self.emulate_hblank_line_y_compare(stat_check=True) + self.status.mode0.emulate_hblank_line_y_compare(stat_check=True) def get_dma(self): return self.dma @@ -623,73 +704,8 @@ def current_mode(self): return self.status.current_mode - - def emulate_oam(self): - self.set_mode_3_begin() - def emulate_transfer(self): - if self.transfer: - if self.display: - self.draw_line() - self.set_mode_3_end() - else: - self.set_mode_0() - def emulate_hblank(self): - self.line_y+=1 - self.emulate_hblank_line_y_compare() - if self.line_y < 144: - self.set_mode_2() - else: - self.emulate_hblank_part_2() - - def emulate_hblank_line_y_compare(self, stat_check=False): - if self.line_y == self.line_y_compare: - if not (stat_check and self.status.line_y_compare_flag): - self.line_y_line_y_compare_interrupt_check() - else: - self.status.line_y_compare_flag = False - - def emulate_hblank_part_2(self): - if self.display: - self.draw_frame() - self.frames += 1 - if self.frames >= self.frame_skip: - self.display = True - self.frames = 0 - else: - self.display = False - self.set_mode_1_begin() - self.v_blank = True - - def emulate_v_blank(self): - if self.v_blank: - self.emulate_v_blank_v_blank() - elif self.line_y == 0: - self.set_mode_2() - else: - self.emulate_v_blank_other() - - def emulate_v_blank_v_blank(self): - self.v_blank = False - self.set_mode_1_between() - self.v_blank_interrupt_check() - - def emulate_v_blank_other(self): - if self.line_y < 153: - self.emulate_v_blank_mode_1() - else: - self.line_y = 0 - self.window_line_y = 0 - self.set_mode_1_between() - self.emulate_hblank_line_y_compare() - - def emulate_v_blank_mode_1(self): - self.line_y += 1 - if self.line_y == 153: - self.set_mode_1_end() - else: - self.set_mode_1() # graphics handling -------------------------------------------------------- From cami at codespeak.net Tue Sep 9 11:50:51 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 9 Sep 2008 11:50:51 +0200 (CEST) Subject: [pypy-svn] r57993 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080909095051.4405F16A232@codespeak.net> Author: cami Date: Tue Sep 9 11:50:50 2008 New Revision: 57993 Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/video.py Log: refactoring moved set_mode to the mode classes adpated tests accoring to refactoring Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Tue Sep 9 11:50:50 2008 @@ -103,36 +103,50 @@ def test_set_line_y_compare(): video = get_video() + video.status.write(0, write_all=True) value = 0xF6 + video.control.lcd_enabled = False + video.write(0xFF45, value) + assert video.line_y_compare == value video.control.write(0x80) + + assert video.status.read() == 0x0 + video.line_y = value -1 video.status.write(0xFF, write_all=True) - assert video.status.read(extend=True) == 0xFF + video.write(0xFF45, value) + assert video.status.read(extend=True) == 0xFB assert video.lcd_interrupt_flag.is_pending() == False video.control.write(0x80) - video.line_y = 0xF6 video.status.write(0x04, write_all=True) + video.line_y = 0xF6 + video.write(0xFF45, value) + assert video.status.read(extend=True) == 0x04 assert video.lcd_interrupt_flag.is_pending() == False video.control.write(0x80) - video.line_y = 0xF6 video.status.write(0x00, write_all=True) + video.line_y = 0xF6 + video.write(0xFF45, value) + assert video.status.read(extend=True) == 0x04 assert video.lcd_interrupt_flag.is_pending() == False video.control.write(0x80) - video.line_y = 0xF6 video.status.write(0x40, write_all=True) + video.line_y = 0xF6 + video.write(0xFF45, value) + assert video.status.read(extend=True) == 0x44 assert video.lcd_interrupt_flag.is_pending() == True @@ -140,10 +154,11 @@ def test_control(): video = get_video() - video.control.write(0x80) video.window_line_y = 1 + video.write(0xFF40, 0x80) + assert video.control.read() == 0x80 assert video.window_line_y == 1 @@ -153,17 +168,21 @@ video.window_y = 0 video.line_y = 1 video.window_line_y = 0 + video.write(0xFF40, 0x80+0x20) + assert video.control.read() == 0x80+0x20 assert video.window_line_y == 144 def test_control_reset1(): video = get_video() - video.control.write(0) video.status.write(0x30, write_all=True) + video.control.write(0) video.line_y = 1 video.display = True + video.write(0xFF40, 0x80) + assert video.control.read() == 0x80 assert video.status.read(extend=True) == 0x30 + 0x02 assert video.cycles == constants.MODE_2_TICKS @@ -172,11 +191,13 @@ def test_control_reset2(): video = get_video() - video.control.write(0x80) video.status.write(0x30, write_all=True) + video.control.write(0x80) video.line_y = 1 video.display = True + video.write(0xFF40, 0x30) + assert video.control.read() == 0x30 assert video.status.read(extend=True) == 0x30 assert video.cycles == constants.MODE_1_TICKS @@ -227,7 +248,9 @@ video.transfer = False video.status.write(0xFE, write_all=True) video.cycles = 0 + video.status.mode2.emulate_oam() + assert video.status.read(extend=True) == 0xFF assert video.cycles == constants.MODE_3_BEGIN_TICKS assert video.transfer == True @@ -235,29 +258,37 @@ def test_emulate_transfer(): video = get_video() + video.status.write(0xF0, write_all=True) video.transfer = False video.cycles = 0 - video.status.write(0xF0, write_all=True) + video.status.mode3.emulate_transfer() + assert video.status.read(extend=True) == 0xF0 assert video.cycles == constants.MODE_0_TICKS assert not video.lcd_interrupt_flag.is_pending() + video.status.write(0xF8, write_all=True) video.transfer = False video.cycles = 0 - video.status.write(0xF8, write_all=True) assert not video.lcd_interrupt_flag.is_pending() + video.status.mode3.emulate_transfer() + assert video.status.read(extend=True) == 0xF8 assert video.cycles == constants.MODE_0_TICKS assert video.lcd_interrupt_flag.is_pending() + video.status.write(0xFF, write_all=True) + assert video.status.get_mode() == 3 video.transfer = True video.cycles = 0 - video.status.write(0xFC, write_all=True) + video.status.mode3.emulate_transfer() + assert video.cycles == constants.MODE_3_END_TICKS assert video.transfer == False + assert video.status.get_mode() == 3 assert video.status.read(extend=True) == 0xFF @@ -267,7 +298,9 @@ video.line_y_compare = 0x13 video.status.line_y_compare_flag = True video.status.line_y_compare_interrupt = False + video.status.mode0.emulate_hblank_line_y_compare() + assert not video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -276,7 +309,9 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = False + video.status.mode0.emulate_hblank_line_y_compare() + assert video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -285,7 +320,9 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = True + video.status.mode0.emulate_hblank_line_y_compare() + assert video.status.line_y_compare_flag assert video.lcd_interrupt_flag.is_pending() @@ -295,7 +332,9 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = True video.status.line_y_compare_interrupt = True + video.status.mode0.emulate_hblank_line_y_compare(stat_check=True) + assert video.status.line_y_compare_flag assert not video.lcd_interrupt_flag.is_pending() @@ -304,19 +343,23 @@ video.line_y_compare = 0x12 video.status.line_y_compare_flag = False video.status.line_y_compare_interrupt = True + video.status.mode0.emulate_hblank_line_y_compare(stat_check=True) + assert video.status.line_y_compare_flag assert video.lcd_interrupt_flag.is_pending() def test_emulate_h_blank_part_1_1(): video = get_video() + video.status.write(0x20, write_all=True) video.line_y = 0 video.line_y_compare = 1 - video.status.write(0x20, write_all=True) video.cycles = 0 video.frames = 0 assert not video.lcd_interrupt_flag.is_pending() + video.status.mode0.emulate_hblank() + assert video.cycles == constants.MODE_2_TICKS assert video.lcd_interrupt_flag.is_pending() assert video.status.get_mode() == 2 @@ -327,12 +370,14 @@ def test_emulate_h_blank_part_2_1(): video = get_video() + video.status.write(0x0F, write_all=True) video.line_y = 1 video.line_y_compare = 0 - video.status.write(0x0F, write_all=True) video.cycles = 0 video.frames = 0 + video.status.mode0.emulate_hblank() + assert video.line_y == 2 assert video.cycles == constants.MODE_2_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -341,15 +386,17 @@ def test_emulate_h_blank_part_2_2(): video = get_video() + video.status.write(0xFB, write_all=True) video.line_y = 144 video.line_y_compare = 0 - video.status.write(0xFB, write_all=True) video.cycles = 0 video.frames = 0 video.frame_skip = 20 video.v_blank = False video.display = False + video.status.mode0.emulate_hblank() + assert video.line_y == 145 assert video.cycles == constants.MODE_1_BEGIN_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -369,7 +416,9 @@ video.frame_skip = 10 video.v_blank = False video.display = False + video.status.mode0.emulate_hblank() + assert video.line_y == 145 assert video.cycles == constants.MODE_1_BEGIN_TICKS assert not video.lcd_interrupt_flag.is_pending() @@ -380,12 +429,15 @@ def test_emulate_v_v_blank_1(): video = get_video() + video.status.write(0xFF-2, write_all=True) + assert video.status.get_mode() == 1 video.lcd_interrupt_flag.set_pending(False) video.v_blank_interrupt_flag.set_pending(False) - video.status.write(0xFE, write_all=True) video.v_blank = True video.cycles = 0 + video.status.mode1.emulate_v_blank() + assert video.v_blank == False assert video.status.get_mode() == 1 assert video.status.read(extend=True) == 0xFD @@ -393,13 +445,16 @@ assert video.v_blank_interrupt_flag.is_pending() assert video.lcd_interrupt_flag.is_pending() + video.status.write(0x01, write_all=True) + assert video.status.get_mode() == 1 video.lcd_interrupt_flag.set_pending(False) video.v_blank_interrupt_flag.set_pending(False) - video.status.write(0x00, write_all=True) video.v_blank = True assert not video.v_blank_interrupt_flag.is_pending() assert not video.lcd_interrupt_flag.is_pending() + video.status.mode1.emulate_v_blank() + assert video.status.read(extend=True) == 0x01 assert video.v_blank_interrupt_flag.is_pending() assert not video.lcd_interrupt_flag.is_pending() @@ -408,24 +463,28 @@ def test_emulate_v_v_blank_2(): video = get_video() + video.status.write(0x2D, write_all=True) video.lcd_interrupt_flag.set_pending(False) video.v_blank_interrupt_flag.set_pending(False) - video.status.write(0x2D, write_all=True) video.v_blank = False video.cycles = 0 video.line_y = 0 + video.status.mode1.emulate_v_blank() + assert video.v_blank == False assert video.status.read(extend=True) == 0x2E assert video.cycles == constants.MODE_2_TICKS assert not video.v_blank_interrupt_flag.is_pending() assert video.lcd_interrupt_flag.is_pending() + video.status.write(0xFD, write_all=True) video.lcd_interrupt_flag.set_pending(False) video.v_blank_interrupt_flag.set_pending(False) video.cycles = 0 - video.status.write(0xFD, write_all=True) + video.status.mode1.emulate_v_blank() + assert video.v_blank == False assert video.status.read(extend=True) == 0xFE assert video.cycles == constants.MODE_2_TICKS Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Tue Sep 9 11:50:50 2008 @@ -1,6 +1,7 @@ from pypy.lang.gameboy import constants from pypy.lang.gameboy.video import ControlRegister from pypy.lang.gameboy.video import StatusRegister +from pypy.lang.gameboy.test.test_video import get_video import py @@ -51,7 +52,7 @@ # StatusRegister --------------------------------------------------------------- def test_video_status_reset(): - status = StatusRegister(None) + status = StatusRegister(get_video()) assert status.read(extend=True) == 0x02 + 0x80 status.write(0x00, write_all=True) @@ -65,7 +66,7 @@ assert status.read(extend=True) == 0x02 + 0x80 def test_video_status_mode(): - status = StatusRegister(None) + status = StatusRegister(get_video()) assert status.get_mode() == 2 for i in range(3): Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 9 11:50:50 2008 @@ -81,25 +81,66 @@ # ----------------------------------------------------------------------------- - +class InvalidModeOrderException(Exception): + def __init__(self, mode, previous_mode): + Exception.__init__(self, \ + "Wrong Mode order! No valid transition %i => %i" \ + % (previous_mode.id(), mode.id())) + + class Mode(object): + + """ + The two lower STAT bits show the current status of the LCD controller. + """ + + def __init__(self, video): + self.video = video + def id(self): raise Exception() - def __init__(self, video): - self.video = video + def activate(self, previous_mode): + raise Exception() + + def emulate(self): + raise Exception() def emulate_hblank_line_y_compare(self, stat_check=False): if self.video.line_y == self.video.line_y_compare: if not (stat_check and self.video.status.line_y_compare_flag): - self.video.line_y_line_y_compare_interrupt_check() + self.line_y_line_y_compare_interrupt_check() else: self.video.status.line_y_compare_flag = False + + def line_y_line_y_compare_interrupt_check(self): + self.video.status.line_y_compare_flag = True + if self.video.status.line_y_compare_interrupt: + self.video.lcd_interrupt_flag.set_pending() class Mode0(Mode): + """ + Mode 0: The LCD controller is in the H-Blank period and + the CPU can access both the display RAM (8000h-9FFFh) + and OAM (FE00h-FE9Fh) + """ def id(self): return 0 + def activate(self, previous_mode): + #if previous_mode.id() == 3: + self.video.cycles += constants.MODE_0_TICKS + self.h_blank_interrupt_check() + #else: + # video.reset_control() can be called in any position + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def h_blank_interrupt_check(self): + if self.video.status.mode_0_h_blank_interrupt and \ + self.video.status.line_y_compare_check(): + self.video.lcd_interrupt_flag.set_pending() + def emulate(self): #self.video.emulate_hblank() self.emulate_hblank() @@ -108,7 +149,7 @@ self.video.line_y += 1 self.emulate_hblank_line_y_compare() if self.video.line_y < 144: - self.video.set_mode_2() + self.video.status.set_mode(2) else: self.emulate_hblank_part_2() @@ -121,14 +162,34 @@ self.video.frames = 0 else: self.video.display = False - self.video.set_mode_1_begin() + self.video.status.set_mode(1) self.video.v_blank = True - - + class Mode1(Mode): + """ + Mode 1: The LCD contoller is in the V-Blank period (or the + display is disabled) and the CPU can access both the + display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) + """ def id(self): return 1 + def activate(self, previous_mode): + #if previous_mode.id() == 0: + self.set_begin() + #else: + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def set_begin(self): + self.video.cycles += constants.MODE_1_BEGIN_TICKS + + def set_between(self): + self.video.cycles += constants.MODE_1_TICKS - constants.MODE_1_BEGIN_TICKS + + def set_end(self): + self.video.cycles += constants.MODE_1_END_TICKS + def emulate(self): self.emulate_v_blank() @@ -136,46 +197,88 @@ if self.video.v_blank: self.emulate_v_blank_v_blank() elif self.video.line_y == 0: - self.video.set_mode_2() + self.video.status.set_mode(2) else: self.emulate_v_blank_other() def emulate_v_blank_v_blank(self): self.video.v_blank = False - self.video.set_mode_1_between() - self.video.v_blank_interrupt_check() + self.set_between() + self.v_blank_interrupt_check() + def v_blank_interrupt_check(self): + if self.video.status.mode_1_v_blank_interrupt: + self.video.lcd_interrupt_flag.set_pending() + self.video.v_blank_interrupt_flag.set_pending() + def emulate_v_blank_other(self): if self.video.line_y < 153: self.emulate_v_blank_mode_1() else: self.video.line_y = 0 self.video.window_line_y = 0 - self.video.set_mode_1_between() + self.set_between() self.emulate_hblank_line_y_compare() def emulate_v_blank_mode_1(self): self.video.line_y += 1 - if self.video.line_y == 153: - self.video.set_mode_1_end() + if self.video.line_y != 153: + self.video.cycles += constants.MODE_1_TICKS else: - self.video.set_mode_1() + self.set_end() class Mode2(Mode): + """ + Mode 2: The LCD controller is reading from OAM memory. + The CPU access OAM memory (FE00h-FE9Fh) + during this period. + """ def id(self): return 2 + def activate(self, previous_mode): + #if previous_mode.id() == 0 or previous_mode.id() == 1: + self.video.cycles += constants.MODE_2_TICKS + self.oam_interrupt_check() + #else: + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def oam_interrupt_check(self): + if self.video.status.mode_2_oam_interrupt and \ + self.video.status.line_y_compare_check(): + self.video.lcd_interrupt_flag.set_pending() + def emulate(self): self.emulate_oam() def emulate_oam(self): - self.video.set_mode_3_begin() + self.video.status.set_mode(3) class Mode3(Mode): - + """ + Mode 3: The LCD controller is reading from both OAM and VRAM, + The CPU access OAM and VRAM during this period. + CGB Mode: Cannot access Palette Data (FF69,FF6B) either. + """ def id(self): return 3 + def activate(self, previous_mode): + #if previous_mode.id() == 2: + self.set_begin() + #else: + # pass + # #raise InvalidModeOrderException(self, previous_mode) + + def set_begin(self): + self.video.cycles += constants.MODE_3_BEGIN_TICKS + self.video.transfer = True + + def set_end(self): + self.video.cycles += constants.MODE_3_END_TICKS + self.video.transfer = False + def emulate(self): self.emulate_transfer() @@ -183,9 +286,10 @@ if self.video.transfer: if self.video.display: self.video.draw_line() - self.video.set_mode_3_end() + print "mode 3 ", self.video.status.get_mode() + self.set_end() else: - self.video.set_mode_0() + self.video.status.set_mode(0) # ----------------------------------------------------------------------------- @@ -213,8 +317,10 @@ self.mode3 = Mode3(video) self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] + + def reset(self): - self.set_mode(0x02) + self.current_mode = self.mode2 self.line_y_compare_flag = False self.mode_0_h_blank_interrupt = False self.mode_1_v_blank_interrupt = False @@ -238,7 +344,7 @@ def write(self, value, write_all=False, keep_mode_0_h_blank_interrupt=False): if write_all: - self.set_mode(value & 0x03) + self.current_mode = self.modes[value & 0x03] self.line_y_compare_flag = bool(value & (1 << 2)) self.status = bool(value & (1 << 7)) self.mode_0_h_blank_interrupt = bool(value & (1 << 3)) @@ -250,7 +356,9 @@ return self.current_mode.id() def set_mode(self, mode): + old_mode = self.current_mode self.current_mode = self.modes[mode & 0x03] + self.current_mode.activate(old_mode) def line_y_compare_check(self): return not self.line_y_compare_flag or not self.line_y_compare_interrupt @@ -617,83 +725,6 @@ self.window_x = data - # interrupt checks --------------------------------------------------- - - def h_blank_interrupt_check(self): - if self.status.mode_0_h_blank_interrupt and \ - self.status.line_y_compare_check(): - self.lcd_interrupt_flag.set_pending() - - def oam_interrupt_check(self): - if self.status.mode_2_oam_interrupt and \ - self.status.line_y_compare_check(): - self.lcd_interrupt_flag.set_pending() - - def v_blank_interrupt_check(self): - if self.status.mode_1_v_blank_interrupt: - self.lcd_interrupt_flag.set_pending() - self.v_blank_interrupt_flag.set_pending() - - def line_y_line_y_compare_interrupt_check(self): - self.status.line_y_compare_flag = True - if self.status.line_y_compare_interrupt: - self.lcd_interrupt_flag.set_pending() - - # mode setting ----------------------------------------------------------- - """ - The two lower STAT bits show the current status of the LCD controller. - Mode 0: The LCD controller is in the H-Blank period and - the CPU can access both the display RAM (8000h-9FFFh) - and OAM (FE00h-FE9Fh) - - Mode 1: The LCD contoller is in the V-Blank period (or the - display is disabled) and the CPU can access both the - display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) - - Mode 2: The LCD controller is reading from OAM memory. - The CPU access OAM memory (FE00h-FE9Fh) - during this period. - - Mode 3: The LCD controller is reading from both OAM and VRAM, - The CPU access OAM and VRAM during this period. - CGB Mode: Cannot access Palette Data (FF69,FF6B) either. - """ - def set_mode_3_begin(self): - self.status.set_mode(3) - self.cycles += constants.MODE_3_BEGIN_TICKS - self.transfer = True - - def set_mode_3_end(self): - self.status.set_mode(3) - self.cycles += constants.MODE_3_END_TICKS - self.transfer = False - - def set_mode_0(self): - self.status.set_mode(0) - self.cycles += constants.MODE_0_TICKS - self.h_blank_interrupt_check() - - def set_mode_2(self): - self.status.set_mode(2) - self.cycles += constants.MODE_2_TICKS - self.oam_interrupt_check() - - def set_mode_1_begin(self): - self.status.set_mode(1) - self.cycles += constants.MODE_1_BEGIN_TICKS - - def set_mode_1(self): - self.status.set_mode(1) - self.cycles += constants.MODE_1_TICKS - - def set_mode_1_between(self): - self.status.set_mode(1) - self.cycles += constants.MODE_1_TICKS - constants.MODE_1_BEGIN_TICKS - - def set_mode_1_end(self): - self.status.set_mode(1) - self.cycles += constants.MODE_1_END_TICKS - # emulation ---------------------------------------------------------------- def emulate(self, ticks): From cami at codespeak.net Tue Sep 9 11:57:49 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 9 Sep 2008 11:57:49 +0200 (CEST) Subject: [pypy-svn] r57994 - pypy/dist/pypy/lang/gameboy Message-ID: <20080909095749.05D07169EA9@codespeak.net> Author: cami Date: Tue Sep 9 11:57:47 2008 New Revision: 57994 Modified: pypy/dist/pypy/lang/gameboy/video.py Log: minor refactoring move interrupt check staes into the corresponindg modes Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 9 11:57:47 2008 @@ -96,15 +96,19 @@ def __init__(self, video): self.video = video + self.reset() + def reset(self): + raise Exception("unimplemented method") + def id(self): - raise Exception() + raise Exception("unimplemented method") def activate(self, previous_mode): - raise Exception() + raise Exception("unimplemented method") def emulate(self): - raise Exception() + raise Exception("unimplemented method") def emulate_hblank_line_y_compare(self, stat_check=False): if self.video.line_y == self.video.line_y_compare: @@ -124,6 +128,10 @@ the CPU can access both the display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) """ + + def reset(self): + self.h_blank_interrupt = False + def id(self): return 0 @@ -137,7 +145,7 @@ #raise InvalidModeOrderException(self, previous_mode) def h_blank_interrupt_check(self): - if self.video.status.mode_0_h_blank_interrupt and \ + if self.h_blank_interrupt and \ self.video.status.line_y_compare_check(): self.video.lcd_interrupt_flag.set_pending() @@ -171,6 +179,10 @@ display is disabled) and the CPU can access both the display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) """ + + def reset(self): + self.v_blank_interrupt = False + def id(self): return 1 @@ -207,7 +219,7 @@ self.v_blank_interrupt_check() def v_blank_interrupt_check(self): - if self.video.status.mode_1_v_blank_interrupt: + if self.v_blank_interrupt: self.video.lcd_interrupt_flag.set_pending() self.video.v_blank_interrupt_flag.set_pending() @@ -233,6 +245,10 @@ The CPU access OAM memory (FE00h-FE9Fh) during this period. """ + + def reset(self): + self.oam_interrupt = False + def id(self): return 2 @@ -245,7 +261,7 @@ #raise InvalidModeOrderException(self, previous_mode) def oam_interrupt_check(self): - if self.video.status.mode_2_oam_interrupt and \ + if self.oam_interrupt and \ self.video.status.line_y_compare_check(): self.video.lcd_interrupt_flag.set_pending() @@ -255,12 +271,17 @@ def emulate_oam(self): self.video.status.set_mode(3) + class Mode3(Mode): """ Mode 3: The LCD controller is reading from both OAM and VRAM, The CPU access OAM and VRAM during this period. CGB Mode: Cannot access Palette Data (FF69,FF6B) either. """ + + def reset(self): + pass + def id(self): return 3 @@ -322,19 +343,21 @@ def reset(self): self.current_mode = self.mode2 self.line_y_compare_flag = False - self.mode_0_h_blank_interrupt = False - self.mode_1_v_blank_interrupt = False - self.mode_2_oam_interrupt = False self.line_y_compare_interrupt = False + for mode in self.modes: + mode.reset() + #self.mode_0_h_blank_interrupt = False + #self.mode_1_v_blank_interrupt = False + #self.mode_2_oam_interrupt = False self.status = True def read(self, extend=False): value = self.get_mode() value += self.line_y_compare_flag << 2 - value += self.mode_0_h_blank_interrupt << 3 - value += self.mode_1_v_blank_interrupt << 4 - value += self.mode_2_oam_interrupt << 5 + value += self.mode0.h_blank_interrupt << 3 + value += self.mode1.v_blank_interrupt << 4 + value += self.mode2.oam_interrupt << 5 value += self.line_y_compare_interrupt << 6 if extend: value += int(self.status) << 7 @@ -344,13 +367,13 @@ def write(self, value, write_all=False, keep_mode_0_h_blank_interrupt=False): if write_all: - self.current_mode = self.modes[value & 0x03] - self.line_y_compare_flag = bool(value & (1 << 2)) - self.status = bool(value & (1 << 7)) - self.mode_0_h_blank_interrupt = bool(value & (1 << 3)) - self.mode_1_v_blank_interrupt = bool(value & (1 << 4)) - self.mode_2_oam_interrupt = bool(value & (1 << 5)) - self.line_y_compare_interrupt = bool(value & (1 << 6)) + self.current_mode = self.modes[value & 0x03] + self.line_y_compare_flag = bool(value & (1 << 2)) + self.status = bool(value & (1 << 7)) + self.mode0.h_blank_interrupt = bool(value & (1 << 3)) + self.mode1.v_blank_interrupt = bool(value & (1 << 4)) + self.mode2.oam_interrupt = bool(value & (1 << 5)) + self.line_y_compare_interrupt = bool(value & (1 << 6)) def get_mode(self): return self.current_mode.id() From fijal at codespeak.net Tue Sep 9 12:08:54 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 12:08:54 +0200 (CEST) Subject: [pypy-svn] r57995 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080909100854.65E82169FF1@codespeak.net> Author: fijal Date: Tue Sep 9 12:08:51 2008 New Revision: 57995 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: Use py.execnet instead of homegrown solution. on one hand this saves a bit of effort, on the other hand it makes impossible to measure --no-allworkingmodules by now. Now we spawn another interpreter to make sure we don't have effects of copy on write Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Tue Sep 9 12:08:51 2008 @@ -5,12 +5,14 @@ Dump output (which is step & private memory used) into a file suitable for plotting with gnuplot. """ +import autopath import os import py import time import re import signal import sys +import socket class Result(object): def __init__(self, priv_map, shared_map, starttime=None): @@ -85,41 +87,39 @@ def __del__(self): self.close() -def run_cooperative(func): +SOCK_FILE = '/tmp/bench_mem' + +def run_cooperative(f, func_name=None): """ The idea is that we fork() and execute the function func in one process. In parent process we measure private memory obtained by a process when we read anything in pipe. We send back that we did that """ - parentread, childwrite = os.pipe() - childread, parentwrite = os.pipe() - pid = os.fork() - if not pid: - os.close(parentread) - os.close(parentwrite) - try: - func(lambda : os.read(childread, 1), - lambda x: os.write(childwrite, x)) - except (SystemExit, KeyboardInterrupt): - pass - except: - import traceback - traceback.print_tb(sys.exc_info()[2]) - os._exit(1) - os.close(childread) - os.close(childwrite) - return (lambda : os.read(parentread, 1), - lambda x: os.write(parentwrite, x), - pid) + gw = py.execnet.PopenGateway() + if func_name is None: + func_name = f.func_name + code = py.code.Source(py.code.Source(""" + import os + channel.send(os.getpid()) + """), py.code.Source(f), py.code.Source( + "%s(channel.receive, channel.send)" % func_name)) + channel = gw.remote_exec(code) + pid = channel.receive() + return channel.receive, channel.send, pid + +def smaps_measure_func(pid): + return parse_smaps_output(open('/proc/%d/smaps' % pid).read()) def measure(measure_func, funcs_to_run): results = [] for func in funcs_to_run: - read, write, pid = run_cooperative(func) + if isinstance(func, tuple): + read, write, pid = run_cooperative(*func) + else: + read, write, pid = run_cooperative(func) elem = [] while 1: res = read() if res == 'e': - os.waitpid(pid, 0) break else: elem.append(measure_func(pid)) Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Tue Sep 9 12:08:51 2008 @@ -28,7 +28,7 @@ x = read() assert x == 'y' write('x') - + read, write, pid = bench_mem.run_cooperative(f) assert pid write('y') From fijal at codespeak.net Tue Sep 9 12:22:26 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 12:22:26 +0200 (CEST) Subject: [pypy-svn] r57996 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080909102226.579C5169FFF@codespeak.net> Author: fijal Date: Tue Sep 9 12:22:24 2008 New Revision: 57996 Added: pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py (contents, props changed) Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: Add some microbenchmarks Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Tue Sep 9 12:22:24 2008 @@ -111,7 +111,7 @@ def measure(measure_func, funcs_to_run): results = [] - for func in funcs_to_run: + for num, func in enumerate(funcs_to_run): if isinstance(func, tuple): read, write, pid = run_cooperative(*func) else: @@ -122,7 +122,7 @@ if res == 'e': break else: - elem.append(measure_func(pid)) + elem.append(measure_func(num, pid)) write('x') results.append(elem) return results Added: pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py ============================================================================== --- (empty file) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Tue Sep 9 12:22:24 2008 @@ -0,0 +1,63 @@ +#!/usr/bin/env python +import autopath +""" This file attempts to measure how much memory is taken by various objects. +""" + +from pypy.translator.benchmark.bench_mem import measure, smaps_measure_func +import py + +def measure_func(num, pid): + res = smaps_measure_func(pid) + print num, res.private + +def tuples(read, write, coeff): + import gc + x = () + for i in range(1000 * coeff): + x = (x,) + gc.collect() + write('x') + read() + write('e') + +def linked_list(read, write, coeff): + import gc + + class A(object): + def __init__(self, other): + self.other = other + + x = None + for i in range(1000 * coeff): + x = A(x) + gc.collect() + write('x') + read() + write('e') + +def list_of_instances_with_int(read, write, coeff): + import gc + + class A(object): + def __init__(self, x): + self.x = x + + x = [A(i) for i in range(1000 * coeff)] + gc.collect() + write('x') + read() + write('e') + +if __name__ == '__main__': + coeff = 1 + i = 0 + funcs = [] + while i < 10: + f = py.code.Source(list_of_instances_with_int, """ + def f(r, w): + list_of_instances_with_int(r, w, %d) + """ % coeff) + funcs.append((f, 'f')) + coeff *= 2 + i += 1 + res = measure(measure_func, funcs) Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Tue Sep 9 12:22:24 2008 @@ -42,7 +42,7 @@ read() write('e') - def measure(arg): + def measure(arg1, arg): return 42 measurments = bench_mem.measure(measure, [f1, f1]) From fijal at codespeak.net Tue Sep 9 12:45:34 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 12:45:34 +0200 (CEST) Subject: [pypy-svn] r57997 - in pypy/branch/cross-compilation/pypy/translator/benchmark: . test Message-ID: <20080909104534.72786169FA8@codespeak.net> Author: fijal Date: Tue Sep 9 12:45:32 2008 New Revision: 57997 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Log: A test and a fix for slightly different /proc/$pid/smaps output. Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Tue Sep 9 12:45:32 2008 @@ -48,7 +48,9 @@ priv_map[name] = priv + priv_map.get(name, 0) if shared: shared_map[name] = shared + shared_map.get(name, 0) - num += 8 + num += 7 + if num < len(lines) and lines[num].startswith('Referenced'): + num += 1 return Result(priv_map, shared_map) class ChildProcess(object): Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/test/test_bench_mem.py Tue Sep 9 12:45:32 2008 @@ -23,6 +23,19 @@ '/lib/libncurses.so.5.6' : 60, } +def test_parse2(): + res = bench_mem.parse_smaps_output(example_data2) + assert res.private == 796 + 120 + 924 + assert res.shared == 60 + assert res.priv_map == { + '/usr/bin/python2.5': 796 + 120, + '[heap]' : 924, + } + assert res.shared_map == { + '/lib/libncurses.so.5.6' : 60, + } + + def test_run_cooperative(): def f(read, write): x = read() @@ -48,6 +61,37 @@ measurments = bench_mem.measure(measure, [f1, f1]) assert measurments == [[42, 42], [42, 42]] +example_data2 = ''' +08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 +Size: 988 kB +Rss: 796 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 796 kB +Private_Dirty: 0 kB +0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 +Size: 148 kB +Rss: 120 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 12 kB +Private_Dirty: 108 kB +08164000-0825c000 rw-p 08164000 00:00 0 [heap] +Size: 992 kB +Rss: 924 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 924 kB +b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 +Size: 240 kB +Rss: 60 kB +Shared_Clean: 60 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 0 kB +''' + example_data = ''' 08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 Size: 988 kB From nshepperd at codespeak.net Tue Sep 9 13:10:08 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Tue, 9 Sep 2008 13:10:08 +0200 (CEST) Subject: [pypy-svn] r57998 - pypy/branch/new-platformcheck Message-ID: <20080909111008.0234516A02C@codespeak.net> Author: nshepperd Date: Tue Sep 9 13:10:06 2008 New Revision: 57998 Added: pypy/branch/new-platformcheck/ - copied from r57997, pypy/dist/ Log: Branch for new cross-compilable platformcheck. From cami at codespeak.net Tue Sep 9 13:31:24 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 9 Sep 2008 13:31:24 +0200 (CEST) Subject: [pypy-svn] r57999 - pypy/dist/pypy/lang/gameboy Message-ID: <20080909113124.708E116A01A@codespeak.net> Author: cami Date: Tue Sep 9 13:31:23 2008 New Revision: 57999 Modified: pypy/dist/pypy/lang/gameboy/video.py Log: added sprite implementation and some methods to update those sprites Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 9 13:31:23 2008 @@ -395,19 +395,88 @@ self.reset() def reset(self): - self.x = 0 - self.y = 0 - self._tile_number = 0 + self.x = 0 + self.y = 0 + self.tile = None self.object_behind_background = False - self.x_flipped = False - self.y_flipped = False - self.use_object_pallette_1 = False + self.x_flipped = False + self.y_flipped = False + self.palette_number = 0 + self.hidden = True - def get_tile_number(self): - return self._tile_number + + def set_data(self, byte0=None, byte1=None, byte2=None, byte3=None): + """ + extracts the sprite data from an oam entry + + """ + if byte0 is not None: + self.extract_y_position(byte0) + if byte0 is not None: + self.extract_x_position(byte1) + if byte0 is not None: + self.extract_tile_number(byte2) + if byte0 is not None: + self.extract_attributes_and_flags(byte3) + + def extract_y_position(self, data): + """ + extracts the Y Position + Specifies the sprites vertical position on the screen (minus 16). + An offscreen value (for example, Y=0 or Y>=160) hides the sprite. + """ + self.y = data # - 16 + self.hide_check() + + def extract_x_position(self, data): + """ + extracts the X Position + Specifies the sprites horizontal position on the screen (minus 8). + An offscreen value (X=0 or X>=168) hides the sprite, but the sprite + still affects the priority ordering - a better way to hide a sprite is + to set its Y-coordinate offscreen. + """ + self.x = data # - 8 + self.hide_check() - def set_tile_number(self, patter_number): - self._tile_number = patter_number & 0xFF + def extract_tile_number(self, data): + """ + extracts the Tile/Pattern Number + Specifies the sprites Tile Number (00-FF). This (unsigned) value selects + a tile from memory at 8000h-8FFFh. In CGB Mode this could be either in + VRAM Bank 0 or 1, depending on Bit 3 of the following byte. + In 8x16 mode, the lower bit of the tile number is ignored. Ie. the + upper 8x8 tile is "NN AND FEh", and the lower 8x8 tile is "NN OR 01h". + """ + self.tile_number = data + + def extract_attributes_and_flags(self, data): + """ + extracts the Attributes/Flags: + Bit7 OBJ-to-BG Priority (0=OBJ Above BG, 1=OBJ Behind BG color 1-3) + (Used for both BG and Window. BG color 0 is always behind OBJ) + Bit6 Y flip (0=Normal, 1=Vertically mirrored) + Bit5 X flip (0=Normal, 1=Horizontally mirrored) + Bit4 Palette number **Non CGB Mode Only** (0=OBP0, 1=OBP1) + Bit3 Tile VRAM-Bank **CGB Mode Only** (0=Bank 0, 1=Bank 1) + Bit2-0 Palette number **CGB Mode Only** (OBP0-7) + """ + self.object_behind_background = bool(data & (1 << 7)) + self.x_flipped = bool(data & (1 << 6)) + self.y_flipped = bool(data & (1 << 5)) + self.palette_number = bool(data & (1 << 3)) + + + def hide_check(self): + if self.y <= 0 or self.y >= 160: + self.hidden = True + elif self.x <= 0 or self.x >= 168: + self.hidden = True + else: + self.hidden = False + + def get_tile_number(self): + return self.tile.id def get_width(self): return 8 @@ -429,8 +498,8 @@ def __init__(self): pass - def set_tile_data(self, rom, height): - self.height = height + def set_tile_data(self, data): + pass def get_tile_data(self): pass @@ -447,13 +516,17 @@ self.control = ControlRegister() self.status = StatusRegister(self) self.memory = memory + self.create_tiles() + self.create_sprites() self.reset() - - def get_frame_skip(self): - return self.frame_skip - - def set_frame_skip(self, frame_skip): - self.frame_skip = frame_skip + + def create_tiles(self): + self.tiles = [None] + + def create_sprites(self): + self.sprites = [None] * 40 + for i in range(40): + self.sprites[i] = Sprite() def reset(self): self.cycles = constants.MODE_2_TICKS @@ -521,9 +594,9 @@ elif address == constants.WX: self.set_window_x(data) elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: - self.oam[address - constants.OAM_ADDR] = data & 0xFF + self.set_oam(address, data) elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: - self.vram[address - constants.VRAM_ADDR] = data & 0xFF + self.set_vram(address, data) def read(self, address): if address == constants.LCDC: @@ -551,13 +624,19 @@ elif address == constants.WX: return self.get_window_x() elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: - return self.oam[address - constants.OAM_ADDR] + return self.get_oam(address) elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: - return self.vram[address - constants.VRAM_ADDR] + return self.get_vram(address) return 0xFF # Getters and Setters ------------------------------------------------------ + def get_frame_skip(self): + return self.frame_skip + + def set_frame_skip(self, frame_skip): + self.frame_skip = frame_skip + def get_cycles(self): return self.cycles @@ -676,7 +755,16 @@ self.dma = data for index in range(constants.OAM_SIZE): self.oam[index] = self.memory.read((self.dma << 8) + index) + self.update_sprites() + def update_sprites(self): + for i in range(40): + address = 1 * 4 + self.sprites[i].set_data(self.oam[address + 0], + self.oam[address + 1], + self.oam[address + 2], + self.oam[address + 3]) + def get_background_palette(self): """ see set_background_palette""" return self.background_palette @@ -747,6 +835,17 @@ def set_window_x(self, data): self.window_x = data + def set_oam(self, address, data): + self.oam[address - constants.OAM_ADDR] = data & 0xFF + + def get_oam(self, address): + return self.oam[address - constants.OAM_ADDR] + + def set_vram(self, address, data): + self.vram[address - constants.VRAM_ADDR] = data & 0xFF + + def get_vram(self, address): + return self.vram[address - constants.VRAM_ADDR] # emulation ---------------------------------------------------------------- From nshepperd at codespeak.net Tue Sep 9 13:49:21 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Tue, 9 Sep 2008 13:49:21 +0200 (CEST) Subject: [pypy-svn] r58000 - in pypy/branch/new-platformcheck/pypy: rpython/tool tool Message-ID: <20080909114921.DA66016A02C@codespeak.net> Author: nshepperd Date: Tue Sep 9 13:49:18 2008 New Revision: 58000 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py pypy/branch/new-platformcheck/pypy/tool/gcc_cache.py Log: Import already-done work. So far all tests in pypy/rpython/tool/ pass, at least on my machine. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Tue Sep 9 13:49:18 2008 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import llmemory -from pypy.tool.gcc_cache import build_executable_cache, try_compile_cache +from pypy.tool.gcc_cache import build_executable_cache_read, try_compile_cache from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.tool.cbuild import CompilationError from pypy.tool.udir import udir @@ -83,6 +83,17 @@ return _memory_alignment _memory_alignment = None +def getendianness(eci): + if not hasattr(eci, 'endianness'): + class CConfig: + _compilation_info_ = eci + DATA = ConstantString('(unsigned long int)0xFF') + if configure(CConfig)['DATA'][0] == '\x00': + eci.endianness = 'BE' + else: + eci.endianness = 'LE' + return eci.endianness + # ____________________________________________________________ # # General interface @@ -98,11 +109,10 @@ try: return self.result[entry] except KeyError: - pass - name = self.entries[entry] - info = self.info[name] - self.result[entry] = entry.build_result(info, self) - return self.result[entry] + name = self.entries[entry] + info = self.info[name] + self.result[entry] = entry.build_result(info, self) + return self.result[entry] def get_result(self): return dict([(name, self.result[entry]) @@ -125,19 +135,11 @@ def write_entry(self, key, entry): f = self.f - print >> f, 'void dump_section_%s(void) {' % (key,) + entry.key = key for line in entry.prepare_code(): - if line and line[0] != '#': - line = '\t' + line print >> f, line - print >> f, '}' print >> f - def write_entry_main(self, key): - print >> self.f, '\tprintf("-+- %s\\n");' % (key,) - print >> self.f, '\tdump_section_%s();' % (key,) - print >> self.f, '\tprintf("---\\n");' - def start_main(self): print >> self.f, 'int main(int argc, char *argv[]) {' @@ -162,29 +164,31 @@ """ for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_', '_include_dirs_', '_header_']: - assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,) + assert not hasattr(CConfig, attr), "Found legacy attribute %s on CConfig" % (attr,) entries = [] for key in dir(CConfig): value = getattr(CConfig, key) if isinstance(value, CConfigEntry): entries.append((key, value)) - if entries: # can be empty if there are only CConfigSingleEntries + if entries: # can be empty if there are only CConfigSingleEntries + infolist = [] writer = _CWriter(CConfig) writer.write_header() for key, entry in entries: writer.write_entry(key, entry) - - f = writer.f writer.start_main() - for key, entry in entries: - writer.write_entry_main(key) writer.close() eci = CConfig._compilation_info_ - infolist = list(run_example_code(writer.path, eci)) + info = run_example_code(writer.path, eci) + for key, entry in entries: + entry.eci = eci + if key in info: + infolist.append(info[key]) + else: + infolist.append({}) assert len(infolist) == len(entries) - resultinfo = {} resultentries = {} for info, (key, entry) in zip(infolist, entries): @@ -209,9 +213,49 @@ # ____________________________________________________________ +def c_safe_string(string): + return string.replace('\\', '\\\\').replace('"', '\\"') class CConfigEntry(object): "Abstract base class." + def dump(self, name, expr): + beginpad = "__START_PLATCHECK_%s\0%s\0" % ( + c_safe_string(self.key), + c_safe_string(name)) + return ''' + struct __attribute__((packed)) { + char begin_pad[%(sizeof_beginpad)i]; + typeof(%(expr)s) contents; + char end_pad[]; + } pypy_test_%(id)s = { + .begin_pad = "%(beginpad)s", + .contents = %(expr)s, + .end_pad = "__END_PLATCHECK__" + }; + ''' % {'expr' : expr, 'beginpad' : beginpad.replace('\0', '\\0'), + 'sizeof_beginpad' : len(beginpad), + 'id' : filter(str.isalnum, self.key+'PLATCHECK'+name)} + def dumpbool(self, name, expr): + return self.dump(name, '((char)((%s)?1:0))' % (expr,)) + def bool(self, data): + return data[0] != '\x00' + def dump_by_size(self, name, expr): + beginpad = "__START_PLATCHECK_%s\0%s\0" % ( + c_safe_string(self.key), + c_safe_string(name)) + return ''' + struct { + char begin_pad[%(sizeof_beginpad)i]; + char contents[%(expr)s]; + char end_pad[]; + } pypy_test_%(id)s = { + .begin_pad = "%(beginpad)s", + .contents = {0}, + .end_pad = "__END_PLATCHECK__" + }; + ''' % {'expr' : expr, 'beginpad' : beginpad.replace('\0', '\\0'), + 'sizeof_beginpad' : len(beginpad), + 'id' : filter(str.isalnum, self.key+'PLATCHECK'+name)} class Struct(CConfigEntry): @@ -226,48 +270,37 @@ def prepare_code(self): if self.ifdef is not None: yield '#ifdef %s' % (self.ifdef,) - yield 'typedef %s platcheck_t;' % (self.name,) - yield 'typedef struct {' - yield ' char c;' - yield ' platcheck_t s;' - yield '} platcheck2_t;' - yield '' - yield 'platcheck_t s;' + platcheck_t = 'struct { char c; %s s; }' % (self.name,) if self.ifdef is not None: - yield 'dump("defined", 1);' - yield 'dump("align", offsetof(platcheck2_t, s));' - yield 'dump("size", sizeof(platcheck_t));' + yield self.dumpbool("defined", '1') + yield self.dump_by_size("align", 'offsetof(%s, s)' % (platcheck_t,)) + yield self.dump_by_size("size", 'sizeof(%s)' % (self.name,)) for fieldname, fieldtype in self.interesting_fields: - yield 'dump("fldofs %s", offsetof(platcheck_t, %s));'%( - fieldname, fieldname) - yield 'dump("fldsize %s", sizeof(s.%s));' % ( - fieldname, fieldname) + yield self.dump_by_size("fldofs " + fieldname, 'offsetof(%s, %s)' % (self.name, fieldname)) + yield self.dump_by_size("fldsize " + fieldname, 'sizeof(((%s*)0)->%s)' % ( + self.name, fieldname)) if fieldtype in integer_class: - yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname, - fieldname, - fieldname) - yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname, - fieldname) + yield self.dumpbool("fldunsigned " + fieldname, '((typeof(((%s*)0)->%s))(-1)) > 0' % (self.name, fieldname)) if self.ifdef is not None: yield '#else' - yield 'dump("defined", 0);' + yield self.dumpbool("defined", '0') yield '#endif' def build_result(self, info, config_result): if self.ifdef is not None: - if not info['defined']: + if not self.bool(info['defined']): return None - layout = [None] * info['size'] + layout = [None] * len(info['size']) for fieldname, fieldtype in self.interesting_fields: if isinstance(fieldtype, Struct): - offset = info['fldofs ' + fieldname] - size = info['fldsize ' + fieldname] + offset = len(info['fldofs ' + fieldname]) + size = len(info['fldsize ' + fieldname]) c_fieldtype = config_result.get_entry_result(fieldtype) layout_addfield(layout, offset, c_fieldtype, fieldname) else: - offset = info['fldofs ' + fieldname] - size = info['fldsize ' + fieldname] - sign = info.get('fldunsigned ' + fieldname, False) + offset = len(info['fldofs ' + fieldname]) + size = len(info['fldsize ' + fieldname]) + sign = self.bool(info.get('fldunsigned ' + fieldname, '\0')) if (size, sign) != rffi.size_and_sign(fieldtype): fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign)) layout_addfield(layout, offset, fieldtype, fieldname) @@ -294,8 +327,8 @@ seen[cell] = True name = self.name - hints = {'align': info['align'], - 'size': info['size'], + hints = {'align': len(info['align']), + 'size': len(info['size']), 'fieldoffsets': tuple(fieldoffsets), 'padding': tuple(padfields)} if name.startswith('struct '): @@ -313,34 +346,41 @@ self.name = name self.ctype_hint = ctype_hint self.ifdef = ifdef - + def prepare_code(self): if self.ifdef is not None: yield '#ifdef %s' % (self.ifdef,) - yield 'typedef %s platcheck_t;' % (self.name,) - yield '' - yield 'platcheck_t x;' if self.ifdef is not None: - yield 'dump("defined", 1);' - yield 'dump("size", sizeof(platcheck_t));' + yield self.dumpbool("defined", '1') + yield self.dump("size", '((%s)0)' % (self.name,)) if self.ctype_hint in integer_class: - yield 'x = 0; x = ~x;' - yield 'dump("unsigned", x > 0);' + yield self.dumpbool("unsigned", '((%s)(-1)) > 0' % (self.name,)) if self.ifdef is not None: yield '#else' - yield 'dump("defined", 0);' + yield self.dumpbool("defined", '0') yield '#endif' def build_result(self, info, config_result): - if self.ifdef is not None and not info['defined']: + if self.ifdef is not None and not self.bool(info['defined']): return None - size = info['size'] - sign = info.get('unsigned', False) + size = len(info['size']) + sign = self.bool(info.get('unsigned', '\0')) ctype = self.ctype_hint if (size, sign) != rffi.size_and_sign(ctype): ctype = fixup_ctype(ctype, self.name, (size, sign)) return ctype +def _load_int_le(data): + result = 0 + for i in xrange(len(data)): + result |= ord(data[i]) << (i*8) + return result +def _load_int_be(data): + result = 0 + for byte in data: + result |= ord(byte) + result <<= 8 + return result class ConstantInteger(CConfigEntry): """An entry in a CConfig class that stands for an externally @@ -350,19 +390,38 @@ self.name = name def prepare_code(self): - yield 'if ((%s) < 0) {' % (self.name,) - yield ' long long x = (long long)(%s);' % (self.name,) - yield ' printf("value: %lld\\n", x);' - yield '} else {' - yield ' unsigned long long x = (unsigned long long)(%s);' % ( - self.name,) - yield ' printf("value: %llu\\n", x);' - yield '}' + yield self.dump('value', self.name) + yield self.dump('negvalue', '-(%s)' % (self.name,)) + yield self.dump('positive', '(%s) >= 0' % (self.name,)) + + def build_result(self, info, config_result): + if self.bool(info['positive']): + value = info['value'] + else: + value = info['negvalue'] + if getendianness(self.eci) is 'BE': + magnitude = _load_int_be(value) + else: + magnitude = _load_int_le(value) + if self.bool(info['positive']): + return magnitude + else: + return -magnitude + +class ConstantString(CConfigEntry): + """An entry in a CConfig class that stands for an externally + defined string constant. + """ + def __init__(self, name): + self.name = name + + def prepare_code(self): + yield self.dump('value', self.name) def build_result(self, info, config_result): return info['value'] -class DefinedConstantInteger(CConfigEntry): +class DefinedConstantInteger(ConstantInteger): """An entry in a CConfig class that stands for an externally defined integer constant. If not #defined the value will be None. """ @@ -371,22 +430,14 @@ def prepare_code(self): yield '#ifdef %s' % self.macro - yield 'dump("defined", 1);' - yield 'if ((%s) < 0) {' % (self.macro,) - yield ' long long x = (long long)(%s);' % (self.macro,) - yield ' printf("value: %lld\\n", x);' - yield '} else {' - yield ' unsigned long long x = (unsigned long long)(%s);' % ( - self.macro,) - yield ' printf("value: %llu\\n", x);' - yield '}' - yield '#else' - yield 'dump("defined", 0);' + yield self.dumpbool('defined', '1') + for line in ConstantInteger.prepare_code(self): + yield line yield '#endif' def build_result(self, info, config_result): - if info["defined"]: - return info['value'] + if 'defined' in info: + return ConstantInteger.build_result(self, info, config_result) return None class DefinedConstantString(CConfigEntry): @@ -397,25 +448,13 @@ self.name = macro def prepare_code(self): - yield '#ifdef %s' % self.macro - yield 'int i;' - yield 'char *p = %s;' % self.macro - yield 'dump("defined", 1);' - yield 'for (i = 0; p[i] != 0; i++ ) {' - yield ' printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);' - yield '}' - yield '#else' - yield 'dump("defined", 0);' + yield '#ifdef %s' % (self.macro,) + yield self.dump('macro', self.macro) yield '#endif' def build_result(self, info, config_result): - if info["defined"]: - string = '' - d = 0 - while info.has_key('value_%d' % d): - string += chr(info['value_%d' % d]) - d += 1 - return string + if "macro" in info: + return info["macro"] return None @@ -428,13 +467,13 @@ def prepare_code(self): yield '#ifdef %s' % (self.macro,) - yield 'dump("defined", 1);' + yield self.dumpbool("defined", '1') yield '#else' - yield 'dump("defined", 0);' + yield self.dumpbool("defined", '0') yield '#endif' def build_result(self, info, config_result): - return bool(info['defined']) + return self.bool(info['defined']) class CConfigSingleEntry(object): """ An abstract class of type which requires @@ -465,10 +504,10 @@ self.name = name def prepare_code(self): - yield 'dump("size", sizeof(%s));' % self.name + yield self.dump_by_size("size", 'sizeof(%s)' % (self.name,)) def build_result(self, info, config_result): - return info['size'] + return len(info['size']) # ____________________________________________________________ # @@ -528,31 +567,33 @@ C_HEADER = """ -#include #include /* for offsetof() */ - -void dump(char* key, int value) { - printf("%s: %d\\n", key, value); -} """ def run_example_code(filepath, eci): eci = eci.convert_sources_to_files(being_main=True) files = [filepath] + [py.path.local(f) for f in eci.separate_module_files] - output = build_executable_cache(files, eci) - section = None - for line in output.splitlines(): - line = line.strip() - if line.startswith('-+- '): # start of a new section - section = {} - elif line == '---': # section end - assert section is not None - yield section - section = None - elif line: - assert section is not None - key, value = line.split(': ') - section[key] = int(value) + data = build_executable_cache_read(files, eci) + + end = 0 + section = {} + while True: + start = data.find('__START_PLATCHECK_', end) + if start >= 0: + start += len('__START_PLATCHECK_') + keyend = data.find('\0', start) + key = data[start:keyend] + nameend = data.find('\0', keyend+1) + name = data[keyend+1:nameend] + start = nameend + 1 + end = data.find('__END_PLATCHECK__', start) + if key in section: + section[key][name] = data[start:end] + else: + section[key] = {name : data[start:end]} + else: + break + return section # ____________________________________________________________ Modified: pypy/branch/new-platformcheck/pypy/tool/gcc_cache.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/tool/gcc_cache.py (original) +++ pypy/branch/new-platformcheck/pypy/tool/gcc_cache.py Tue Sep 9 13:49:18 2008 @@ -25,6 +25,15 @@ path.write(result) return result +def build_executable_cache_read(c_files, eci): + path = cache_file_path(c_files, eci, 'build_executable_cache_read') + try: + return path.read() + except py.error.Error: + result = open(build_executable(c_files, eci), 'rb').read() + path.write(result) + return result + def try_compile_cache(c_files, eci): path = cache_file_path(c_files, eci, 'try_compile_cache') try: From fijal at codespeak.net Tue Sep 9 13:53:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 13:53:58 +0200 (CEST) Subject: [pypy-svn] r58001 - pypy/branch/cross-compilation/pypy/translator/benchmark Message-ID: <20080909115358.D694716845E@codespeak.net> Author: fijal Date: Tue Sep 9 13:53:57 2008 New Revision: 58001 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Log: revert usage of py.execnet - it seems not to be the best idea. Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Tue Sep 9 13:53:57 2008 @@ -91,22 +91,31 @@ SOCK_FILE = '/tmp/bench_mem' -def run_cooperative(f, func_name=None): +def run_cooperative(func): """ The idea is that we fork() and execute the function func in one process. In parent process we measure private memory obtained by a process when we read anything in pipe. We send back that we did that """ - gw = py.execnet.PopenGateway() - if func_name is None: - func_name = f.func_name - code = py.code.Source(py.code.Source(""" - import os - channel.send(os.getpid()) - """), py.code.Source(f), py.code.Source( - "%s(channel.receive, channel.send)" % func_name)) - channel = gw.remote_exec(code) - pid = channel.receive() - return channel.receive, channel.send, pid + parentread, childwrite = os.pipe() + childread, parentwrite = os.pipe() + pid = os.fork() + if not pid: + os.close(parentread) + os.close(parentwrite) + try: + func(lambda : os.read(childread, 1), + lambda x: os.write(childwrite, x)) + except (SystemExit, KeyboardInterrupt): + pass + except: + import traceback + traceback.print_tb(sys.exc_info()[2]) + os._exit(1) + os.close(childread) + os.close(childwrite) + return (lambda : os.read(parentread, 1), + lambda x: os.write(parentwrite, x), + pid) def smaps_measure_func(pid): return parse_smaps_output(open('/proc/%d/smaps' % pid).read()) @@ -114,10 +123,7 @@ def measure(measure_func, funcs_to_run): results = [] for num, func in enumerate(funcs_to_run): - if isinstance(func, tuple): - read, write, pid = run_cooperative(*func) - else: - read, write, pid = run_cooperative(func) + read, write, pid = run_cooperative(func) elem = [] while 1: res = read() Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Tue Sep 9 13:53:57 2008 @@ -53,11 +53,7 @@ i = 0 funcs = [] while i < 10: - f = py.code.Source(list_of_instances_with_int, """ - def f(r, w): - list_of_instances_with_int(r, w, %d) - """ % coeff) - funcs.append((f, 'f')) + funcs.append(lambda r, w, coeff=coeff: list_of_instances_with_int(r, w, coeff)) coeff *= 2 i += 1 res = measure(measure_func, funcs) From fijal at codespeak.net Tue Sep 9 14:07:46 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 14:07:46 +0200 (CEST) Subject: [pypy-svn] r58002 - in pypy/branch/tuple-nonresizable-395/pypy/annotation: . test Message-ID: <20080909120746.D8CD9169F69@codespeak.net> Author: fijal Date: Tue Sep 9 14:07:44 2008 New Revision: 58002 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py Log: Revert SomeListIterator. Based on discussion with armin we should instead a) create different helpers if necessary b) grow superclass of W_TupleObject and W_ListObject (W_SeqObject) and use it's method calls instead for non-performance-critical operations. Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/binaryop.py Tue Sep 9 14:07:44 2008 @@ -20,7 +20,6 @@ from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import SomeGenericCallable from pypy.annotation.model import SomeExternalInstance, SomeUnicodeString -from pypy.annotation.model import SomeListIterator from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.rlib import rarithmetic @@ -779,17 +778,6 @@ raise UnionError("merging incompatible iterators variants") return SomeIterator(s_cont, *iter1.variant) -class __extend__(pairtype(SomeListIterator, SomeListIterator)): - def union((iter1, iter2)): - # merge here s_value of listdefs, but not listdefs themselves - # (ie resizable and mergeable parameters) - # XXX should we perform some better union??? - for listdef in iter1.listdefs: - listdef.generalize(iter2.listdefs[0].listitem.s_value) - for listdef in iter2.listdefs: - listdef.generalize(iter1.listdefs[0].listitem.s_value) - listdefs = dict.fromkeys(iter1.listdefs + iter2.listdefs).keys() - return SomeListIterator(listdefs) class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/model.py Tue Sep 9 14:07:44 2008 @@ -323,15 +323,6 @@ def can_be_none(self): return False -class SomeListIterator(SomeObject): - def __init__(self, listdefs): - self.listdefs = listdefs - - def __eq__(self, other): - if not isinstance(other, SomeListIterator): - return False - return dict.fromkeys(self.listdefs) == dict.fromkeys(other.listdefs) - class SomeInstance(SomeObject): "Stands for an instance of a (user-defined) class." Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Tue Sep 9 14:07:44 2008 @@ -443,7 +443,7 @@ def test_simple_iter_list(self): a = self.RPythonAnnotator() s = a.build_types(snippet.simple_iter, [list]) - assert isinstance(s, annmodel.SomeListIterator) + assert isinstance(s, annmodel.SomeIterator) def test_simple_iter_next(self): def f(x): @@ -3097,19 +3097,7 @@ a = self.RPythonAnnotator() py.test.raises(TooLateForChange, a.build_types, f, []) - - def test_mixing_iterators(self): - def f(i): - if i: - return iter([1,2,3]) - else: - l = [1,2,3] - l.append(4) - return iter(l) - - a = self.RPythonAnnotator() - s_res = a.build_types(f, [int]) - assert isinstance(s_res, annmodel.SomeListIterator) + def g(n): return [0,1,2,n] Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/unaryop.py Tue Sep 9 14:07:44 2008 @@ -9,7 +9,7 @@ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ - SomeGenericCallable, SomeWeakRef, SomeUnicodeString, SomeListIterator + SomeGenericCallable, SomeWeakRef, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? @@ -315,7 +315,7 @@ return SomeObject.len(lst) def iter(lst): - return SomeListIterator([lst.listdef]) + return SomeIterator(lst) iter.can_only_throw = [] def getanyitem(lst): @@ -529,15 +529,6 @@ next.can_only_throw = _can_only_throw method_next = next -class __extend__(SomeListIterator): - def iter(itr): - return itr - iter.can_only_throw = [] - - def next(itr): - return itr.listdefs[0].read_item() - next.can_only_throw = [StopIteration] - method_next = next class __extend__(SomeInstance): From fijal at codespeak.net Tue Sep 9 14:23:17 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 14:23:17 +0200 (CEST) Subject: [pypy-svn] r58003 - in pypy/branch/tuple-nonresizable-395/pypy: module/marshal objspace/std Message-ID: <20080909122317.E543316A027@codespeak.net> Author: fijal Date: Tue Sep 9 14:23:15 2008 New Revision: 58003 Added: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py (contents, props changed) Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: * Add common superclass for W_ListObject and W_TupleObject * Kill shortcut for list[slice] = tuple * Use common interface for list/tuple when marshalling hence killing two places that share wrappeditems from listobject and tupleobject Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py Tue Sep 9 14:23:15 2008 @@ -244,14 +244,29 @@ idx += 1 self.nesting -= 1 - def put_list_w(self, list_w, lng): + def put_list_w(self, lst_w, lng): self.nesting += 1 self.put_int(lng) idx = 0 space = self.space if self.nesting < MAX_MARSHAL_DEPTH: while idx < lng: - w_obj = list_w[idx] + w_obj = lst_w[idx] + self.space.marshal_w(w_obj, self) + idx += 1 + else: + self._overflow() + self.nesting -= 1 + + def put_w_seq(self, w_seq): + self.nesting += 1 + lng = w_seq.getlength() + self.put_int(lng) + idx = 0 + space = self.space + if self.nesting < MAX_MARSHAL_DEPTH: + while idx < lng: + w_obj = w_seq.getitem(idx) self.space.marshal_w(w_obj, self) idx += 1 else: Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Tue Sep 9 14:23:15 2008 @@ -3,13 +3,14 @@ from pypy.objspace.std.listtype import get_list_index from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.tupleobject import W_TupleObject +from pypy.objspace.std.seqinterface import W_SeqObject from pypy.objspace.std import slicetype from pypy.interpreter import gateway, baseobjspace from pypy.rlib.listsort import TimSort -class W_ListObject(W_Object): +class W_ListObject(W_SeqObject): from pypy.objspace.std.listtype import list_typedef as typedef def __init__(w_self, wrappeditems): @@ -253,10 +254,6 @@ l = w_list2.wrappeditems return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) -def setitem__List_Slice_Tuple(space, w_list, w_slice, w_tuple): - t = w_tuple.wrappeditems - return _setitem_slice_helper(space, w_list, w_slice, t, len(t)) - def setitem__List_Slice_ANY(space, w_list, w_slice, w_iterable): l = space.unpackiterable(w_iterable) return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Tue Sep 9 14:23:15 2008 @@ -87,7 +87,8 @@ put_int(int) puts an integer put_pascal(s) puts a short string put_w_obj(w_obj) puts a wrapped object -put_list_w(list_w, lng) puts a list of lng wrapped objects +put_list_w(lst_w, lng) puts a list with lng of wrapped objects +put_w_seq(w_seq) puts a w_seqobject subclass """ @@ -315,8 +316,7 @@ def marshal_w__Tuple(space, w_tuple, m): m.start(TYPE_TUPLE) - items = w_tuple.wrappeditems - m.put_list_w(items, len(items)) + m.put_w_seq(w_tuple) def unmarshal_Tuple(space, u, tc): items_w = u.get_list_w() @@ -325,8 +325,7 @@ def marshal_w__List(space, w_list, m): m.start(TYPE_LIST) - items = w_list.wrappeditems - m.put_list_w(items, len(items)) + m.put_w_seq(w_list) def unmarshal_List(space, u, tc): items_w = u.get_list_w() Added: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py ============================================================================== --- (empty file) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py Tue Sep 9 14:23:15 2008 @@ -0,0 +1,15 @@ + +from pypy.objspace.std.objspace import W_Object + +class W_SeqObject(W_Object): + """ This is a common superclass for W_ListObject and W_TupleObject. + it's purpose is to have some methods that present common interface + to accessing items from interp-level. The whole idea is to not have + wrappeditems of both shared + """ + + def getlength(self): + return len(self.wrappeditems) + + def getitem(self, i): + return self.wrappeditems[i] Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Tue Sep 9 14:23:15 2008 @@ -1,4 +1,5 @@ from pypy.objspace.std.objspace import * +from pypy.objspace.std.seqinterface import W_SeqObject from pypy.objspace.std.inttype import wrapint from pypy.rlib.rarithmetic import intmask from pypy.objspace.std.sliceobject import W_SliceObject @@ -6,7 +7,7 @@ from pypy.rlib.debug import make_sure_not_resized from pypy.annotation import model as annmodel -class W_TupleObject(W_Object): +class W_TupleObject(W_SeqObject): from pypy.objspace.std.tupletype import tuple_typedef as typedef def __init__(w_self, wrappeditems): From fijal at codespeak.net Tue Sep 9 17:15:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 17:15:20 +0200 (CEST) Subject: [pypy-svn] r58005 - in pypy/branch/tuple-nonresizable-395/pypy: module/marshal objspace/std Message-ID: <20080909151520.2C7CF169EC4@codespeak.net> Author: fijal Date: Tue Sep 9 17:15:19 2008 New Revision: 58005 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/model.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: *IN-PROGRESS* Create 2 fastseqiters for tuple and list. avoid some other merges in marshal Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py Tue Sep 9 17:15:19 2008 @@ -486,28 +486,34 @@ return w_ret # inlined version to save a nesting level - def get_list_w(self): - self.nesting += 1 - lng = self.get_lng() - res_w = [None] * lng - idx = 0 - space = self.space - w_ret = space.w_None # something not None - if self.nesting < MAX_MARSHAL_DEPTH: - while idx < lng: - tc = self.get1() - w_ret = self._dispatch[ord(tc)](space, self, tc) - if w_ret is None: - break - res_w[idx] = w_ret - idx += 1 - else: - self._overflow() - if w_ret is None: - raise OperationError(space.w_TypeError, space.wrap( - 'NULL object in marshal data')) - self.nesting -= 1 - return res_w + def _new_get_list_w(): + def get_list_w(self): + self.nesting += 1 + lng = self.get_lng() + res_w = [None] * lng + idx = 0 + space = self.space + w_ret = space.w_None # something not None + if self.nesting < MAX_MARSHAL_DEPTH: + while idx < lng: + tc = self.get1() + w_ret = self._dispatch[ord(tc)](space, self, tc) + if w_ret is None: + break + res_w[idx] = w_ret + idx += 1 + else: + self._overflow() + if w_ret is None: + raise OperationError(space.w_TypeError, space.wrap( + 'NULL object in marshal data')) + self.nesting -= 1 + return res_w + return get_list_w + + get_list_w = _new_get_list_w() + # another version not to degenerate resulting list to resizable + get_tuple_w = _new_get_list_w() def _overflow(self): self.raise_exc('object too deeply nested to unmarshal') Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py Tue Sep 9 17:15:19 2008 @@ -19,13 +19,21 @@ class W_SeqIterObject(W_AbstractSeqIterObject): """Sequence iterator implementation for general sequences.""" -class W_FastSeqIterObject(W_AbstractSeqIterObject): - """Sequence iterator specialized for lists or tuples, accessing +class W_FastListIterObject(W_AbstractSeqIterObject): + """Sequence iterator specialized for lists, accessing directly their RPython-level list of wrapped objects. """ def __init__(w_self, w_seq, wrappeditems): W_AbstractSeqIterObject.__init__(w_self, w_seq) - w_self.wrappeditems = wrappeditems + w_self.listitems = wrappeditems + +class W_FastTupleIterObject(W_AbstractSeqIterObject): + """Sequence iterator specialized for tuples, accessing + directly their RPython-level list of wrapped objects. + """ + def __init__(w_self, w_seq, wrappeditems): + W_AbstractSeqIterObject.__init__(w_self, w_seq) + w_self.tupleitems = wrappeditems class W_ReverseSeqIterObject(W_Object): from pypy.objspace.std.itertype import reverse_iter_typedef as typedef @@ -37,7 +45,8 @@ registerimplementation(W_SeqIterObject) -registerimplementation(W_FastSeqIterObject) +registerimplementation(W_FastListIterObject) +registerimplementation(W_FastTupleIterObject) registerimplementation(W_ReverseSeqIterObject) def iter__SeqIter(space, w_seqiter): @@ -67,26 +76,52 @@ return w_len -def iter__FastSeqIter(space, w_seqiter): +def iter__FastTupleIter(space, w_seqiter): + return w_seqiter + +def next__FastTupleIter(space, w_seqiter): + if w_seqiter.tupleitems is None: + raise OperationError(space.w_StopIteration, space.w_None) + index = w_seqiter.index + try: + w_item = w_seqiter.tupleitems[index] + except IndexError: + w_seqiter.tupleitems = None + w_seqiter.w_seq = None + raise OperationError(space.w_StopIteration, space.w_None) + w_seqiter.index = index + 1 + return w_item + +def len__FastTupleIter(space, w_seqiter): + if w_seqiter.tupleitems is None: + return space.wrap(0) + totallength = len(w_seqiter.tupleitems) + remaining = totallength - w_seqiter.index + if remaining < 0: + remaining = 0 + return space.wrap(remaining) + + +def iter__FastListIter(space, w_seqiter): return w_seqiter -def next__FastSeqIter(space, w_seqiter): - if w_seqiter.wrappeditems is None: +def next__FastListIter(space, w_seqiter): + if w_seqiter.listitems is None: raise OperationError(space.w_StopIteration, space.w_None) index = w_seqiter.index try: - w_item = w_seqiter.wrappeditems[index] + w_item = w_seqiter.listitems[index] except IndexError: - w_seqiter.wrappeditems = None + w_seqiter.listitems = None w_seqiter.w_seq = None raise OperationError(space.w_StopIteration, space.w_None) w_seqiter.index = index + 1 return w_item -def len__FastSeqIter(space, w_seqiter): - if w_seqiter.wrappeditems is None: +def len__FastListIter(space, w_seqiter): + if w_seqiter.listitems is None: return space.wrap(0) - totallength = len(w_seqiter.wrappeditems) + totallength = len(w_seqiter.listitems) remaining = totallength - w_seqiter.index if remaining < 0: remaining = 0 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Tue Sep 9 17:15:19 2008 @@ -14,6 +14,9 @@ from pypy.objspace.std.listtype import list_typedef as typedef def __init__(w_self, wrappeditems): + if len(wrappeditems) > 0: + elem = wrappeditems.pop() + wrappeditems.append(elem) w_self.wrappeditems = wrappeditems def __repr__(w_self): @@ -96,7 +99,7 @@ def iter__List(space, w_list): from pypy.objspace.std import iterobject - return iterobject.W_FastSeqIterObject(w_list, w_list.wrappeditems) + return iterobject.W_FastListIterObject(w_list, w_list.wrappeditems) def add__List_List(space, w_list1, w_list2): return W_ListObject(w_list1.wrappeditems + w_list2.wrappeditems) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Tue Sep 9 17:15:19 2008 @@ -319,7 +319,7 @@ m.put_w_seq(w_tuple) def unmarshal_Tuple(space, u, tc): - items_w = u.get_list_w() + items_w = u.get_tuple_w() return space.newtuple(items_w) register(TYPE_TUPLE, unmarshal_Tuple) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/model.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/model.py Tue Sep 9 17:15:19 2008 @@ -103,7 +103,8 @@ longobject.W_LongObject: [], noneobject.W_NoneObject: [], iterobject.W_SeqIterObject: [], - iterobject.W_FastSeqIterObject: [], + iterobject.W_FastListIterObject: [], + iterobject.W_FastTupleIterObject: [], iterobject.W_ReverseSeqIterObject: [], unicodeobject.W_UnicodeObject: [], dictproxyobject.W_DictProxyObject: [], Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Tue Sep 9 17:15:19 2008 @@ -66,7 +66,7 @@ def iter__Tuple(space, w_tuple): from pypy.objspace.std import iterobject - return iterobject.W_FastSeqIterObject(w_tuple, w_tuple.wrappeditems) + return iterobject.W_FastTupleIterObject(w_tuple, w_tuple.wrappeditems) def add__Tuple_Tuple(space, w_tuple1, w_tuple2): items1 = w_tuple1.wrappeditems From antocuni at codespeak.net Tue Sep 9 17:48:36 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 9 Sep 2008 17:48:36 +0200 (CEST) Subject: [pypy-svn] r58006 - pypy/extradoc/talk/pycon-uk-2008/jit Message-ID: <20080909154836.C7352169FE0@codespeak.net> Author: antocuni Date: Tue Sep 9 17:48:34 2008 New Revision: 58006 Added: pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.pdf (contents, props changed) Log: generate pdf Added: pypy/extradoc/talk/pycon-uk-2008/jit/pypy-vm.pdf ============================================================================== Binary file. No diff available. From fijal at codespeak.net Tue Sep 9 17:52:14 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 17:52:14 +0200 (CEST) Subject: [pypy-svn] r58007 - pypy/branch/tuple-nonresizable-395/pypy/rlib Message-ID: <20080909155214.1669C169FE0@codespeak.net> Author: fijal Date: Tue Sep 9 17:52:13 2008 New Revision: 58007 Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py Log: those functions return they're arguments Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py Tue Sep 9 17:52:13 2008 @@ -64,7 +64,7 @@ callable which checks if annotation is as expected, arguments passed are (current annotation, bookkeeper) """ - pass + return arg class Entry(ExtRegistryEntry): _about_ = check_annotation @@ -84,7 +84,7 @@ """ Function checking whether annotation of SomeList is never resized, useful for debugging. Does nothing when run directly """ - pass + return arg class Entry(ExtRegistryEntry): _about_ = make_sure_not_resized From hpk at codespeak.net Tue Sep 9 18:35:21 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 9 Sep 2008 18:35:21 +0200 (CEST) Subject: [pypy-svn] r58008 - pypy/branch/pypy-pytrunk/pypy/tool/test Message-ID: <20080909163521.F0BF3169FEE@codespeak.net> Author: hpk Date: Tue Sep 9 18:35:19 2008 New Revision: 58008 Modified: pypy/branch/pypy-pytrunk/pypy/tool/test/test_conftest1.py Log: fix tests by using some test machinery more directly Modified: pypy/branch/pypy-pytrunk/pypy/tool/test/test_conftest1.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/test/test_conftest1.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/test/test_conftest1.py Tue Sep 9 18:35:19 2008 @@ -2,39 +2,24 @@ import py innertest = py.magic.autopath().dirpath('conftest1_innertest.py') -from py.__.test.terminal.terminal import TerminalSession -from py.__.test.outcome import Passed, Failed, Skipped +from py.__.test.testing import suptest -class TestPyPyTests: +class TestPyPyTests(suptest.InlineSession): def test_select_interplevel(self): - config = py.test.config._reparse([innertest, '-k', 'interplevel']) - session = TerminalSession(config, py.std.sys.stdout) - session.main() - l = session.getitemoutcomepairs(Passed) - assert len(l) == 2 - for item in l: - assert item[0].name in ('test_something', 'test_method') - #item = l[0][0] - #assert item.name == 'test_one' - l = session.getitemoutcomepairs(Skipped) - assert len(l) == 2 - for item in l: - assert item[0].name in ('app_test_something', 'test_method_app') + sorter = self.parse_and_run("-k", "interplevel", innertest) + passed, skipped, failed = sorter.listoutcomes() + assert len(passed) == 2 + assert not skipped and not failed + for repevent in passed: + assert repevent.colitem.name in ('test_something', 'test_method') def test_select_applevel(self): - config = py.test.config._reparse([innertest, '-k', 'applevel']) - session = TerminalSession(config, py.std.sys.stdout) - session.main() - l = session.getitemoutcomepairs(Passed) - assert len(l) == 2 - for item in l: - assert item[0].name in ('app_test_something', 'test_method_app') - #item = l[0][0] - #assert item.name == 'test_one' - l = session.getitemoutcomepairs(Skipped) - assert len(l) == 2 - for item in l: - assert item[0].name in ('test_something', 'test_method') + sorter = self.parse_and_run("-k", "applevel", innertest) + passed, skipped, failed = sorter.listoutcomes() + assert len(passed) == 2 + assert not skipped and not failed + for repevent in passed: + assert repevent.colitem.name in ('app_test_something', 'test_method_app') def XXX_test_appdirect(self): config = py.test.config._reparse([innertest, From fijal at codespeak.net Tue Sep 9 18:57:09 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 18:57:09 +0200 (CEST) Subject: [pypy-svn] r58011 - in pypy/branch/tuple-nonresizable-395/pypy: interpreter objspace/std Message-ID: <20080909165709.A45B2169F85@codespeak.net> Author: fijal Date: Tue Sep 9 18:57:08 2008 New Revision: 58011 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Log: Some IN-PROGRESS checkin, playing around Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py Tue Sep 9 18:57:08 2008 @@ -8,6 +8,7 @@ from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -657,6 +658,12 @@ (i, plural)) return items + def immutableiterable(self, w_iterable): + """ More or less the same as unpackiterable, but does not return + a copy. Please don't modify the result + """ + return make_sure_not_resized(self.unpackiterable(w_iterable)[:]) + def unpacktuple(self, w_tuple, expected_length=-1): """Same as unpackiterable(), but only for tuples. Only use for bootstrapping or performance reasons.""" Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Tue Sep 9 18:57:08 2008 @@ -257,9 +257,13 @@ l = w_list2.wrappeditems return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) +def setitem__List_Slice_Tuple(space, w_tuple, w_slice, w_iterable): + l = w_tuple.wrappeditems + return _setitem_slice_helper2(space, w_tuple, w_slice, l, len(l)) + def setitem__List_Slice_ANY(space, w_list, w_slice, w_iterable): - l = space.unpackiterable(w_iterable) - return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) + l = space.immutableiterable(w_iterable) + return _setitem_slice_helper2(space, w_list, w_slice, l, len(l)) def _setitem_slice_helper(space, w_list, w_slice, sequence2, len2): oldsize = len(w_list.wrappeditems) @@ -306,6 +310,57 @@ start += step return space.w_None + +def _setitem_slice_helper2(space, w_list, w_slice, sequence2, len2): + make_sure_not_resized(sequence2) + oldsize = len(w_list.wrappeditems) + start, stop, step, slicelength = w_slice.indices4(space, oldsize) + assert slicelength >= 0 + items = w_list.wrappeditems + + if step == 1: # Support list resizing for non-extended slices + delta = len2 - slicelength + if delta >= 0: + newsize = oldsize + delta + # XXX support this in rlist! + items += [None] * delta + lim = start+len2 + i = newsize - 1 + while i >= lim: + items[i] = items[i-delta] + i -= 1 + else: + # shrinking requires the careful memory management of _del_slice() + _del_slice(w_list, start, start-delta) + elif len2 != slicelength: # No resize for extended slices + raise OperationError(space.w_ValueError, space.wrap("attempt to " + "assign sequence of size %d to extended slice of size %d" % + (len2,slicelength))) + + if sequence2 is items: + if step > 0: + # Always copy starting from the right to avoid + # having to make a shallow copy in the case where + # the source and destination lists are the same list. + i = len2 - 1 + start += i*step + while i >= 0: + items[start] = sequence2[i] + start -= step + i -= 1 + return space.w_None + else: + # Make a shallow copy to more easily handle the reversal case + sequence2 = list(sequence2) + for i in range(len2): + items[start] = sequence2[i] + start += step + return space.w_None +#_setitem_slice_helper.func_name = name + +#_setitem_slice_helper = _new_slice_helper('_setitem_slice_helper') +#_setitem_slice_helper2 = _new_slice_helper('_setitem_slice_helper2') + app = gateway.applevel(""" def listrepr(currently_in_repr, l): 'The app-level part of repr().' From fijal at codespeak.net Tue Sep 9 19:38:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 19:38:45 +0200 (CEST) Subject: [pypy-svn] r58014 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080909173845.15D88169E4D@codespeak.net> Author: fijal Date: Tue Sep 9 19:38:45 2008 New Revision: 58014 Added: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: Add a todo list, not to forget before merge Added: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- (empty file) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Tue Sep 9 19:38:45 2008 @@ -0,0 +1,4 @@ + +* make sure to kill W_ListObject.__init__ hack +* avoid duplication in FastSeqIter, if possible +* add objspace.unpackiterable shortcuts From fijal at codespeak.net Tue Sep 9 19:39:30 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 19:39:30 +0200 (CEST) Subject: [pypy-svn] r58015 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080909173930.A8E0A169E4D@codespeak.net> Author: fijal Date: Tue Sep 9 19:39:29 2008 New Revision: 58015 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Log: Kill shortcut for Tuple Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Tue Sep 9 19:39:29 2008 @@ -257,13 +257,9 @@ l = w_list2.wrappeditems return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) -def setitem__List_Slice_Tuple(space, w_tuple, w_slice, w_iterable): - l = w_tuple.wrappeditems - return _setitem_slice_helper2(space, w_tuple, w_slice, l, len(l)) - def setitem__List_Slice_ANY(space, w_list, w_slice, w_iterable): - l = space.immutableiterable(w_iterable) - return _setitem_slice_helper2(space, w_list, w_slice, l, len(l)) + l = space.unpackiterable(w_iterable) + return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) def _setitem_slice_helper(space, w_list, w_slice, sequence2, len2): oldsize = len(w_list.wrappeditems) @@ -310,57 +306,6 @@ start += step return space.w_None - -def _setitem_slice_helper2(space, w_list, w_slice, sequence2, len2): - make_sure_not_resized(sequence2) - oldsize = len(w_list.wrappeditems) - start, stop, step, slicelength = w_slice.indices4(space, oldsize) - assert slicelength >= 0 - items = w_list.wrappeditems - - if step == 1: # Support list resizing for non-extended slices - delta = len2 - slicelength - if delta >= 0: - newsize = oldsize + delta - # XXX support this in rlist! - items += [None] * delta - lim = start+len2 - i = newsize - 1 - while i >= lim: - items[i] = items[i-delta] - i -= 1 - else: - # shrinking requires the careful memory management of _del_slice() - _del_slice(w_list, start, start-delta) - elif len2 != slicelength: # No resize for extended slices - raise OperationError(space.w_ValueError, space.wrap("attempt to " - "assign sequence of size %d to extended slice of size %d" % - (len2,slicelength))) - - if sequence2 is items: - if step > 0: - # Always copy starting from the right to avoid - # having to make a shallow copy in the case where - # the source and destination lists are the same list. - i = len2 - 1 - start += i*step - while i >= 0: - items[start] = sequence2[i] - start -= step - i -= 1 - return space.w_None - else: - # Make a shallow copy to more easily handle the reversal case - sequence2 = list(sequence2) - for i in range(len2): - items[start] = sequence2[i] - start += step - return space.w_None -#_setitem_slice_helper.func_name = name - -#_setitem_slice_helper = _new_slice_helper('_setitem_slice_helper') -#_setitem_slice_helper2 = _new_slice_helper('_setitem_slice_helper2') - app = gateway.applevel(""" def listrepr(currently_in_repr, l): 'The app-level part of repr().' From cami at codespeak.net Tue Sep 9 19:40:43 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 9 Sep 2008 19:40:43 +0200 (CEST) Subject: [pypy-svn] r58016 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080909174043.6E697169EE1@codespeak.net> Author: cami Date: Tue Sep 9 19:40:41 2008 New Revision: 58016 Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/video.py Log: refactoring: added background and window object adpated tests Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Tue Sep 9 19:40:41 2008 @@ -35,11 +35,11 @@ assert video.line_y == 0 assert video.line_y_compare == 0 assert video.dma == 0xFF - assert video.scroll_x == 0 - assert video.scroll_y == 0 - assert video.window_x == 0 - assert video.window_y == 0 - assert video.window_line_y == 0 + assert video.background.scroll_x == 0 + assert video.background.scroll_y == 0 + assert video.window.x == 0 + assert video.window.y == 0 + assert video.window.line_y == 0 assert video.background_palette == 0xFC assert video.object_palette_0 == 0xFF assert video.object_palette_1 == 0xFF @@ -57,16 +57,16 @@ def test_read_write_properties(): video = get_video() - checks = [(0xFF42, "scroll_y"), - (0xFF43, "scroll_x"), + checks = [(0xFF42, video.get_scroll_y), + (0xFF43, video.get_scroll_x), #(0xFF44, "line_y"), read only - (0xFF45, "line_y_compare"), - (0xFF46, "dma"), - (0xFF47, "background_palette"), - (0xFF48, "object_palette_0"), - (0xFF49, "object_palette_1"), - (0xFF4A, "window_y"), - (0xFF4B, "window_x")] + (0xFF45, video.get_line_y_compare), + (0xFF46, video.get_dma), + (0xFF47, video.get_background_palette), + (0xFF48, video.get_object_palette_0), + (0xFF49, video.get_object_palette_1), + (0xFF4A, video.get_window_y), + (0xFF4B, video.get_window_x)] counted_value = 0 for check in checks: address = check[0] @@ -75,7 +75,7 @@ if len(check) > 2: value = check[2] video.write(address, value) - assert video.__getattribute__(property) == value + assert property() == value assert video.read(address) == value counted_value = (counted_value + 1 ) % 0xFF @@ -155,24 +155,24 @@ def test_control(): video = get_video() video.control.write(0x80) - video.window_line_y = 1 + video.window.line_y = 1 video.write(0xFF40, 0x80) assert video.control.read() == 0x80 - assert video.window_line_y == 1 + assert video.window.line_y == 1 def test_control_window_draw_skip(): video = get_video() video.control.write(0x80) - video.window_y = 0 + video.window.y = 0 video.line_y = 1 - video.window_line_y = 0 + video.window.line_y = 0 video.write(0xFF40, 0x80+0x20) assert video.control.read() == 0x80+0x20 - assert video.window_line_y == 144 + assert video.window.line_y == 144 def test_control_reset1(): video = get_video() Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 9 19:40:41 2008 @@ -228,7 +228,7 @@ self.emulate_v_blank_mode_1() else: self.video.line_y = 0 - self.video.window_line_y = 0 + self.video.window.line_y = 0 self.set_between() self.emulate_hblank_line_y_compare() @@ -271,7 +271,6 @@ def emulate_oam(self): self.video.status.set_mode(3) - class Mode3(Mode): """ Mode 3: The LCD controller is reading from both OAM and VRAM, @@ -408,7 +407,6 @@ def set_data(self, byte0=None, byte1=None, byte2=None, byte3=None): """ extracts the sprite data from an oam entry - """ if byte0 is not None: self.extract_y_position(byte0) @@ -503,7 +501,30 @@ def get_tile_data(self): pass +# ----------------------------------------------------------------------------- + +class Window(object): + + def __init__(self): + self.reset() + + def reset(self): + self.x = 0 + self.y = 0 + self.line_y = 0 + +class Background(object): + + def __init__(self): + self.reset() + + def reset(self): + # SCROLLX and SCROLLY hold the coordinates of background to + # be displayed in the left upper corner of the screen. + self.scroll_x = 0 + self.scroll_y = 0 + # ----------------------------------------------------------------------------- class Video(iMemory): @@ -514,42 +535,53 @@ self.v_blank_interrupt_flag = interrupt.v_blank self.lcd_interrupt_flag = interrupt.lcd self.control = ControlRegister() + self.window = Window() self.status = StatusRegister(self) + self.background = Background() self.memory = memory - self.create_tiles() + self.create_tile_maps() self.create_sprites() self.reset() - def create_tiles(self): - self.tiles = [None] + def create_tile_maps(self): + # create the maxumal possible sprites + self.tile_map_1 = [None]*32*32 + self.tile_map_2 = [None]*32*32 + + def update_tiles(self): + pass def create_sprites(self): self.sprites = [None] * 40 for i in range(40): self.sprites[i] = Sprite() + def update_sprites(self): + for i in range(40): + address = 1 * 4 + self.sprites[i].set_data(self.oam[address + 0], + self.oam[address + 1], + self.oam[address + 2], + self.oam[address + 3]) + + def reset(self): - self.cycles = constants.MODE_2_TICKS self.control.reset() self.status.reset() + self.background.reset() + self.window.reset() + self.cycles = constants.MODE_2_TICKS self.line_y = 0 self.line_y_compare = 0 self.dma = 0xFF - # SCROLLX and SCROLLY hold the coordinates of background to - # be displayed in the left upper corner of the screen. - self.scroll_x = 0 - self.scroll_y = 0 # window position - self.window_x = 0 - self.window_y = 0 - self.window_line_y = 0 self.background_palette = 0xFC self.object_palette_0 = 0xFF self.object_palette_1 = 0xFF self.transfer = True self.display = True - self.v_blank = True + self.v_blank = True self.dirty = True self.vram = [0] * constants.VRAM_SIZE @@ -648,8 +680,8 @@ self.reset_control(data) # don't draw window if it was not enabled and not being drawn before if not self.control.window_enabled and (data & 0x20) != 0 and \ - self.window_line_y == 0 and self.line_y > self.window_y: - self.window_line_y = 144 + self.window.line_y == 0 and self.line_y > self.window.y: + self.window.line_y = 144 self.control.write(data) def reset_control(self, data): @@ -680,7 +712,7 @@ def get_scroll_x(self): """ see set_scroll_x """ - return self.scroll_x + return self.background.scroll_x def set_scroll_x(self, data): """ @@ -690,14 +722,15 @@ controller automatically wraps back to the upper (left) position in BG map when drawing exceeds the lower (right) border of the BG map area. """ - self.scroll_x = data + self.background.scroll_x = data + def get_scroll_y(self): """ see set_scroll_x """ - return self.scroll_y + return self.background.scroll_y def set_scroll_y(self, data): """ see set_scroll_x """ - self.scroll_y = data + self.background.scroll_y = data def get_line_y(self): """ see set_line_y """ @@ -757,14 +790,6 @@ self.oam[index] = self.memory.read((self.dma << 8) + index) self.update_sprites() - def update_sprites(self): - for i in range(40): - address = 1 * 4 - self.sprites[i].set_data(self.oam[address + 0], - self.oam[address + 1], - self.oam[address + 2], - self.oam[address + 3]) - def get_background_palette(self): """ see set_background_palette""" return self.background_palette @@ -814,8 +839,8 @@ self.dirty = True def get_window_y(self): - """ see set_window_y """ - return self.window_y + """ see set_window.y """ + return self.window.y def set_window_y(self, data): """ @@ -827,22 +852,25 @@ WX=0..166, WY=0..143. A postion of WX=7, WY=0 locates the window at upper left, it is then completly covering normal background. """ - self.window_y = data + self.window.y = data def get_window_x(self): - return self.window_x + return self.window.x def set_window_x(self, data): - self.window_x = data + self.window.x = data def set_oam(self, address, data): self.oam[address - constants.OAM_ADDR] = data & 0xFF + #self.update_sprites(address) + self.update_sprites() def get_oam(self, address): return self.oam[address - constants.OAM_ADDR] def set_vram(self, address, data): self.vram[address - constants.VRAM_ADDR] = data & 0xFF + self.update_tiles() def get_vram(self, address): return self.vram[address - constants.VRAM_ADDR] @@ -885,8 +913,8 @@ self.line[x] = 0x00 def draw_background(self): - y = (self.scroll_y + self.line_y) & 0xFF - x = self.scroll_x & 0xFF + y = (self.background.scroll_y + self.line_y) & 0xFF + x = self.background.scroll_x & 0xFF tile_map, tile_data = self.prepare_background_data(x, y) self.draw_tiles(8 - (x & 7), tile_map, tile_data) @@ -898,18 +926,19 @@ return tile_map, tile_data def draw_window(self): - if self.line_y < self.window_y or self.window_x >= 167 or \ - self.window_line_y >= 144: + if self.line_y < self.window.y or self.window.x >= 167 or \ + self.window.line_y >= 144: return - tile_map, tile_data = self.prepare_window_data() - self.draw_tiles(self.window_x + 1, tile_map, tile_data) - self.window_line_y += 1 + else: + tile_map, tile_data = self.prepare_window_data() + self.draw_tiles(self.window.x + 1, tile_map, tile_data) + self.window.line_y += 1 def prepare_window_data(self): tile_map = self.get_tile_map(0x40) - tile_map += (self.window_line_y >> 3) << 5 + tile_map += (self.window.line_y >> 3) << 5 tile_data = self.get_tile_data(0x10) - tile_data += (self.window_line_y & 7) << 1 + tile_data += (self.window.line_y & 7) << 1 return tile_map, tile_data; def get_tile_map(self, mask): From fijal at codespeak.net Tue Sep 9 19:48:36 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 19:48:36 +0200 (CEST) Subject: [pypy-svn] r58018 - pypy/branch/tuple-nonresizable-395/pypy/module/posix Message-ID: <20080909174836.4445D169FBC@codespeak.net> Author: fijal Date: Tue Sep 9 19:48:35 2008 New Revision: 58018 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Log: Avoid list here being resizable. Modified: pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Tue Sep 9 19:48:35 2008 @@ -97,7 +97,7 @@ FIELDS = PORTABLE_STAT_FIELDS else: FIELDS = STAT_FIELDS # also when not translating at all - lst = [] + lst = [None] * ll_os_stat.N_INDEXABLE_FIELDS w_keywords = space.newdict() for i, (name, TYPE) in FIELDS: value = getattr(st, name) @@ -105,7 +105,7 @@ # value = int(value) # rounded to an integer for indexed access w_value = space.wrap(value) if i < ll_os_stat.N_INDEXABLE_FIELDS: - lst.append(w_value) + lst[i] = w_value else: space.setitem(w_keywords, space.wrap(name), w_value) From fijal at codespeak.net Tue Sep 9 21:52:18 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 21:52:18 +0200 (CEST) Subject: [pypy-svn] r58023 - pypy/branch/tuple-nonresizable-395/pypy/annotation Message-ID: <20080909195218.DF49016A25F@codespeak.net> Author: fijal Date: Tue Sep 9 21:52:15 2008 New Revision: 58023 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/bookkeeper.py Log: unpackiterable should always return a list Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/bookkeeper.py Tue Sep 9 21:52:15 2008 @@ -737,7 +737,7 @@ if (expected_length is not None and expected_length != len(s_obj.items)): raise ValueError - return s_obj.items + return list(s_obj.items) if (s_obj.__class__ is SomeObject and getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] From fijal at codespeak.net Tue Sep 9 21:54:44 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 21:54:44 +0200 (CEST) Subject: [pypy-svn] r58024 - pypy/branch/tuple-nonresizable-395/pypy/interpreter Message-ID: <20080909195444.9558E16A258@codespeak.net> Author: fijal Date: Tue Sep 9 21:54:42 2008 New Revision: 58024 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Log: Make sure that newtuple receives a fixed-size list in various places. Rename immutableiterable to viewiterable Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py Tue Sep 9 21:54:42 2008 @@ -315,7 +315,10 @@ def __init__(self, space, args_w, kwds_w=None, w_stararg=None, w_starstararg=None): self.space = space + assert isinstance(args_w, list) self.arguments_w = args_w + from pypy.rlib.debug import make_sure_not_resized + make_sure_not_resized(self.arguments_w) self.kwds_w = kwds_w self.w_stararg = w_stararg self.w_starstararg = w_starstararg @@ -360,7 +363,8 @@ "unpack the *arg and **kwd into w_arguments and kwds_w" # --- unpack the * argument now --- if self.w_stararg is not None: - self.arguments_w += self.space.unpackiterable(self.w_stararg) + self.arguments_w = (self.arguments_w + + self.space.unpackiterable(self.w_stararg)) self.w_stararg = None # --- unpack the ** argument now --- if self.kwds_w is None: @@ -405,8 +409,9 @@ if len(self.arguments_w) > argcount: raise ValueError, "too many arguments (%d expected)" % argcount if self.w_stararg is not None: - self.arguments_w += self.space.unpackiterable(self.w_stararg, - argcount - len(self.arguments_w)) + self.arguments_w = (self.arguments_w + + self.space.unpackiterable(self.w_stararg, + argcount - len(self.arguments_w))) self.w_stararg = None elif len(self.arguments_w) < argcount: raise ValueError, "not enough arguments (%d expected)" % argcount Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py Tue Sep 9 21:54:42 2008 @@ -658,7 +658,7 @@ (i, plural)) return items - def immutableiterable(self, w_iterable): + def viewiterable(self, w_iterable): """ More or less the same as unpackiterable, but does not return a copy. Please don't modify the result """ Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py Tue Sep 9 21:54:42 2008 @@ -244,7 +244,7 @@ values_w = self.defs_w if not values_w: return space.w_None - return space.newtuple(values_w) + return space.newtuple(values_w[:]) def fset_func_defaults(space, self, w_defaults): if space.is_w(w_defaults, space.w_None): Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Tue Sep 9 21:54:42 2008 @@ -240,10 +240,10 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w) + return space.newtuple(self.co_consts_w[:]) def fget_co_names(space, self): - return space.newtuple(self.co_names_w) + return space.newtuple(self.co_names_w[:]) def fget_co_varnames(space, self): return space.newtuple([space.wrap(name) for name in self.co_varnames]) @@ -351,8 +351,8 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w), - space.newtuple(self.co_names_w), + space.newtuple(self.co_consts_w[:]), + space.newtuple(self.co_names_w[:]), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), From fijal at codespeak.net Tue Sep 9 21:58:21 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 9 Sep 2008 21:58:21 +0200 (CEST) Subject: [pypy-svn] r58025 - in pypy/branch/tuple-nonresizable-395/pypy: interpreter module/__builtin__ objspace/std Message-ID: <20080909195821.86CFB169FE0@codespeak.net> Author: fijal Date: Tue Sep 9 21:58:20 2008 New Revision: 58025 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupletype.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Log: Adapt few other places not to merge the list Make hack in listobject.__init__ slightly different Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/function.py Tue Sep 9 21:58:20 2008 @@ -213,7 +213,7 @@ w(self.code), self.w_func_globals, w_closure, - nt(self.defs_w), + nt(self.defs_w[:]), self.w_func_dict, self.w_module, ] Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py Tue Sep 9 21:58:20 2008 @@ -116,7 +116,7 @@ elif name == "__name__": return space.wrap(self.name) elif name == "__bases__": - return space.newtuple(self.bases_w) + return space.newtuple(self.bases_w[:]) w_value = self.lookup(space, w_attr) if w_value is None: raise OperationError( Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Tue Sep 9 21:58:20 2008 @@ -2,7 +2,6 @@ from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.listtype import get_list_index from pypy.objspace.std.sliceobject import W_SliceObject -from pypy.objspace.std.tupleobject import W_TupleObject from pypy.objspace.std.seqinterface import W_SeqObject from pypy.objspace.std import slicetype @@ -14,9 +13,9 @@ from pypy.objspace.std.listtype import list_typedef as typedef def __init__(w_self, wrappeditems): - if len(wrappeditems) > 0: - elem = wrappeditems.pop() - wrappeditems.append(elem) + #if len(wrappeditems) > 0: + wrappeditems.append(None) + wrappeditems.pop() w_self.wrappeditems = wrappeditems def __repr__(w_self): Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupletype.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupletype.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupletype.py Tue Sep 9 21:58:20 2008 @@ -13,7 +13,7 @@ space.is_w(space.type(w_sequence), space.w_tuple)): return w_sequence else: - tuple_w = space.unpackiterable(w_sequence) + tuple_w = space.viewiterable(w_sequence) w_obj = space.allocate_instance(space.TupleObjectCls, w_tupletype) space.TupleObjectCls.__init__(w_obj, tuple_w) return w_obj Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Tue Sep 9 21:58:20 2008 @@ -80,7 +80,7 @@ def descr_get__mro__(space, w_type): w_type = _check(space, w_type) - return space.newtuple(w_type.mro_w) + return space.newtuple(w_type.mro_w[:]) def descr_mro(space, w_type): """Return a type's method resolution order.""" @@ -89,7 +89,7 @@ def descr_get__bases__(space, w_type): w_type = _check(space, w_type) - return space.newtuple(w_type.bases_w) + return space.newtuple(w_type.bases_w[:]) def mro_subclasses(space, w_type, temp): from pypy.objspace.std.typeobject import W_TypeObject, compute_mro From fijal at codespeak.net Wed Sep 10 00:39:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 00:39:50 +0200 (CEST) Subject: [pypy-svn] r58031 - pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser Message-ID: <20080909223950.AB4D916A01A@codespeak.net> Author: fijal Date: Wed Sep 10 00:39:49 2008 New Revision: 58031 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/tuplebuilder.py Log: Parser is insane. Rewrite couple of functions to be a lot faster. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Wed Sep 10 00:39:49 2008 @@ -429,18 +429,12 @@ genexpr_for[0].is_outmost = True builder.push(ast.GenExpr(ast.GenExprInner(expr, genexpr_for, lineno), lineno)) return - isConst = True - values = [] for item in items: - if isinstance(item, ast.Const): - values.append(item.value) - else: - isConst = False - break - if isConst: - builder.push(ast.Const(builder.space.newtuple(values), lineno)) - else: - builder.push(ast.Tuple(items, lineno)) + if not isinstance(item, ast.Const): + builder.push(ast.Tuple(items, lineno)) + return + values = [item.value for item in items] + builder.push(ast.Const(builder.space.newtuple(values), lineno)) return def build_lambdef(builder, nb): @@ -739,21 +733,16 @@ if len(atoms) <= 2: builder.push(atoms[0]) else: - names = [] - values = [] isConst = True - for index in range(0, len(atoms), 2): - item = atoms[index] - names.append(item) - if isinstance(item, ast.Const): - values.append(item) - else: + items = [atoms[index] for index in range(0, len(atoms), 2)] + for item in items: + if not isinstance(item, ast.Const): isConst = False - if isConst: - builder.push(ast.Const(builder.space.newtuple(values), atoms[0].lineno)) + break + if not isConst: + builder.push(ast.Tuple(items, atoms[0].lineno)) else: - builder.push(ast.Tuple(names, atoms[0].lineno)) - + builder.push(ast.Const(builder.space.newtuple(items), atoms[0].lineno)) def build_while_stmt(builder, nb): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/tuplebuilder.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/tuplebuilder.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/tuplebuilder.py Wed Sep 10 00:39:49 2008 @@ -17,9 +17,10 @@ def as_w_tuple(self, space, lineno=False): num, value, lineno = self.nodes[0] - content = [space.wrap(num), space.wrap(value)] if lineno: - content.append(space.wrap(lineno)) + content = [space.wrap(num), space.wrap(value), space.wrap(lineno)] + else: + content = [space.wrap(num), space.wrap(value)] return space.newtuple(content) @@ -34,8 +35,8 @@ return tuple(l) def as_w_tuple(self, space, lineno=False): - l = [space.wrap(self.num)] - l += [node.as_w_tuple(space, lineno) for node in self.nodes] + l = ([space.wrap(self.num)] + + [node.as_w_tuple(space, lineno) for node in self.nodes]) return space.newtuple(l) From fijal at codespeak.net Wed Sep 10 01:21:31 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 01:21:31 +0200 (CEST) Subject: [pypy-svn] r58032 - in pypy/branch/tuple-nonresizable-395/pypy: interpreter interpreter/astcompiler module/_file module/_pickle_support module/_rawffi module/_sre module/struct objspace/std Message-ID: <20080909232131.4168116A04B@codespeak.net> Author: fijal Date: Wed Sep 10 01:21:27 2008 New Revision: 58032 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py pypy/branch/tuple-nonresizable-395/pypy/module/_file/interp_file.py pypy/branch/tuple-nonresizable-395/pypy/module/_pickle_support/maker.py pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py pypy/branch/tuple-nonresizable-395/pypy/module/struct/interp_struct.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: another checkin in progress. add few hacks and kill some merges between tuple and list wrappeditems. still something explodes. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py Wed Sep 10 01:21:27 2008 @@ -136,9 +136,13 @@ unfiltered_kwds_w[argname] = w_arg assert not space.is_true(data_w_stararg) else: - args_w = data_args_w[:] - for w_stararg in space.unpackiterable(data_w_stararg): - args_w.append(w_stararg) + stararg_w = space.unpackiterable(data_w_stararg) + datalen = len(data_args_w) + args_w = [None] * (datalen + len(stararg_w)) + for i in range(0, datalen): + args_w[i] = data_args_w[i] + for i in range(0, len(stararg_w)): + args_w[i + datalen] = stararg_w[i] assert len(args_w) == need_cnt kwds_w = {} @@ -410,7 +414,7 @@ raise ValueError, "too many arguments (%d expected)" % argcount if self.w_stararg is not None: self.arguments_w = (self.arguments_w + - self.space.unpackiterable(self.w_stararg, + self.space.viewiterable(self.w_stararg, argcount - len(self.arguments_w))) self.w_stararg = None elif len(self.arguments_w) < argcount: @@ -534,7 +538,7 @@ assert extravarargs is not None starargs_w = extravarargs if num_args: - starargs_w.extend(args_w) + starargs_w = starargs_w + args_w elif num_args > args_left: starargs_w = args_w[args_left:] else: Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py Wed Sep 10 01:21:27 2008 @@ -231,11 +231,10 @@ return self._visitAbstractTest(node, False) def visitTuple(self, node): - consts_w = [] for subnode in node.nodes: if not isinstance(subnode, ast.Const): return node # not all constants - consts_w.append(subnode.value) + consts_w = [subnode.value for subnode in node.nodes] return ast.Const(self.space.newtuple(consts_w)) def visitFor(self, node): Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py Wed Sep 10 01:21:27 2008 @@ -656,13 +656,17 @@ plural = "s" raise UnpackValueError("need more than %d value%s to unpack" % (i, plural)) + # XXX remember to kill + items.append(None) + items.pop() return items - def viewiterable(self, w_iterable): + def viewiterable(self, w_iterable, expected_length=-1): """ More or less the same as unpackiterable, but does not return a copy. Please don't modify the result """ - return make_sure_not_resized(self.unpackiterable(w_iterable)[:]) + return make_sure_not_resized(self.unpackiterable(w_iterable, + expected_length)[:]) def unpacktuple(self, w_tuple, expected_length=-1): """Same as unpackiterable(), but only for tuples. Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_file/interp_file.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_file/interp_file.py Wed Sep 10 01:21:27 2008 @@ -419,7 +419,7 @@ result.append(space.wrap('\n')) if newlines & 4: result.append(space.wrap('\r\n')) - return space.newtuple(result) + return space.newtuple(result[:]) def descr_file_softspace(space, file): return space.wrap(file.softspace) Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_pickle_support/maker.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_pickle_support/maker.py Wed Sep 10 01:21:27 2008 @@ -88,15 +88,18 @@ create a tuple with the object and store a tuple with the positions of NULLs as first element. """ - nulls = [] - tup = [space.w_None] + tup = [None] * (len(seq_w) + 1) w = space.wrap - + num = 1 + nulls = [None for i in seq_w if i is None] + null_num = 0 for w_obj in seq_w: if w_obj is None: - nulls.append(w(len(tup)-1)) + nulls[null_num] = w(num - 1) + null_num += 1 w_obj = space.w_None - tup.append(w_obj) + tup[num] = w_obj + num += 1 tup[0] = space.newtuple(nulls) return space.newtuple(tup) Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py Wed Sep 10 01:21:27 2008 @@ -148,7 +148,7 @@ """ ffi_restype, resshape = unpack_resshape(space, w_restype) w = space.wrap - argtypes_w = space.unpackiterable(w_argtypes) + argtypes_w = space.viewiterable(w_argtypes) w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w(name), w_argtypes, w(resshape)]) try: Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Wed Sep 10 01:21:27 2008 @@ -95,11 +95,10 @@ """Creates a tuple of index pairs representing matched groups, a format that's convenient for SRE_Match.""" space = self.space - lst = [] - for value1, value2 in self.create_regs(group_count): - lst.append(space.newtuple([space.wrap(value1), - space.wrap(value2)])) - return space.newtuple(lst) + return space.newtuple([ + space.newtuple([space.wrap(value1), + space.wrap(value2)]) + for value1, value2 in self.create_regs(group_count)]) w_create_regs.unwrap_spec = ['self', int] def fget_start(space, self): Modified: pypy/branch/tuple-nonresizable-395/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/struct/interp_struct.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/struct/interp_struct.py Wed Sep 10 01:21:27 2008 @@ -33,5 +33,5 @@ fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space) - return space.newtuple(fmtiter.result_w) + return space.newtuple(fmtiter.result_w[:]) unpack.unwrap_spec = [ObjSpace, str, 'bufferstr'] Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Wed Sep 10 01:21:27 2008 @@ -29,6 +29,12 @@ def append(w_list, w_item): w_list.wrappeditems.append(w_item) + def getlength(self): + return len(self.wrappeditems) + + def getitem(self, i): + return self.wrappeditems[i] + registerimplementation(W_ListObject) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py Wed Sep 10 01:21:27 2008 @@ -9,7 +9,7 @@ """ def getlength(self): - return len(self.wrappeditems) + raise NotImplementedError def getitem(self, i): - return self.wrappeditems[i] + raise NotImplementedError Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Wed Sep 10 01:21:27 2008 @@ -26,6 +26,12 @@ def getitems(self): return self.wrappeditems + def getlength(self): + return len(self.wrappeditems) + + def getitem(self, i): + return self.wrappeditems[i] + registerimplementation(W_TupleObject) From hpk at codespeak.net Wed Sep 10 09:00:55 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 09:00:55 +0200 (CEST) Subject: [pypy-svn] r58033 - pypy/build/doc Message-ID: <20080910070055.2CA6416A1F1@codespeak.net> Author: hpk Date: Wed Sep 10 09:00:52 2008 New Revision: 58033 Added: pypy/build/doc/ pypy/build/doc/ssh_config Log: new doc directory about build/test infrastructure ssh_config file listing all pypy relevant hosts i know of Added: pypy/build/doc/ssh_config ============================================================================== --- (empty file) +++ pypy/build/doc/ssh_config Wed Sep 10 09:00:52 2008 @@ -0,0 +1,20 @@ +Host bigdog2 + Hostname 8.8.197.72 + +Host snake + HostName snake.cs.uni-duesseldorf.de + User pypy + +Host wyvern + HostName wyvern.cs.uni-duesseldorf.de + Port 922 + +Host cobra + HostName cobra.cs.uni-duesseldorf.de + Port 922 + +Host tuatara + HostName tuatara.cs.uni-duesseldorf.de + +Host t20 + Hostname t20.pypy.org From hpk at codespeak.net Wed Sep 10 12:51:43 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 12:51:43 +0200 (CEST) Subject: [pypy-svn] r58036 - pypy/build/doc/tool Message-ID: <20080910105143.11C17169FC7@codespeak.net> Author: hpk Date: Wed Sep 10 12:51:41 2008 New Revision: 58036 Added: pypy/build/doc/tool/ pypy/build/doc/tool/sysinfo.py (contents, props changed) Log: adding a small script (requires py/trunk) to get various machine info from the hosts listed in doc/ssh_config Added: pypy/build/doc/tool/sysinfo.py ============================================================================== --- (empty file) +++ pypy/build/doc/tool/sysinfo.py Wed Sep 10 12:51:41 2008 @@ -0,0 +1,116 @@ + +import py +import sys +mydir = py.magic.autopath().dirpath() +ssh_config = mydir.dirpath("ssh_config") + +class SshConfig(object): + def __init__(self, path): + self.path = path + + def parsehosts(self, ignores): + if isinstance(ignores, str): + ignores = ignores.split(",") + l = [] + rex = py.std.re.compile(r'Host\s*(\S+)') + for line in self.path.readlines(): + m = rex.match(line) + if m is not None: + sshname, = m.groups() + if sshname not in ignores: + l.append(sshname) + return l + +class RemoteInfo: + def __init__(self, gateway): + self.gw = gateway + self._cache = {} + + def exreceive(self, execstring): + if execstring not in self._cache: + channel = self.gw.remote_exec(execstring) + self._cache[execstring] = channel.receive() + return self._cache[execstring] + + def getmodattr(self, modpath): + module = modpath.split(".")[0] + return self.exreceive(""" + import %s + channel.send(%s) + """ %(module, modpath)) + + def islinux(self): + return self.getmodattr('sys.platform').find("linux") != -1 + + def getmemswap(self): + if self.islinux(): + return self.exreceive(""" + import commands, re + out = commands.getoutput("free") + mem = re.search(r"Mem:\s+(\S*)", out).group(1) + swap = re.search(r"Swap:\s+(\S*)", out).group(1) + channel.send((mem, swap)) + """) + + def getcpuinfo(self): + if self.islinux(): + return self.exreceive(""" + numcpus = 0 + model = None + for line in open("/proc/cpuinfo"): + if not line.strip(): + continue + key, value = line.split(":") + key = key.strip() + if key == "processor": + numcpus += 1 + elif key == "model name": + model = value.strip() + channel.send((numcpus, model)) + """) + +def debug(*args): + print >>sys.stderr, " ".join(map(str, args)) +def error(*args): + debug("ERROR", args[0] + ":", *args[1:]) + +def getinfo(sshname, loginfo=sys.stdout): + debug("connecting to", sshname) + try: + gw = py.execnet.SshGateway(sshname, ssh_config=ssh_config) + except IOError: + error("could not get sshagteway", sshname) + else: + ri = RemoteInfo(gw) + #print "%s info:" % sshname + prefix = sshname.upper() + " " + for attr in ( + "sys.platform", + "sys.version_info", + ): + loginfo.write("%s %s: " %(prefix, attr,)) + loginfo.flush() + value = ri.getmodattr(attr) + loginfo.write(str(value)) + loginfo.write("\n") + loginfo.flush() + memswap = ri.getmemswap() + if memswap: + mem,swap = memswap + print >>loginfo, prefix, "Memory:", mem, "Swap:", swap + cpuinfo = ri.getcpuinfo() + if cpuinfo: + numcpu, model = cpuinfo + print >>loginfo, prefix, "number of cpus:", numcpu + print >>loginfo, prefix, "cpu model", model + return ri + +if __name__ == '__main__': + if len(sys.argv) < 2: + sc = SshConfig(ssh_config) + hosts = sc.parsehosts(ignores="t20,snake") + else: + hosts = sys.argv[1:] + for host in hosts: + getinfo(host) + From hpk at codespeak.net Wed Sep 10 13:29:07 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 13:29:07 +0200 (CEST) Subject: [pypy-svn] r58038 - pypy/build/doc/tool Message-ID: <20080910112907.B8B1716A2C2@codespeak.net> Author: hpk Date: Wed Sep 10 13:29:05 2008 New Revision: 58038 Removed: pypy/build/doc/tool/ Log: removing the tool here, it is now in svn/py/trunk/contrib From witulski at codespeak.net Wed Sep 10 15:31:28 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Wed, 10 Sep 2008 15:31:28 +0200 (CEST) Subject: [pypy-svn] r58039 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080910133128.D0385169EED@codespeak.net> Author: witulski Date: Wed Sep 10 15:31:25 2008 New Revision: 58039 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: Added a new test which is broken some work on goto (but still dont works) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Wed Sep 10 15:31:25 2008 @@ -66,7 +66,7 @@ self.write_modRM_byte(3, modrm2, modrm1) self.write(chr(arg2.value)) elif isinstance(arg2,Register64): - rexB, modrm2 = self.get_register_bits(arg2.reg) + rexB, modrm2 = self.get_register_bits(arg2.reg) # FIXME: exchange the two arguments (rexB/rexR) self.write_rex_byte(rexW, rexB, rexX, rexR) self.write(opcode) @@ -144,16 +144,16 @@ # The opcodes differs depending on the operands # Params: - # W, R, X, B, Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode + # W (64bit Operands), R, X, B, Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode - # FIXME: rexX,rexB are set + # FIXME: rexB is set _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 2) _ADD_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x00", 3, None, None) # FIXME: rexB is set _CMP_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 1, "\x81", 3, None, 7) _CMP_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x39", 3, None, None) - # FIXME: rex B is set + # FIXME: rexB is set _CMP_8REG_IMM8 = make_two_operand_instr( 0, 0, 0, 0, "\x82", 3, None, 7) _DEC_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 1) @@ -202,6 +202,12 @@ #self.write("\xE9") #self.writeImm32(displ) + # op1 is and 32bit displ + def JNE(self,op1): + self.write("\x0F") + self.write("\x85") + self.writeImm32(op1) + def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) method(op1) @@ -240,20 +246,24 @@ def get_register_bits_8Bit(self, register): return REGISTER_MAP_8BIT[register] + # TODO: sign extention? # Parse the integervalue to an charakter # and write it def writeImm32(self, imm32): x = hex(imm32) + if x[0]=='-': + x = self.cast_to_neg_hex(x) # parse to string and cut "0x" off # fill with zeros if to short y = "0"*(10-len(x))+x[2:len(x)] - assert len(y) == 8 + assert len(y) == 8 self.write(chr(int(y[6:8],16))) self.write(chr(int(y[4:6],16))) self.write(chr(int(y[2:4],16))) self.write(chr(int(y[0:2],16))) + # TODO: sign extention? # Parse the integervalue to an charakter # and write it def writeImm64(self, imm64): @@ -281,3 +291,7 @@ byte = mod << 6 | (reg << 3) | rm self.write(chr(byte)) + # TODO: write me + def cast_to_neg_hex(self,a_hex): + return a_hex + Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Wed Sep 10 15:31:25 2008 @@ -103,9 +103,12 @@ op_int_pop = make_one_argument_method("POP") op_int_sub = make_two_argument_method("SUB") + #FIXME: can only jump 32bit def jump_if_true(self, gv_condition, args_for_jump_gv): targetbuilder = Builder() self.mc.CMP(gv_condition, Immediate32(0)) + displ = self.calc_32bit_displacement(self.mc.tell(),targetbuilder.mc.tell()) + self.mc.JNE(displ) #targetbuilder.come_from(self.mc, 'JNE') return targetbuilder @@ -149,12 +152,17 @@ #TODO: Implementation def enter_next_block(self, args_gv): - print "WriteMe: enter_next_block" - return Label(self.mc.tell(), [], 0) + #print "WriteMe: enter_next_block" + L = Label(self.mc.tell(), [], 0) + #print "DEBUG2:",L.startaddr + return L def _close(self): pass - + + def calc_32bit_displacement(self, curr_addr, want_jump_to): + return want_jump_to-curr_addr + class RX86_64GenOp(model.AbstractRGenOp): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Wed Sep 10 15:31:25 2008 @@ -125,6 +125,8 @@ assert res == 1 res = fnptr(4,4) assert res == 0 + res = fnptr(4,0) + assert res == 1 # def test_push_and_pop(self): # rgenop = self.RGenOp() Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Wed Sep 10 15:31:25 2008 @@ -38,16 +38,32 @@ genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) - ten = fp(4, 6) + ten = fp(4, 6) # 4+6= 10 assert ten == 10 print ten + def test_add_neg(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "place" + genv1 = inputargs_gv[1] + genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + ten = fp(-4, -6) + assert ten == -10 + print ten + two = fp(-4, 6) + assert two == 2 + print two + two = fp(4, -6) + assert two == -2 + print two + def test_add_imm32(self): builder, fp, inputargs_gv, token = make_testbuilder(1) genv0 = inputargs_gv[0] #the first argument "place" genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) - num = fp(1111) + num = fp(1111) # 1111+1000 = 2111 assert num == 2111 print num @@ -55,7 +71,7 @@ builder, fp, inputargs_gv, token = make_testbuilder(1) builder.finish_and_return(token, inputargs_gv[0]) print repr("".join(builder.mc._all)) - four = fp(4) + four = fp(4) # return 4 assert four == 4 print four @@ -65,7 +81,7 @@ genv1 = inputargs_gv[1] genv_result = builder.genop2("int_sub", genv0, genv1) #creates the subtraction and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) - four = fp(10, 6) + four = fp(10, 6) # 10 - 6 = 4 assert four == 4 print four @@ -74,7 +90,7 @@ genv0 = inputargs_gv[0] #the first argument "place" genv_result = builder.genop2("int_sub", genv0, rgenop.genconst(2)) #creates the subtraction and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) - eight = fp(10) + eight = fp(10) # 10-2 = 8 assert eight == 8 print eight \ No newline at end of file From hpk at codespeak.net Wed Sep 10 17:42:54 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 17:42:54 +0200 (CEST) Subject: [pypy-svn] r58040 - pypy/build/doc Message-ID: <20080910154254.EF8C916A2CE@codespeak.net> Author: hpk Date: Wed Sep 10 17:42:50 2008 New Revision: 58040 Added: pypy/build/doc/benchmark_memory.txt (contents, props changed) Log: draft doc about benchmarking memory usage Added: pypy/build/doc/benchmark_memory.txt ============================================================================== --- (empty file) +++ pypy/build/doc/benchmark_memory.txt Wed Sep 10 17:42:50 2008 @@ -0,0 +1,31 @@ +Considerations regarding Memory Benchmarks +=============================================== + +* measure min/max/avg process-private RAM usage of a Benchmark + +* measure (min/max/avg) collection time of an APP/Benchmark + +* measure CPU usage of benchmark + +* Microbenchmarks probably with checkpoints in order to have + reproducible RAM usage measurements and to analyse + specific object types (strings/dicts/lists/old-style/new-style + instances ...) and scenarios like reading lines in large file. + (reuse some of the speed benchmarks?) + +* Scenario/App Benchmarks: + * allocate objects all the time but have mostly constant + number of life objects + * have small number of objects objects, create many, + free them all, repeat + + * use several sets of "small" and "many" + +* real APPS: + * some sympy computation/test? + * an app involving c-extensions? + * ask around on pypy-dev for example apps + +* measurements against CPython, pypy-c with various options + + From fijal at codespeak.net Wed Sep 10 18:36:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 18:36:45 +0200 (CEST) Subject: [pypy-svn] r58041 - pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler Message-ID: <20080910163645.D3C3F16A31D@codespeak.net> Author: fijal Date: Wed Sep 10 18:36:40 2008 New Revision: 58041 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/ast.py Log: Mention wrong algorithm Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/ast.py Wed Sep 10 18:36:40 2008 @@ -5923,6 +5923,7 @@ child.accept(self) def _mutate_list(self, lst): + # XXX O(n^2) i = 0 while i < len(lst): item = lst[i].mutate(self) From fijal at codespeak.net Wed Sep 10 18:51:41 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 18:51:41 +0200 (CEST) Subject: [pypy-svn] r58042 - pypy/branch/tuple-nonresizable-395/pypy/interpreter Message-ID: <20080910165141.4AB3A16A293@codespeak.net> Author: fijal Date: Wed Sep 10 18:51:39 2008 New Revision: 58042 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Log: Make sure pycode slots and arguments are never modified. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/argument.py Wed Sep 10 18:51:39 2008 @@ -271,9 +271,9 @@ if has_vararg: if upfront > co_argcount: assert extravarargs is not None - stararg_w = extravarargs + stararg_w = extravarargs + [None] * self.nargs for i in range(self.nargs): - stararg_w.append(self.frame.peekvalue(self.nargs - 1 - i)) + stararg_w[i + len(extravarargs)] = self.frame.peekvalue(self.nargs - 1 - i) else: args_left = co_argcount - upfront stararg_w = [None] * (avail - co_argcount) Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Wed Sep 10 18:51:39 2008 @@ -11,15 +11,12 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rlib.rarithmetic import intmask +from pypy.rlib.debug import make_sure_not_resized # helper def unpack_str_tuple(space,w_str_tuple): - els = [] - for w_el in space.unpackiterable(w_str_tuple): - els.append(space.str_w(w_el)) - return els - + return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] # code object contants, for co_flags below CO_OPTIMIZED = 0x0001 @@ -66,7 +63,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = consts + self.co_consts_w = make_sure_not_resized(consts) self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -122,12 +119,13 @@ This method is called by our compile builtin function. """ assert isinstance(code, types.CodeType) - newconsts_w = [] + newconsts_w = [None] * len(code.co_consts) + num = 0 for const in code.co_consts: if isinstance(const, types.CodeType): # from stable compiler const = PyCode._from_code(space, const, hidden_applevel=hidden_applevel) - - newconsts_w.append(space.wrap(const)) + newconsts_w[num] = space.wrap(const) + num += 1 # stick the underlying CPython magic value, if the code object # comes from there return PyCode(space, code.co_argcount, @@ -213,12 +211,14 @@ def _to_code(self): """For debugging only.""" - consts = [] + consts = [None] * len(self.co_consts_w) + num = 0 for w in self.co_consts_w: if isinstance(w, PyCode): - consts.append(w._to_code()) + consts[num] = w._to_code() else: - consts.append(self.space.unwrap(w)) + consts[num] = self.space.unwrap(w) + num += 1 return new.code( self.co_argcount, self.co_nlocals, self.co_stacksize, @@ -240,10 +240,10 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w[:]) + return space.newtuple(self.co_consts_w) def fget_co_names(space, self): - return space.newtuple(self.co_names_w[:]) + return space.newtuple(self.co_names_w) def fget_co_varnames(space, self): return space.newtuple([space.wrap(name) for name in self.co_varnames]) @@ -321,8 +321,11 @@ space.wrap("code: argcount must not be negative")) if nlocals < 0: raise OperationError(space.w_ValueError, - space.wrap("code: nlocals must not be negative")) - consts_w = space.unpacktuple(w_constants) + space.wrap("code: nlocals must not be negative")) + if not space.is_true(space.isinstance(w_constants, space.w_tuple)): + raise OperationError(space.w_TypeError, + space.wrap("Expected tuple for constants")) + consts_w = space.viewiterable(w_constants) names = unpack_str_tuple(space, w_names) varnames = unpack_str_tuple(space, w_varnames) if w_freevars is not None: @@ -351,8 +354,8 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w[:]), - space.newtuple(self.co_names_w[:]), + space.newtuple(self.co_consts_w), + space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), From fijal at codespeak.net Wed Sep 10 18:56:04 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 18:56:04 +0200 (CEST) Subject: [pypy-svn] r58043 - in pypy/branch/tuple-nonresizable-395/pypy: module/rctime objspace/std Message-ID: <20080910165604.15E6D16A2EF@codespeak.net> Author: fijal Date: Wed Sep 10 18:56:01 2008 New Revision: 58043 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/rctime/interp_time.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Log: Few other sources of merge between tuple and list wrappeditems Modified: pypy/branch/tuple-nonresizable-395/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/rctime/interp_time.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/rctime/interp_time.py Wed Sep 10 18:56:01 2008 @@ -166,17 +166,16 @@ return rffi.r_time_t(seconds) def _tm_to_tuple(space, t): - time_tuple = [] - - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900)) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1)) # want january == 1 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_mday')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_hour')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_min')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_sec')) ) - time_tuple.append(space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7)) # want monday == 0 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1)) # want january, 1 == 1 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_isdst')) ) + time_tuple = [ + space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900), + space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1 + space.wrap(rffi.getintfield(t, 'c_tm_mday')), + space.wrap(rffi.getintfield(t, 'c_tm_hour')), + space.wrap(rffi.getintfield(t, 'c_tm_min')), + space.wrap(rffi.getintfield(t, 'c_tm_sec')), + space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0 + space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1 + space.wrap(rffi.getintfield(t, 'c_tm_isdst'))] w_struct_time = _get_module_object(space, 'struct_time') w_time_tuple = space.newtuple(time_tuple) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Wed Sep 10 18:56:01 2008 @@ -379,7 +379,7 @@ m.put_int(x.co_flags) m.atom_str(TYPE_STRING, x.co_code) m.start(TYPE_TUPLE) - m.put_list_w(x.co_consts_w, len(x.co_consts_w)) + m.put_list_w(x.co_consts_w[:], len(x.co_consts_w)) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w]) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars) @@ -422,7 +422,8 @@ flags = u.get_int() code = unmarshal_str(u) u.start(TYPE_TUPLE) - consts_w = u.get_list_w() + consts_w = u.get_tuple_w() + # copy in order not to merge it with anything else names = unmarshal_strlist(u, TYPE_TUPLE) varnames = unmarshal_strlist(u, TYPE_TUPLE) freevars = unmarshal_strlist(u, TYPE_TUPLE) From hpk at codespeak.net Wed Sep 10 18:56:08 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 18:56:08 +0200 (CEST) Subject: [pypy-svn] r58044 - in pypy: branch/cross-compilation/pypy/doc/discussion build/doc Message-ID: <20080910165608.684E016A2EF@codespeak.net> Author: hpk Date: Wed Sep 10 18:56:06 2008 New Revision: 58044 Removed: pypy/branch/cross-compilation/pypy/doc/discussion/gc-memory-notes.txt Modified: pypy/build/doc/benchmark_memory.txt Log: merging fijal notes into new benchmark_memory doc Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Wed Sep 10 18:56:06 2008 @@ -1,4 +1,7 @@ -Considerations regarding Memory Benchmarks + +XXX draft doc + +what we want to measure =============================================== * measure min/max/avg process-private RAM usage of a Benchmark @@ -7,6 +10,11 @@ * measure CPU usage of benchmark +* measure one and multiple interpreter instances at a given time + +Benchmark targets +-------------------- + * Microbenchmarks probably with checkpoints in order to have reproducible RAM usage measurements and to analyse specific object types (strings/dicts/lists/old-style/new-style @@ -14,11 +22,10 @@ (reuse some of the speed benchmarks?) * Scenario/App Benchmarks: - * allocate objects all the time but have mostly constant - number of life objects - * have small number of objects objects, create many, + * allocate objects all the time, total number of objects stay constant + * allocate objects in bursts and immediately throw them away + * constant small number live objects, burst-allocate many, free them all, repeat - * use several sets of "small" and "many" * real APPS: @@ -26,6 +33,38 @@ * an app involving c-extensions? * ask around on pypy-dev for example apps -* measurements against CPython, pypy-c with various options +Interpreters +--------------- + +* CPython 2.5 +* pypy-c --opt=3 +* pypy-c various options/GCs + + +notes about memory-saving GCs +=============================== + +XXX go into more detail, discuss which pypy GCs we want to try + +* squeak-like mark-compact collector + +* deferred refcounting strategy + +* deferred refcounting with a nursery + + +Implementation +=========================== + +measuring memory on Linux +----------------------------- + +Unless we can use some external tool we +can parse info located in /proc/PID/smaps. +This provides detailed info, see e.g. here +for some explanations: + http://bmaurer.blogspot.com/2006/03/memory-usage-with-smaps.html +For us mostly process private memory, RSS (Resident Set Size) +and shared memory will be interesting to measure. From fijal at codespeak.net Wed Sep 10 19:04:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 19:04:45 +0200 (CEST) Subject: [pypy-svn] r58045 - pypy/build/doc Message-ID: <20080910170445.ADD38169EF6@codespeak.net> Author: fijal Date: Wed Sep 10 19:04:42 2008 New Revision: 58045 Modified: pypy/build/doc/benchmark_memory.txt Log: add XXX Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Wed Sep 10 19:04:42 2008 @@ -37,7 +37,10 @@ --------------- * CPython 2.5 -* pypy-c --opt=3 +* pypy-c --opt=3 +* pypy-c --opt=mem +XXX instead of trying various things, we need to make opt=mem good. opt=3 is + good for comparison * pypy-c various options/GCs From hpk at codespeak.net Wed Sep 10 19:18:49 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 10 Sep 2008 19:18:49 +0200 (CEST) Subject: [pypy-svn] r58046 - pypy/build/doc Message-ID: <20080910171849.A1DAA16A316@codespeak.net> Author: hpk Date: Wed Sep 10 19:18:48 2008 New Revision: 58046 Modified: pypy/build/doc/benchmark_memory.txt Log: revive some remarks from fijal Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Wed Sep 10 19:18:48 2008 @@ -63,11 +63,18 @@ ----------------------------- Unless we can use some external tool we -can parse info located in /proc/PID/smaps. -This provides detailed info, see e.g. here -for some explanations: +can parse info located in /proc/PID/smaps +(beginning from linux 2.6.16) and contains +Size, Rss, Shared and Private. - http://bmaurer.blogspot.com/2006/03/memory-usage-with-smaps.html +Note that all addresses are virtual, which means +that having the same address in two processes +doesn't mean it's the same memory. -For us mostly process private memory, RSS (Resident Set Size) -and shared memory will be interesting to measure. +Explanation: +Size: total (virtual) size of memory, possibly irrelevant +RSS (Resident Set Size): indicates physically used RAM +Shared: memory shared with other processes. Note that this is simply a counter +how many processes reference it. Memory can move private -> shared in case +some other process will load the same library or so. +Private: private memory owned by a process. From fijal at codespeak.net Wed Sep 10 19:23:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 19:23:43 +0200 (CEST) Subject: [pypy-svn] r58047 - in pypy/branch/tuple-nonresizable-395/pypy/interpreter: astcompiler pyparser Message-ID: <20080910172343.9058016A292@codespeak.net> Author: fijal Date: Wed Sep 10 19:23:42 2008 New Revision: 58047 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Log: Make annotator happier. This also means that we copy stuff without any good reason. To fix this we would probably need to rewrite pyparser and astcompier a fair bit :( Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py Wed Sep 10 19:23:42 2008 @@ -234,7 +234,9 @@ for subnode in node.nodes: if not isinstance(subnode, ast.Const): return node # not all constants - consts_w = [subnode.value for subnode in node.nodes] + # this isinstance is only to make annotator happier + consts_w = [subnode.value for subnode in node.nodes + if isinstance(subnode, ast.Const)] return ast.Const(self.space.newtuple(consts_w)) def visitFor(self, node): Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Wed Sep 10 19:23:42 2008 @@ -416,9 +416,9 @@ if l == 1: builder.push(atoms[0]) return - items = [] token = atoms[1] if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COMMA']: + items = [] for i in range(0, l, 2): # this is atoms not 1 items.append(atoms[i]) else: @@ -433,7 +433,8 @@ if not isinstance(item, ast.Const): builder.push(ast.Tuple(items, lineno)) return - values = [item.value for item in items] + # isinstance as a hint for annotator + values = [item.value for item in items if isinstance(item, ast.Const)] builder.push(ast.Const(builder.space.newtuple(values), lineno)) return @@ -740,9 +741,10 @@ isConst = False break if not isConst: - builder.push(ast.Tuple(items, atoms[0].lineno)) + builder.push(ast.Tuple([i for i in items if isinstance(i, ast.Node)], atoms[0].lineno)) else: - builder.push(ast.Const(builder.space.newtuple(items), atoms[0].lineno)) + builder.push(ast.Const(builder.space.newtuple( + [i for i in items if isinstance(i, ast.Const)]), atoms[0].lineno)) def build_while_stmt(builder, nb): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" From fijal at codespeak.net Wed Sep 10 19:24:18 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 19:24:18 +0200 (CEST) Subject: [pypy-svn] r58048 - pypy/branch/tuple-nonresizable-395/pypy/interpreter Message-ID: <20080910172418.269C616A292@codespeak.net> Author: fijal Date: Wed Sep 10 19:24:16 2008 New Revision: 58048 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyframe.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py Log: Make two versions of frame.popvalues, one for tuple and one for list. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyframe.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyframe.py Wed Sep 10 19:24:16 2008 @@ -152,15 +152,22 @@ dic_w[key] = w_value return dic_w - def popvalues(self, n): - values_w = [None] * n - while True: - n -= 1 - if n < 0: - break - hint(n, concrete=True) - values_w[n] = self.popvalue() - return values_w + # we need two popvalues that return different data types: + # one in case we want list another in case of tuple + def _new_popvalues(): + def popvalues(self, n): + values_w = [None] * n + while True: + n -= 1 + if n < 0: + break + hint(n, concrete=True) + values_w[n] = self.popvalue() + return values_w + return popvalues + popvalues = _new_popvalues() + popvalues_mutable = _new_popvalues() + del _new_popvalues def peekvalues(self, n): values_w = [None] * n Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py Wed Sep 10 19:24:16 2008 @@ -678,7 +678,7 @@ f.pushvalue(w_tuple) def BUILD_LIST(f, itemcount, *ignored): - items = f.popvalues(itemcount) + items = f.popvalues_mutable(itemcount) w_list = f.space.newlist(items) f.pushvalue(w_list) @@ -904,7 +904,7 @@ def MAKE_FUNCTION(f, numdefaults, *ignored): w_codeobj = f.popvalue() codeobj = f.space.interp_w(PyCode, w_codeobj) - defaultarguments = f.popvalues(numdefaults) + defaultarguments = f.popvalues_mutable(numdefaults) fn = function.Function(f.space, codeobj, f.w_globals, defaultarguments) f.pushvalue(f.space.wrap(fn)) From fijal at codespeak.net Wed Sep 10 19:26:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 19:26:47 +0200 (CEST) Subject: [pypy-svn] r58049 - in pypy/branch/tuple-nonresizable-395/pypy: . interpreter objspace/std Message-ID: <20080910172647.3590616A2A4@codespeak.net> Author: fijal Date: Wed Sep 10 19:26:46 2008 New Revision: 58049 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Log: Kill obscure hacks. Update TODO Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Wed Sep 10 19:26:46 2008 @@ -1,4 +1,3 @@ -* make sure to kill W_ListObject.__init__ hack * avoid duplication in FastSeqIter, if possible * add objspace.unpackiterable shortcuts Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py Wed Sep 10 19:26:46 2008 @@ -656,9 +656,6 @@ plural = "s" raise UnpackValueError("need more than %d value%s to unpack" % (i, plural)) - # XXX remember to kill - items.append(None) - items.pop() return items def viewiterable(self, w_iterable, expected_length=-1): Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Wed Sep 10 19:26:46 2008 @@ -13,9 +13,6 @@ from pypy.objspace.std.listtype import list_typedef as typedef def __init__(w_self, wrappeditems): - #if len(wrappeditems) > 0: - wrappeditems.append(None) - wrappeditems.pop() w_self.wrappeditems = wrappeditems def __repr__(w_self): From fijal at codespeak.net Wed Sep 10 19:31:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 10 Sep 2008 19:31:45 +0200 (CEST) Subject: [pypy-svn] r58050 - pypy/build/doc Message-ID: <20080910173145.F073916A2A4@codespeak.net> Author: fijal Date: Wed Sep 10 19:31:44 2008 New Revision: 58050 Modified: pypy/build/doc/benchmark_memory.txt Log: add possible measurment scenarios Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Wed Sep 10 19:31:44 2008 @@ -78,3 +78,16 @@ how many processes reference it. Memory can move private -> shared in case some other process will load the same library or so. Private: private memory owned by a process. + +Probable measurment scenarios: + +1. Measure private memory owned by process when no other interpreter's + process exist. + +2. The same in case there is other one. + +3. Measure amount of private memory owned by process that was forked from + other interpreter process. + +4. Measure total RAM usage/RSS size in case process is running vs + total RAM usage when process is not running. From mwh at codespeak.net Thu Sep 11 08:58:16 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 11 Sep 2008 08:58:16 +0200 (CEST) Subject: [pypy-svn] r58052 - pypy/extradoc/talk/osdc2008 Message-ID: <20080911065816.78BC216A04E@codespeak.net> Author: mwh Date: Thu Sep 11 08:58:13 2008 New Revision: 58052 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: some status Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Thu Sep 11 08:58:13 2008 @@ -100,29 +100,38 @@ The code that does the analysis -- now revealed in its true colours to basically be a compiler for this restricted subset of Python -- is -generally referred to as the 'translator. It is unlike most compilers -takes as input live Python objects (as opposed to source code). It -abstractly interprets the bytecode of functions to produce flow -graphs. Further layers of abstract interpretation perform more -analysis and gradually reduce the level of abstraction before finally -C or other source code is generated +generally referred to as the 'translator'. It is unlike most +compilers in that takes as input live Python objects (as opposed to +source code). It abstractly interprets the bytecode of functions to +produce flow graphs. Further layers of abstract interpretation +perform more analysis and gradually reduce the level of abstraction +before finally C or other source code is generated The *LxOxP* problem ------------------- We'd written this compiler framework, with only one expected -non-trivial input (our Python interpreter). We realized that it would -be suitable for implementations of other dynamically-typed programming -languages. Now have implementations of Prolog, JavaScript and Scheme -(to varying extents) +non-trivial input (our Python interpreter). We eventually realized +that it would be suitable for implementations of other +dynamically-typed programming languages. Now have implementations of +Prolog, JavaScript and Scheme (to varying extents). This leads to one of PyPy's meta-goals, ameliorating the so-called LxOxP problem: given * L dynamic languages + + * for example Python, Scheme, Ruby... + * O target platforms + + * for example C/POSIX, CLI, JVM... + * P implementation decisions + * for example include a JIT, a moving garbage collector, + stackless-style features... + we don't want to have to write LxOxP different interpreters by hand. PyPy aims to reduce this to an L+O+P problem: @@ -135,10 +144,40 @@ Status ------ +PyPy when translated with the C backend (often called pypy-c) and most +optimizations enabled is currently a reasonably fast and extremely +conformant implementation of Python 2.4.4. + +Reasonably fast: depending on the code being run, pypy-c ranges from +being 20% faster to about twice as slow as CPython (the +still-experimental Just-In-Time compiler runs some (admittedly +carefully chosen) programs 60 times faster). + +Extremely conformant: pypy-c runs Django 1.0, Pylons, Twisted and +Nevow unmodified (thanks to Google's Open Source office for funding +work on this). There are two main sources of incompatibilities: + + - extension modules. PyPy supports a fair selection of the standard + extension modules (socket, struct, zlib, ctypes...) but hardly any + third party modules. + + - finalizer semantics, for example assuming that immediately after + executing this code:: + + >>> open('file.txt', 'w').write('blah') + + that the file object is already closed. + +The former is usually a much more significant problem, of course. + +Compatibility with Python 2.5 is almost complete in a branch. 2.6 +shouldn't be too hard. No Python 3 yet :) -An Example Translation ----------------------- +The currently supported backends target C/POSIX (like CPython), CLI +(like IronPython) and the JVM (like Jython). +There are incomplete backends for LLVM (the Low Level Virtual +Machine), Common Lisp, Squeak and JavaScript. The Future ---------- From mwh at codespeak.net Thu Sep 11 09:17:45 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 11 Sep 2008 09:17:45 +0200 (CEST) Subject: [pypy-svn] r58053 - pypy/extradoc/talk/osdc2008 Message-ID: <20080911071745.614C016A2BC@codespeak.net> Author: mwh Date: Thu Sep 11 09:17:43 2008 New Revision: 58053 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: i think this finishes the paper, content wise. needs much bulking out and clarification though. comments welcome :) Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Thu Sep 11 09:17:43 2008 @@ -179,6 +179,42 @@ There are incomplete backends for LLVM (the Low Level Virtual Machine), Common Lisp, Squeak and JavaScript. -The Future ----------- +About The Project +----------------- +PyPy started as a classic open-source, spare-time project, was funded +for a while, and is currently mostly a spare-time project again. + +Right from the start, however, the project was driven along by +*sprints* -- focused week long coding sessions. During the funding +period we had sprints about every 6 weeks, but now they're happening +about 2-3 times a year. + +Future +------ + +For all that we've acheived so far, I think that the bulk of the work +so far has being laying the groundwork for the really fun stuff. + +The area with the most exciting potential is the JIT. PyPy has +already extended the state of the art in automatically generating a +JIT compiler for an implementation of a dynamic language. + +Down at the more nuts and bolts implementation level, something I'm +interested in myself is stealing ideas -- and maybe even code -- from +garbage collectors developed by the Jikes RVM project. + +Something we'd really like to see are implementations of other dynamic +languages -- Ruby being an obvious example :) -- which would, when the +JIT magic is more advanced, get a Just in Time compiler almost for +free. + +Join the fun! +------------- + +Read documentation: + + http://codespeak.net/pypy/ + +Come hang out in the #pypy IRC channel on freenode, subscribe to and +post to the pypy-dev mailing list. From pedronis at codespeak.net Thu Sep 11 09:41:37 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 11 Sep 2008 09:41:37 +0200 (CEST) Subject: [pypy-svn] r58054 - pypy/branch/pypy-pytrunk/pypy/tool/test Message-ID: <20080911074137.BA42D16A2D0@codespeak.net> Author: pedronis Date: Thu Sep 11 09:41:35 2008 New Revision: 58054 Modified: pypy/branch/pypy-pytrunk/pypy/tool/test/test_pytestsupport.py Log: (iko, pedronis) make pytestsupport tests pass again, notice that pytestsupport itself is stil using deprecated interfaces though all the tool dir tests pass at least on 2.4 Modified: pypy/branch/pypy-pytrunk/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/test/test_pytestsupport.py Thu Sep 11 09:41:35 2008 @@ -117,7 +117,7 @@ tdir = udir.ensure("t", dir=True) dest = tdir.join("test_expect.py") dest.write(source) - col = conftest.Module(dest) + col = conftest.Module(dest, config="dummy") result = col.run() assert len(result) == 1 @@ -138,7 +138,7 @@ dest = udir.join("test_expect_safefilename.py") dest.write(source) from pypy import conftest - col = conftest.Module(dest) + col = conftest.Module(dest, config="dummy") methcol = col.join('ExpectTestOne').join('()').join('test_one') name = 'test_expect_safefilename_ExpectTestOne_paren_test_one.py' assert methcol.safe_filename() == name From cfbolz at codespeak.net Thu Sep 11 09:51:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 11 Sep 2008 09:51:09 +0200 (CEST) Subject: [pypy-svn] r58055 - pypy/extradoc/talk/osdc2008 Message-ID: <20080911075109.1541416A1E5@codespeak.net> Author: cfbolz Date: Thu Sep 11 09:51:07 2008 New Revision: 58055 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: some fixes, some XXXs Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Thu Sep 11 09:51:07 2008 @@ -92,7 +92,7 @@ XXX Get copy of that image in here somehow. We chose to specify the Python language by writing an implementation -if Python in a restricted subset of Python that is amenable to +of Python in a restricted subset of Python that is amenable to analysis. This let us write unit tests for parts of our specification/implementation before we had the whole thing done, and also let us test the whole specification/implementation before the code @@ -107,6 +107,10 @@ perform more analysis and gradually reduce the level of abstraction before finally C or other source code is generated +XXX (cfbolz) I found this section a bit confusing, giving that it's the +technical intro. It's all a bit clunky, and I think that the fact that we are +using abstract interpretation is mostly a detail. + The *LxOxP* problem ------------------- @@ -114,7 +118,8 @@ non-trivial input (our Python interpreter). We eventually realized that it would be suitable for implementations of other dynamically-typed programming languages. Now have implementations of -Prolog, JavaScript and Scheme (to varying extents). +Prolog, JavaScript, Smalltalk and Scheme (to varying extents). We also have a +mostly working Gameboy emulator. This leads to one of PyPy's meta-goals, ameliorating the so-called LxOxP problem: given @@ -147,6 +152,7 @@ PyPy when translated with the C backend (often called pypy-c) and most optimizations enabled is currently a reasonably fast and extremely conformant implementation of Python 2.4.4. +XXX is conformant a word? Reasonably fast: depending on the code being run, pypy-c ranges from being 20% faster to about twice as slow as CPython (the @@ -193,7 +199,7 @@ Future ------ -For all that we've acheived so far, I think that the bulk of the work +For all that we've achieved so far, I think that the bulk of the work so far has being laying the groundwork for the really fun stuff. The area with the most exciting potential is the JIT. PyPy has From pedronis at codespeak.net Thu Sep 11 10:29:01 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 11 Sep 2008 10:29:01 +0200 (CEST) Subject: [pypy-svn] r58056 - pypy/branch/pypy-pytrunk/pypy/interpreter/test Message-ID: <20080911082901.230F216A27A@codespeak.net> Author: pedronis Date: Thu Sep 11 10:28:59 2008 New Revision: 58056 Modified: pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_code.py Log: (iko, pedronis) fix the test by ignoring obscure details of py.lib reprs Modified: pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_code.py Thu Sep 11 10:28:59 2008 @@ -8,9 +8,14 @@ space = gettestobjspace() cls.space = space if py.test.config.option.runappdirect: - cls.w_file = space.wrap(__file__[:-1]) + filename = __file__ else: - cls.w_file = space.wrap("None<%s" % gateway.__file__[:-1]) + filename = gateway.__file__ + + if filename[-3:] != '.py': + filename = filename[:-1] + + cls.w_file = space.wrap(filename) def test_attributes(self): def f(): pass From hpk at codespeak.net Thu Sep 11 10:32:06 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 10:32:06 +0200 (CEST) Subject: [pypy-svn] r58057 - pypy/build/doc Message-ID: <20080911083206.00D2C16A29C@codespeak.net> Author: hpk Date: Thu Sep 11 10:32:06 2008 New Revision: 58057 Modified: pypy/build/doc/benchmark_memory.txt Log: adding much more info, explanations, links, handling one XXX Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 10:32:06 2008 @@ -1,5 +1,5 @@ -XXX draft doc +XXX draft doc, needs discussion, review what we want to measure =============================================== @@ -10,7 +10,8 @@ * measure CPU usage of benchmark -* measure one and multiple interpreter instances at a given time +* measure for 1/2/N interpreters running at the same time + and maybe also for forked processes. Benchmark targets -------------------- @@ -37,12 +38,11 @@ --------------- * CPython 2.5 -* pypy-c --opt=3 * pypy-c --opt=mem -XXX instead of trying various things, we need to make opt=mem good. opt=3 is - good for comparison -* pypy-c various options/GCs +* pypy-c --opt=3 (for comparison purposes) +We want to select and optimize good underlying settings for +PyPy's choices regarding "--opt=mem". notes about memory-saving GCs =============================== @@ -59,35 +59,108 @@ Implementation =========================== -measuring memory on Linux ------------------------------ +Understanding linux /proc/pid/smaps info +------------------------------------------- -Unless we can use some external tool we -can parse info located in /proc/PID/smaps -(beginning from linux 2.6.16) and contains -Size, Rss, Shared and Private. - -Note that all addresses are virtual, which means -that having the same address in two processes -doesn't mean it's the same memory. - -Explanation: -Size: total (virtual) size of memory, possibly irrelevant -RSS (Resident Set Size): indicates physically used RAM -Shared: memory shared with other processes. Note that this is simply a counter -how many processes reference it. Memory can move private -> shared in case -some other process will load the same library or so. -Private: private memory owned by a process. +The most detailed info apparently is provided +by /proc/PID/smaps, starting from linux 2.6.14. +Here is an example output of running +"python2.5" on a linux 2.6.24 ubuntu machine:: + +XXX please review, correct, complete so we get to a shared good understanding + +08048000-08140000 r-xp 00000000 08:01 89921 /usr/bin/python2.5 +Size: 992 kB +Rss: 768 kB +Shared_Clean: 764 kB +Shared_Dirty: 0 kB +Private_Clean: 4 kB +Private_Dirty: 0 kB +Referenced: 768 kB + +The first line indicates that the /usr/bin/python2.5 file is +mapped as Read/eXecute into the given process and is +seen at address 08048000 by the process. +Virtual memory size is 992kB, of which 768 kB are actually +mapped into RAM (Rss = Resident Set Size) - the rest of the +file has not been accessed yet and is thus not mapped. +764 kB are shared (Shared_Clean) - so if there are other python processes they +will get their mapping but no additional RAM will be +used for these 764 kBs. "clean" means that these pages can easily +get swapped out by dropping them and - upon access - retrieving them +from the file. XXX what does Private_Clean / 4kB mean exactly here? + +Let's look at a mapping that is more indicative +of the per-process "incremental" RAM usage:: + +08165000-081e0000 rw-p 08165000 00:00 0 [heap] +Size: 492 kB +Rss: 452 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 452 kB +Referenced: 452 kB + +Here we have a readwrite anonymous mapping, objects +allocated on the heap. It uses 492 kB virtual address +space of which 452 kB are actually mapped in physical +RAM. Dirty means that these pages have been modified. +"Dirty" or "clean" is important info for Swapping +but not too relevant for us regarding measuring +memory footprint. + +Of coures, there are many more mappings, also for +the stack. Doing:: + + >>> l = ["xasd"] * 1000000" + +i get this additional mapping:: + +b7890000-b7c61000 rw-p b7890000 00:00 0 +Size: 3908 kB +Rss: 3908 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 3908 kB +Referenced: 3908 kB + +We have an anonymous readwrite mapping and the 3908 KBs +for the list and strings are mapped into physical ram. + +For some more information here is a link +http://bmaurer.blogspot.com/2006/03/memory-usage-with-smaps.html +which also points to the mem_usage.py tool that presents +a process mappings in a somewhat nicer summarized format. + + +Tool to measure python interpreter mem foot print +------------------------------------------------------- + +We need a tool that can invoke python apps and benchmarks +and measure memory foot print - producing data that can +be parsed back and used for producing graphs, tables etc. + +See also tools for maemo: + +http://maemo.org/development/tools/doc/diablo/sp-smaps-measure/ +http://maemo.org/development/tools/doc/diablo/sp-memusage/ + + +XXX NOTES XXXX +----------------- +notes on scenarios from fijal which need to be checked if they +are sufficiently reflected in the above sections. -Probable measurment scenarios: - -1. Measure private memory owned by process when no other interpreter's +1. Measure private memory when no other interpreter's process exist. -2. The same in case there is other one. +2. Measure private memory with another interprerter running at the same time. 3. Measure amount of private memory owned by process that was forked from other interpreter process. 4. Measure total RAM usage/RSS size in case process is running vs total RAM usage when process is not running. + From pedronis at codespeak.net Thu Sep 11 10:34:01 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 11 Sep 2008 10:34:01 +0200 (CEST) Subject: [pypy-svn] r58058 - pypy/branch/pypy-pytrunk/pypy/interpreter/test Message-ID: <20080911083401.9510216A2AE@codespeak.net> Author: pedronis Date: Thu Sep 11 10:34:00 2008 New Revision: 58058 Modified: pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_appinterp.py Log: (iko, pedronis) fix test, don't depend on excinfo str, depend on the SyntaxError one Modified: pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/branch/pypy-pytrunk/pypy/interpreter/test/test_appinterp.py Thu Sep 11 10:34:00 2008 @@ -22,7 +22,7 @@ (): y y """) - assert str(excinfo).find('y y') != -1 + assert str(excinfo.value).find('y y') != -1 def test_simple_applevel(space): app = appdef("""app(x,y): From fijal at codespeak.net Thu Sep 11 11:18:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Sep 2008 11:18:39 +0200 (CEST) Subject: [pypy-svn] r58059 - in pypy/dist/pypy/lib: . app_test Message-ID: <20080911091839.A1C5016A321@codespeak.net> Author: fijal Date: Thu Sep 11 11:18:36 2008 New Revision: 58059 Added: pypy/dist/pypy/lib/app_test/test_locale.py (contents, props changed) Modified: pypy/dist/pypy/lib/_locale.py Log: (kuchtak, fijal) * Adapt cpython's test suite to py.test style * Fix a bug with CHAR_MAX Modified: pypy/dist/pypy/lib/_locale.py ============================================================================== --- pypy/dist/pypy/lib/_locale.py (original) +++ pypy/dist/pypy/lib/_locale.py Thu Sep 11 11:18:36 2008 @@ -2,6 +2,7 @@ """ Support for POSIX locales. """ + raise ImportError("_locale.py is still incomplete") from ctypes import (Structure, POINTER, create_string_buffer, @@ -12,6 +13,8 @@ ConstantInteger, DefinedConstantInteger, SimpleType) size_t = c_int + +# XXX check where this comes from CHAR_MAX = 127 _CONSTANTS = ( @@ -416,7 +419,7 @@ 'Error', 'setlocale', 'localeconv', 'strxfrm', 'strcoll', 'gettext', 'dgettext', 'dcgettext', 'textdomain', - 'bindtextdomain', + 'bindtextdomain', 'CHAR_MAX', ) + _CONSTANTS if _bind_textdomain_codeset: __all__ += ('bind_textdomain_codeset',) Added: pypy/dist/pypy/lib/app_test/test_locale.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/test_locale.py Thu Sep 11 11:18:36 2008 @@ -0,0 +1,69 @@ +import py +import locale +import sys + +def setup_module(mod): + if sys.platform == 'darwin': + py.test.skip("Locale support on MacOSX is minimal and cannot be tested") + +class TestLocale: + def setup_class(cls): + cls.oldlocale = locale.setlocale(locale.LC_NUMERIC) + if sys.platform.startswith("win"): + cls.tloc = "en" + elif sys.platform.startswith("freebsd"): + cls.tloc = "en_US.US-ASCII" + else: + cls.tloc = "en_US.UTF8" + try: + locale.setlocale(locale.LC_NUMERIC, cls.tloc) + except locale.Error: + py.test.skip("test locale %s not supported" % cls.tloc) + + def teardown_class(cls): + locale.setlocale(locale.LC_NUMERIC, cls.oldlocale) + + def test_format(self): + + def testformat(formatstr, value, grouping = 0, output=None): + if output: + print "%s %% %s =? %s ..." %\ + (repr(formatstr), repr(value), repr(output)), + else: + print "%s %% %s works? ..." % (repr(formatstr), repr(value)), + result = locale.format(formatstr, value, grouping = grouping) + assert result == output + + testformat("%f", 1024, grouping=1, output='1,024.000000') + testformat("%f", 102, grouping=1, output='102.000000') + testformat("%f", -42, grouping=1, output='-42.000000') + testformat("%+f", -42, grouping=1, output='-42.000000') + testformat("%20.f", -42, grouping=1, output=' -42') + testformat("%+10.f", -4200, grouping=1, output=' -4,200') + testformat("%-10.f", 4200, grouping=1, output='4,200 ') + # Invoke getpreferredencoding to make sure it does not cause exceptions, + locale.getpreferredencoding() + + # Test BSD Rune locale's bug for isctype functions. + def test_bsd_bug(self): + def teststrop(s, method, output): + print "%s.%s() =? %s ..." % (repr(s), method, repr(output)), + result = getattr(s, method)() + assert result == output + + oldlocale = locale.setlocale(locale.LC_CTYPE) + locale.setlocale(locale.LC_CTYPE, self.tloc) + try: + teststrop('\x20', 'isspace', True) + teststrop('\xa0', 'isspace', False) + teststrop('\xa1', 'isspace', False) + teststrop('\xc0', 'isalpha', False) + teststrop('\xc0', 'isalnum', False) + teststrop('\xc0', 'isupper', False) + teststrop('\xc0', 'islower', False) + teststrop('\xec\xa0\xbc', 'split', ['\xec\xa0\xbc']) + teststrop('\xed\x95\xa0', 'strip', '\xed\x95\xa0') + teststrop('\xcc\x85', 'lower', '\xcc\x85') + teststrop('\xed\x95\xa0', 'upper', '\xed\x95\xa0') + finally: + locale.setlocale(locale.LC_CTYPE, oldlocale) From antocuni at codespeak.net Thu Sep 11 11:42:54 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 11 Sep 2008 11:42:54 +0200 (CEST) Subject: [pypy-svn] r58060 - in pypy/branch/oo-jit/pypy/rpython: . lltypesystem ootypesystem test Message-ID: <20080911094254.5B4B416A360@codespeak.net> Author: antocuni Date: Thu Sep 11 11:42:53 2008 New Revision: 58060 Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/rdict.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py pypy/branch/oo-jit/pypy/rpython/rpbc.py pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py Log: r57808 caused a lot of tests to fail. Revert it, and put a workaround directly in the rdict code Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/lltypesystem/rdict.py Thu Sep 11 11:42:53 2008 @@ -194,6 +194,11 @@ def convert_const(self, dictobj): + def convert_value_if_not_Void(repr, value): + if repr.lowleveltype is lltype.Void: + return None + return repr.convert_const(value) + # get object from bound dict methods #dictobj = getattr(dictobj, '__self__', dictobj) if dictobj is None: @@ -219,7 +224,7 @@ for dictkeycontainer, dictvalue in dictobj._dict.items(): llkey = r_key.convert_const(dictkeycontainer.key) - llvalue = r_value.convert_const(dictvalue) + llvalue = convert_value_if_not_Void(r_value, dictvalue) ll_dict_insertclean(l_dict, llkey, llvalue, dictkeycontainer.hash) return l_dict @@ -227,7 +232,7 @@ else: for dictkey, dictvalue in dictobj.items(): llkey = r_key.convert_const(dictkey) - llvalue = r_value.convert_const(dictvalue) + llvalue = convert_value_if_not_Void(r_value, dictvalue) ll_dict_insertclean(l_dict, llkey, llvalue, l_dict.keyhash(llkey)) return l_dict Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py Thu Sep 11 11:42:53 2008 @@ -166,6 +166,11 @@ def convert_const(self, dictobj): + def convert_value_if_not_Void(repr, value): + if repr.lowleveltype is ootype.Void: + return None + return repr.convert_const(value) + if dictobj is None: return self.DICT._defl() if not isinstance(dictobj, dict) and not isinstance(dictobj, objectmodel.r_dict): @@ -191,14 +196,14 @@ if self.custom_eq_hash: for dictkeycont, dictvalue in dictobj._dict.items(): llkey = r_key.convert_const(dictkeycont.key) - llvalue = r_value.convert_const(dictvalue) + llvalue = convert_value_if_not_Void(r_value, dictvalue) llhash = dictkeycont.hash l_dictkeycont = objectmodel._r_dictkey_with_hash(l_dict._dict, llkey, llhash) l_dict._dict._dict[l_dictkeycont] = llvalue else: for dictkey, dictvalue in dictobj.items(): llkey = r_key.convert_const(dictkey) - llvalue = r_value.convert_const(dictvalue) + llvalue = convert_value_if_not_Void(r_value, dictvalue) l_dict.ll_set(llkey, llvalue) return l_dict Modified: pypy/branch/oo-jit/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/rpbc.py (original) +++ pypy/branch/oo-jit/pypy/rpython/rpbc.py Thu Sep 11 11:42:53 2008 @@ -403,9 +403,6 @@ raise TyperError("getattr on a constant PBC returns a non-constant") return hop.inputconst(hop.r_result, hop.s_result.const) - def convert_const(self, value): - return None - def convert_desc(self, frozendesc): assert frozendesc is self.frozendesc return object() # lowleveltype is Void Modified: pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_rpbc.py Thu Sep 11 11:42:53 2008 @@ -1594,16 +1594,6 @@ self.interpret(f, [int]) - def test_specialize_singlefrozenpbc(self): - py.test.skip("r57808 makes this test fail, and llgraph tests rely on this") - from pypy.rlib.objectmodel import specialize - @specialize.arg(0) - def revealconst(T, x): - return lltype.cast_primitive(T, value) - def fn(x): - return revealconst(lltype.Signed, x) - res = self.interpret(fn, [42], backendopt=False) - assert res == 42 class TestLLtype(BaseTestRPBC, LLRtypeMixin): pass From hpk at codespeak.net Thu Sep 11 11:45:59 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 11:45:59 +0200 (CEST) Subject: [pypy-svn] r58061 - pypy/extradoc/talk/osdc2008 Message-ID: <20080911094559.B7FA316A2EF@codespeak.net> Author: hpk Date: Thu Sep 11 11:45:55 2008 New Revision: 58061 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: some comments in the form of XXX Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Thu Sep 11 11:45:55 2008 @@ -3,6 +3,10 @@ :author: Michael Hudson +XXX the focus in the paper shifts from translation framework +XXX to pypy as python interpreter. I'd recommend to up-front +XXX distinguish more clearly. + Abstract -------- @@ -48,6 +52,11 @@ Motivation ---------- +XXX it's good to highlight the shared idea/value of +XXX focusing on implementation rather than language design +XXX (it's till the case today, i'd say). +XXX if it's meant to tell how PyPy came about it's a mispresentation though + The beginnings PyPy can be traced to the first EuroPython conference in 2002, where some of the people who ultimately became involved in the project met in person for the first time, and realized they had a @@ -149,11 +158,16 @@ Status ------ +XXX some words about GCs, memory usage? + PyPy when translated with the C backend (often called pypy-c) and most optimizations enabled is currently a reasonably fast and extremely conformant implementation of Python 2.4.4. XXX is conformant a word? +XXX isn't it a wider range like 0.8 to 5 times, see e.g. recent sympy feedback on pypy-dev from Ondrej +with an emphasis on it rather being slower usually? + Reasonably fast: depending on the code being run, pypy-c ranges from being 20% faster to about twice as slow as CPython (the still-experimental Just-In-Time compiler runs some (admittedly @@ -176,6 +190,8 @@ The former is usually a much more significant problem, of course. +XXX mention sandboxing as a distinguished feature? + Compatibility with Python 2.5 is almost complete in a branch. 2.6 shouldn't be too hard. No Python 3 yet :) @@ -190,6 +206,7 @@ PyPy started as a classic open-source, spare-time project, was funded for a while, and is currently mostly a spare-time project again. +XXX not a completely adequate description, e.g. google contracts Right from the start, however, the project was driven along by *sprints* -- focused week long coding sessions. During the funding @@ -206,6 +223,11 @@ already extended the state of the art in automatically generating a JIT compiler for an implementation of a dynamic language. +XXX mention that PyPy provides unprecedented possibilities in + producing Interpreters for specific environments (memory-constrained, + sandboxing, gaming-engines etc.) ? (hpk) i at least think it's a + very exciting area as well. + Down at the more nuts and bolts implementation level, something I'm interested in myself is stealing ideas -- and maybe even code -- from garbage collectors developed by the Jikes RVM project. From hpk at codespeak.net Thu Sep 11 11:49:44 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 11:49:44 +0200 (CEST) Subject: [pypy-svn] r58062 - pypy/build/doc Message-ID: <20080911094944.207EB16A31D@codespeak.net> Author: hpk Date: Thu Sep 11 11:49:35 2008 New Revision: 58062 Modified: pypy/build/doc/benchmark_memory.txt Log: add another link that nicely explains swap/dirty/clean linux mem management Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 11:49:35 2008 @@ -134,6 +134,8 @@ which also points to the mem_usage.py tool that presents a process mappings in a somewhat nicer summarized format. +For understanding how swapping and linux memory management works here is a nice read: http://sourcefrog.net/weblog/software/linux-kernel/swap.html + Tool to measure python interpreter mem foot print ------------------------------------------------------- From hpk at codespeak.net Thu Sep 11 12:00:57 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 12:00:57 +0200 (CEST) Subject: [pypy-svn] r58063 - pypy/build/doc Message-ID: <20080911100057.630641684D2@codespeak.net> Author: hpk Date: Thu Sep 11 12:00:51 2008 New Revision: 58063 Modified: pypy/build/doc/benchmark_memory.txt Log: some ReST fixes, clarify XXX Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 12:00:51 2008 @@ -25,8 +25,7 @@ * Scenario/App Benchmarks: * allocate objects all the time, total number of objects stay constant * allocate objects in bursts and immediately throw them away - * constant small number live objects, burst-allocate many, - free them all, repeat + * constant small number live objects, burst-allocate many, free them all, repeat * use several sets of "small" and "many" * real APPS: @@ -62,21 +61,22 @@ Understanding linux /proc/pid/smaps info ------------------------------------------- +XXX please review, correct, complete so we get to a shared good understanding + The most detailed info apparently is provided by /proc/PID/smaps, starting from linux 2.6.14. Here is an example output of running "python2.5" on a linux 2.6.24 ubuntu machine:: -XXX please review, correct, complete so we get to a shared good understanding -08048000-08140000 r-xp 00000000 08:01 89921 /usr/bin/python2.5 -Size: 992 kB -Rss: 768 kB -Shared_Clean: 764 kB -Shared_Dirty: 0 kB -Private_Clean: 4 kB -Private_Dirty: 0 kB -Referenced: 768 kB + 08048000-08140000 r-xp 00000000 08:01 89921 /usr/bin/python2.5 + Size: 992 kB + Rss: 768 kB + Shared_Clean: 764 kB + Shared_Dirty: 0 kB + Private_Clean: 4 kB + Private_Dirty: 0 kB + Referenced: 768 kB The first line indicates that the /usr/bin/python2.5 file is mapped as Read/eXecute into the given process and is @@ -88,19 +88,19 @@ will get their mapping but no additional RAM will be used for these 764 kBs. "clean" means that these pages can easily get swapped out by dropping them and - upon access - retrieving them -from the file. XXX what does Private_Clean / 4kB mean exactly here? +from the file. XXX but why is the Private_Clean page there in this readonly /usr/bin/python mapping? Let's look at a mapping that is more indicative of the per-process "incremental" RAM usage:: -08165000-081e0000 rw-p 08165000 00:00 0 [heap] -Size: 492 kB -Rss: 452 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 452 kB -Referenced: 452 kB + 08165000-081e0000 rw-p 08165000 00:00 0 [heap] + Size: 492 kB + Rss: 452 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 452 kB + Referenced: 452 kB Here we have a readwrite anonymous mapping, objects allocated on the heap. It uses 492 kB virtual address @@ -111,22 +111,24 @@ memory footprint. Of coures, there are many more mappings, also for -the stack. Doing:: +the stack. + +Let's see what changes if we do:: >>> l = ["xasd"] * 1000000" -i get this additional mapping:: +we get this new mapping in the python process:: -b7890000-b7c61000 rw-p b7890000 00:00 0 -Size: 3908 kB -Rss: 3908 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 3908 kB -Referenced: 3908 kB + b7890000-b7c61000 rw-p b7890000 00:00 0 + Size: 3908 kB + Rss: 3908 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 3908 kB + Referenced: 3908 kB -We have an anonymous readwrite mapping and the 3908 KBs +which is anonymous readwrite mapping and the 3908 KBs for the list and strings are mapped into physical ram. For some more information here is a link From fijal at codespeak.net Thu Sep 11 13:35:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Sep 2008 13:35:50 +0200 (CEST) Subject: [pypy-svn] r58064 - pypy/build/doc Message-ID: <20080911113550.96A1316A346@codespeak.net> Author: fijal Date: Thu Sep 11 13:35:48 2008 New Revision: 58064 Modified: pypy/build/doc/benchmark_memory.txt Log: I cannot resist adding tuples there ;] Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 13:35:48 2008 @@ -18,7 +18,7 @@ * Microbenchmarks probably with checkpoints in order to have reproducible RAM usage measurements and to analyse - specific object types (strings/dicts/lists/old-style/new-style + specific object types (strings/dicts/lists/tuples/old-style/new-style instances ...) and scenarios like reading lines in large file. (reuse some of the speed benchmarks?) From fijal at codespeak.net Thu Sep 11 13:40:16 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 11 Sep 2008 13:40:16 +0200 (CEST) Subject: [pypy-svn] r58065 - pypy/build/doc Message-ID: <20080911114016.66B3A16A353@codespeak.net> Author: fijal Date: Thu Sep 11 13:40:15 2008 New Revision: 58065 Modified: pypy/build/doc/benchmark_memory.txt Log: add somewhat random notes about GCs. It's likely we want a separate document for that Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 13:40:15 2008 @@ -47,6 +47,13 @@ =============================== XXX go into more detail, discuss which pypy GCs we want to try +XXX [fijal] I think both approaches are worth trying. We need to see + how they behave or try to think how much memory it'll be needed for + a GC header. note that deferred refcounting without nursery is needed + for deferred refcounting with nursery (just because it's easier to have + this in steps). We also want to consider impacts of fragmentation and + how underlaying C library handles stuff (ie do we use obmalloc or no. + does it return memory to the OS or no for small objects etc.) * squeak-like mark-compact collector From antocuni at codespeak.net Thu Sep 11 14:09:54 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 11 Sep 2008 14:09:54 +0200 (CEST) Subject: [pypy-svn] r58066 - in pypy/branch/oo-jit/pypy: jit/codegen/cli/test translator/oosupport translator/oosupport/test_template Message-ID: <20080911120954.B3EBE16A2ED@codespeak.net> Author: antocuni Date: Thu Sep 11 14:09:49 2008 New Revision: 58066 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/translator/oosupport/constant.py pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py Log: make sure that constants of type Object are recorded only once by the backends. Some more jit tests pass. Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 11 14:09:49 2008 @@ -100,8 +100,6 @@ def test_indirect_gray_call(self): py.test.skip('mono 1.2 crashes, try again with a newer version') - test_constant_indirect_red_call = skip - test_constant_indirect_red_call_no_result = skip test_indirect_sometimes_residual_pure_red_call = skip test_red_int_add_ovf = skip test_nonzeroness_assert_while_compiling = skip Modified: pypy/branch/oo-jit/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/oosupport/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/oosupport/constant.py Thu Sep 11 14:09:49 2008 @@ -186,6 +186,9 @@ should be an ootype constant value. Not generally called directly, but it can be if desired. """ assert not is_primitive(value) + if isinstance(value, ootype._object) and value: # leave ootype.NULL as is + value = value.obj + self.db.cts.lltype_to_cts(value._TYPE) # record const if value in self.cache: return self.cache[value] const = self._create_complex_const(value) @@ -200,10 +203,6 @@ """ A helper method which creates a Constant wrapper object for the given value. Uses the types defined in the sub-class. """ - if isinstance(value, ootype._object) and value: # leave ootype.NULL as is - value = value.obj - self.db.cts.lltype_to_cts(value._TYPE) # record const - # Determine if the static type differs from the dynamic type. if isinstance(value, ootype._view): static_type = value._TYPE Modified: pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py (original) +++ pypy/branch/oo-jit/pypy/translator/oosupport/test_template/constant.py Thu Sep 11 14:09:49 2008 @@ -154,3 +154,20 @@ return s1 res = self.interpret(fn, [], backendopt=False) assert res == 'hello world' + + def test_unwrap_object(self): + A = ootype.Instance("A", ootype.ROOT, {}) + a1 = ootype.new(A) + a2 = ootype.new(A) + obj1 = ootype.cast_to_object(a1) + obj2 = ootype.cast_to_object(a2) + def fn(flag): + if flag: + obj = obj1 + else: + obj = obj2 + a3 = ootype.cast_from_object(A, obj) + return a3 is a1 + res = self.interpret(fn, [True], backendopt=False) + assert res is True + From antocuni at codespeak.net Thu Sep 11 14:14:52 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 11 Sep 2008 14:14:52 +0200 (CEST) Subject: [pypy-svn] r58067 - pypy/branch/oo-jit/pypy/jit/codegen/cli/test Message-ID: <20080911121452.3920A16A291@codespeak.net> Author: antocuni Date: Thu Sep 11 14:14:51 2008 New Revision: 58067 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: more passing tests Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Thu Sep 11 14:14:51 2008 @@ -100,10 +100,9 @@ def test_indirect_gray_call(self): py.test.skip('mono 1.2 crashes, try again with a newer version') - test_indirect_sometimes_residual_pure_red_call = skip - test_red_int_add_ovf = skip - test_nonzeroness_assert_while_compiling = skip - test_segfault_while_compiling = skip + def test_red_int_add_ovf(self): + py.test.skip("TODO: exceptions") + test_learn_nonzeroness = skip test_freeze_booleffects_correctly = skip test_ptrequality = skip From pedronis at codespeak.net Thu Sep 11 16:05:38 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 11 Sep 2008 16:05:38 +0200 (CEST) Subject: [pypy-svn] r58068 - in pypy/branch/pypy-pytrunk/pypy: . tool/pytest tool/pytest/test Message-ID: <20080911140538.6EC9716A3CD@codespeak.net> Author: pedronis Date: Thu Sep 11 16:05:34 2008 New Revision: 58068 Added: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py (contents, props changed) pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py (contents, props changed) Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py Log: (iko, pedronis) start of a py.test Session subclass that additionally also records outcome events in a log file in a format similar to htmlconftest outcome Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Thu Sep 11 16:05:34 2008 @@ -29,7 +29,10 @@ help="run applevel tests directly on python interpreter (not through PyPy)"), Option('--direct', action="store_true", default=False, dest="rundirect", - help="run pexpect tests directly") + help="run pexpect tests directly"), + Option('--filelog', action="store", + default=None, dest="filelog", + help="path for FileLogSession logging") ) _SPACECACHE={} @@ -494,3 +497,5 @@ # disable recursion in symlinked subdirectories return (py.test.collect.Directory.recfilter(self, path) and path.check(link=0)) + +from pypy.tool.pytest.filelog import FileLogSession Added: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py Thu Sep 11 16:05:34 2008 @@ -0,0 +1,23 @@ +from py.__.test.session import Session +from py.__.test import event + +class FileLogSession(Session): + + def __init__(self, config): + super(FileLogSession, self).__init__(config) + self.bus.subscribe(self.log_event_to_file) + if hasattr(config.option, 'filelog'): + filelog = config.option.filelog + self.logfile = open(filelog, 'w') # line buffering ? + + + def log_event_to_file(self, ev): + if isinstance(ev, event.ItemTestReport): + outcome = ev.outcome + metainfo = ev.colitem.repr_metainfo() + path = metainfo.fspath + modpath = metainfo.modpath + if modpath: + path += ":%s" % modpath + print >>self.logfile, "%s %s" % (outcome.shortrepr, path) + Added: pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Thu Sep 11 16:05:34 2008 @@ -0,0 +1,72 @@ +from pypy.tool.pytest import filelog +import os, StringIO + +from py.__.test.event import ItemTestReport + + +class Fake(object): + def __init__(self, **kwds): + self.__dict__.update(kwds) + + +class TestFileLogSession(object): + + + def test_sanity(self): + option = Fake(eventlog=None) + config = Fake(option=option) + + filelog.FileLogSession(config) + + def test_open_logfile(self): + logfname = os.tempnam() + + option = Fake(eventlog=None, filelog=logfname) + config = Fake(option=option) + + sess = filelog.FileLogSession(config) + + assert len(sess.bus._subscribers) == 1 + + assert sess.logfile + assert os.path.exists(logfname) + + sess.logfile.close() + os.unlink(logfname) + + def test_item_test_passed_or_skipped(self): + option = Fake(eventlog=None) + config = Fake(option=option) + sess = filelog.FileLogSession(config) + sess.logfile = StringIO.StringIO() + + colitem = Fake(repr_metainfo=lambda: Fake(fspath='some/path', + modpath="a.b")) + outcome=Fake(shortrepr='.') + rep_ev = ItemTestReport(colitem, outcome=outcome) + + sess.bus.notify(rep_ev) + + lines = sess.logfile.getvalue().splitlines() + assert len(lines) == 1 + line = lines[0] + assert line.startswith(". ") + assert line[2:] == 'some/path:a.b' + + sess.logfile = StringIO.StringIO() + colitem = Fake(repr_metainfo=lambda: Fake(fspath='some/path', + modpath=None)) + outcome=Fake(shortrepr='s') + rep_ev = ItemTestReport(colitem, outcome=outcome) + + sess.bus.notify(rep_ev) + + lines = sess.logfile.getvalue().splitlines() + assert len(lines) == 1 + line = lines[0] + + assert line.startswith("s ") + assert line[2:] == 'some/path' + + +# XXX integration tests From arigo at codespeak.net Thu Sep 11 16:28:13 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 11 Sep 2008 16:28:13 +0200 (CEST) Subject: [pypy-svn] r58069 - pypy/build/doc Message-ID: <20080911142813.19FD916A340@codespeak.net> Author: arigo Date: Thu Sep 11 16:28:12 2008 New Revision: 58069 Modified: pypy/build/doc/benchmark_memory.txt Log: Updates. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 16:28:12 2008 @@ -4,30 +4,56 @@ what we want to measure =============================================== -* measure min/max/avg process-private RAM usage of a Benchmark +* measure max/avg RAM usage of a process running various + GC benchmarks, for a relevant definition of RAM usage. -* measure (min/max/avg) collection time of an APP/Benchmark +* measure max/avg combined RAM usage of a few processes + running the GC benchmarks - either launched + independently, or forked from a single process + +* measure the "percieved pauses" for interactive apps + when running on GCs with stop-the-world collection + (i.e. all our framework GCs so far). + +* measure the CPU usage and total execution time of some + apps (for cross-reference, e.g. to show the space-time + trade-offs of the various GCs). -* measure CPU usage of benchmark - -* measure for 1/2/N interpreters running at the same time - and maybe also for forked processes. - -Benchmark targets +Benchmark specifics -------------------- -* Microbenchmarks probably with checkpoints in order to have - reproducible RAM usage measurements and to analyse - specific object types (strings/dicts/lists/tuples/old-style/new-style - instances ...) and scenarios like reading lines in large file. +* RAM usage: we need to refine what is exactly meant by this. + An approximation is the amount of process-private RAM + reported by the kernel. See discussion below. + +* GC microbenchmarks: allocating specific object types + (strings/dicts/lists/tuples/old-style/new-style instances ...) + and scenarios like reading lines in large file. (reuse some of the speed benchmarks?) -* Scenario/App Benchmarks: - * allocate objects all the time, total number of objects stay constant - * allocate objects in bursts and immediately throw them away - * constant small number live objects, burst-allocate many, free them all, repeat - * use several sets of "small" and "many" - +* allocation patterns: + 1. allocate objects all the time, total number of objects + stays constant; + 2. allocate many objects in a burst, then throw them away, + and repeat; + 3. combined: do 1 for most of the time with a small number + of live objects, but occasionally get a large bunch of + objects and immediately free them. + Use several sets of "small" and "large". + +* Aim for reproducible results, e.g. by combining some of + these techniques: + - checkpoints (perform measures at known execution points); + - high-res sampling (either in the real system or in + emulators, e.g. look at valgrind tools). + +* The "percieved pause" is probably best approximated by + the time it takes to perform a single collection. For + generational GCs we should measure the time of the + collections of various levels; for example, nursery-only + collections are very fast, probably too fast to be + noticeable in interactive apps. + * real APPS: * some sympy computation/test? * an app involving c-extensions? @@ -43,6 +69,27 @@ We want to select and optimize good underlying settings for PyPy's choices regarding "--opt=mem". +Measuring memory usage +------------------------ + +Linux's "RSS" cannot be directly used because in a running +pypy-c it incorrectly counts many megabytes that are null +pages, loaded from /dev/zero. Such pages are not included in +the process-private memory, which is why the latter might be a +better measure. It might be that even more complicated +measures would be even better. + +We can test and compare different ways to measure memory +usage. One testing methodology would be to run a virtual +machine which pretends to have X megabytes of RAM, start one +or several pypy-c's running some benchmarks, see when it +starts swapping, and apply the proposed memory usage measure +at this point to get a number Y. Then we replace the pypy-c's +with a single trivial C program that clearly consumes Z +megabytes of RAM, and see for which value of Z it starts +swapping. When Y and Z agree, we found a good way to measure +memory usage. + notes about memory-saving GCs =============================== From hpk at codespeak.net Thu Sep 11 18:51:19 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 18:51:19 +0200 (CEST) Subject: [pypy-svn] r58071 - pypy/build/doc Message-ID: <20080911165119.5130D16A484@codespeak.net> Author: hpk Date: Thu Sep 11 18:51:16 2008 New Revision: 58071 Added: pypy/build/doc/benchmark_memory_gcnotes.txt (contents, props changed) Modified: pypy/build/doc/benchmark_memory.txt (contents, props changed) Log: * shifting gc notes to a separate doc for now as discussed with fijal (it's too unmotivated in the benchmark_memory.txt section for now) * shifting "measuring memory" on linux to the implementation section where we also discuss /proc/pid/smaps afterwards, kind of serves as motiviation for this. * some more boilerplate / warnings here and there Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 18:51:16 2008 @@ -1,5 +1,14 @@ +=============================================== +Benchmarking Memory usage of Python/PyPy +=============================================== + +$Id$ -XXX draft doc, needs discussion, review +XXX this is a draft incomplete document +XXX next step is to lay out and write +XXX the benchmarking infrastructure and +XXX to incrementally complete and update +XXX the document what we want to measure =============================================== @@ -59,8 +68,8 @@ * an app involving c-extensions? * ask around on pypy-dev for example apps -Interpreters ---------------- +Interpreter instances to consider for measurement +---------------------------------------------------- * CPython 2.5 * pypy-c --opt=mem @@ -69,6 +78,11 @@ We want to select and optimize good underlying settings for PyPy's choices regarding "--opt=mem". +XXX consider more specific target environments + +Implementation +=========================== + Measuring memory usage ------------------------ @@ -90,37 +104,14 @@ swapping. When Y and Z agree, we found a good way to measure memory usage. -notes about memory-saving GCs -=============================== - -XXX go into more detail, discuss which pypy GCs we want to try -XXX [fijal] I think both approaches are worth trying. We need to see - how they behave or try to think how much memory it'll be needed for - a GC header. note that deferred refcounting without nursery is needed - for deferred refcounting with nursery (just because it's easier to have - this in steps). We also want to consider impacts of fragmentation and - how underlaying C library handles stuff (ie do we use obmalloc or no. - does it return memory to the OS or no for small objects etc.) - -* squeak-like mark-compact collector - -* deferred refcounting strategy - -* deferred refcounting with a nursery - - -Implementation -=========================== - Understanding linux /proc/pid/smaps info ------------------------------------------- XXX please review, correct, complete so we get to a shared good understanding -The most detailed info apparently is provided -by /proc/PID/smaps, starting from linux 2.6.14. -Here is an example output of running -"python2.5" on a linux 2.6.24 ubuntu machine:: +The most detailed info is provided by /proc/PID/smaps, +starting from linux 2.6.14. Here is an example output +of running "python2.5" on a linux 2.6.24 ubuntu machine:: 08048000-08140000 r-xp 00000000 08:01 89921 /usr/bin/python2.5 @@ -200,25 +191,10 @@ and measure memory foot print - producing data that can be parsed back and used for producing graphs, tables etc. -See also tools for maemo: +Cross-Check also for tools against maemo: http://maemo.org/development/tools/doc/diablo/sp-smaps-measure/ http://maemo.org/development/tools/doc/diablo/sp-memusage/ - -XXX NOTES XXXX ------------------ -notes on scenarios from fijal which need to be checked if they -are sufficiently reflected in the above sections. - -1. Measure private memory when no other interpreter's - process exist. - -2. Measure private memory with another interprerter running at the same time. - -3. Measure amount of private memory owned by process that was forked from - other interpreter process. - -4. Measure total RAM usage/RSS size in case process is running vs - total RAM usage when process is not running. +XXX discuss and write toolchain Added: pypy/build/doc/benchmark_memory_gcnotes.txt ============================================================================== --- (empty file) +++ pypy/build/doc/benchmark_memory_gcnotes.txt Thu Sep 11 18:51:16 2008 @@ -0,0 +1,20 @@ + +notes about memory-saving GCs +=============================== + +XXX go into more detail, discuss which pypy GCs we want to try +XXX [fijal] I think both approaches are worth trying. We need to see + how they behave or try to think how much memory it'll be needed for + a GC header. note that deferred refcounting without nursery is needed + for deferred refcounting with nursery (just because it's easier to have + this in steps). We also want to consider impacts of fragmentation and + how underlaying C library handles stuff (ie do we use obmalloc or no. + does it return memory to the OS or no for small objects etc.) + +* squeak-like mark-compact collector + +* deferred refcounting strategy + +* deferred refcounting with a nursery + + From hpk at codespeak.net Thu Sep 11 19:05:01 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 11 Sep 2008 19:05:01 +0200 (CEST) Subject: [pypy-svn] r58072 - pypy/build/doc Message-ID: <20080911170501.9FB9216A475@codespeak.net> Author: hpk Date: Thu Sep 11 19:04:57 2008 New Revision: 58072 Modified: pypy/build/doc/benchmark_memory.txt Log: some words regarding base interpreter size and that we want to mainly measure RAM usage caused by the app/benchmark. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Thu Sep 11 19:04:57 2008 @@ -28,17 +28,22 @@ apps (for cross-reference, e.g. to show the space-time trade-offs of the various GCs). + Benchmark specifics -------------------- * RAM usage: we need to refine what is exactly meant by this. An approximation is the amount of process-private RAM reported by the kernel. See discussion below. + Also, we likely want to mainly measure the incremental + RAM usage of a particular benchmark/app - i.e. the + RAM that is used in addition to the bare interpreter + loaded. * GC microbenchmarks: allocating specific object types (strings/dicts/lists/tuples/old-style/new-style instances ...) - and scenarios like reading lines in large file. - (reuse some of the speed benchmarks?) + and little scenarios like reading lines from a large file. + also consider reuse some of the speed benchmarks. * allocation patterns: 1. allocate objects all the time, total number of objects @@ -56,7 +61,7 @@ - high-res sampling (either in the real system or in emulators, e.g. look at valgrind tools). -* The "percieved pause" is probably best approximated by +* The "perceived pause" is probably best approximated by the time it takes to perform a single collection. For generational GCs we should measure the time of the collections of various levels; for example, nursery-only @@ -76,7 +81,11 @@ * pypy-c --opt=3 (for comparison purposes) We want to select and optimize good underlying settings for -PyPy's choices regarding "--opt=mem". +PyPy's choices regarding "--opt=mem". We +also want to measure builds that include +a working set of modules and ones that include +no modules at all (the bare minimum for an +interactive prompt). XXX consider more specific target environments From cami at codespeak.net Fri Sep 12 00:08:34 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Fri, 12 Sep 2008 00:08:34 +0200 (CEST) Subject: [pypy-svn] r58076 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080911220834.550FE16A61A@codespeak.net> Author: cami Date: Fri Sep 12 00:08:30 2008 New Revision: 58076 Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/video.py Log: refactored video adapted tests Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Fri Sep 12 00:08:30 2008 @@ -489,3 +489,13 @@ assert video.status.read(extend=True) == 0xFE assert video.cycles == constants.MODE_2_TICKS assert not video.lcd_interrupt_flag.is_pending() + +def test_draw_clean_background(): + video = get_video() + assert video.line == [0] * (8+160+8) + + video.line = range(8+160+8) + video.draw_clean_background() + + assert video.line == [0] * (8+160+8) + \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Fri Sep 12 00:08:30 2008 @@ -1,14 +1,23 @@ from pypy.lang.gameboy import constants from pypy.lang.gameboy.video import ControlRegister from pypy.lang.gameboy.video import StatusRegister +from pypy.lang.gameboy.video import Window +from pypy.lang.gameboy.video import Background from pypy.lang.gameboy.test.test_video import get_video import py +def get_control_register(): + video = get_video() + return ControlRegister(Window(video), Background()) + +def get_status_register(): + return StatusRegister(get_video()) + # ControlRegister -------------------------------------------------------------- def test_video_control_reset(): - control = ControlRegister() + control = get_control_register() assert control.read() == 0x91 control.write(0xFF) assert control.read() == 0xFF @@ -17,42 +26,17 @@ def test_video_control_read_write_properties(): - control = ControlRegister() - properties = ["lcd_enabled", - "window_upper_tile_map_selected", - "window_enabled", - "background_and_window_lower_tile_data_selected", - "background_upper_tile_map_selected", - "big_sprite_size_selected", - "sprite_display_enabled", - "background_enabled"] - properties.reverse() - for index in range(8): - property = properties[index]; - control.write(0x00) - assert control.read() == 0x00 - assert control.__getattribute__(property) == False - - control.write(0xFF) - assert control.read() == 0xFF - assert control.__getattribute__(property) == True - - control.write(0x00) - control.__setattr__(property, True) - assert control.__getattribute__(property) == True - assert control.read() & (1 << index) == (1 << index) - assert control.read() & (~(1 << index)) == 0 - - control.write(1 << index) - assert control.__getattribute__(property) == True - assert control.read() & (1 << index) == (1 << index) - assert control.read() & (~(1 << index)) == 0 + control = get_control_register() + + for i in range(0xFF): + control.write(i) + assert control.read() == i # StatusRegister --------------------------------------------------------------- def test_video_status_reset(): - status = StatusRegister(get_video()) + status = get_status_register() assert status.read(extend=True) == 0x02 + 0x80 status.write(0x00, write_all=True) @@ -66,7 +50,7 @@ assert status.read(extend=True) == 0x02 + 0x80 def test_video_status_mode(): - status = StatusRegister(get_video()) + status = get_status_register() assert status.get_mode() == 2 for i in range(3): Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Fri Sep 12 00:08:30 2008 @@ -43,42 +43,56 @@ Bit 1 - OBJ (Sprite) Display Enable (0=Off, 1=On) Bit 0 - BG Display (for CGB see below) (0=Off, 1=On) """ - def __init__(self): + def __init__(self, window, background): + self.window = window + self.background = background self.reset() def reset(self): self.lcd_enabled = True - self.window_upper_tile_map_selected = False - self.window_enabled = False + self.window.upper_tile_map_selected = False + self.window.enabled = False self.background_and_window_lower_tile_data_selected = True - self.background_upper_tile_map_selected = False + self.background.upper_tile_map_selected = False self.big_sprite_size_selected = False - self.sprite_display_enabled = False - self.background_enabled = True + self.sprites_enabled = False + self.background.enabled = True def read(self): value = 0 value += int(self.lcd_enabled) << 7 - value += int(self.window_upper_tile_map_selected) << 6 - value += int(self.window_enabled) << 5 + value += int(self.window.upper_tile_map_selected) << 6 + value += int(self.window.enabled) << 5 value += int(self.background_and_window_lower_tile_data_selected) << 4 - value += int(self.background_upper_tile_map_selected) << 3 + value += int(self.background.upper_tile_map_selected) << 3 value += int(self.big_sprite_size_selected) << 2 - value += int(self.sprite_display_enabled) << 1 - value += int(self.background_enabled) + value += int(self.sprites_enabled) << 1 + value += int(self.background.enabled) return value def write(self, value): self.lcd_enabled = bool(value & (1 << 7)) - self.window_upper_tile_map_selected = bool(value & (1 << 6)) - self.window_enabled = bool(value & (1 << 5)) + self.window.upper_tile_map_selected = bool(value & (1 << 6)) + self.window.enabled = bool(value & (1 << 5)) self.background_and_window_lower_tile_data_selected = \ bool(value & (1 << 4)) - self.background_upper_tile_map_selected = bool(value & (1 << 3)) + self.background.upper_tile_map_selected = bool(value & (1 << 3)) self.big_sprite_size_selected = bool(value & (1 << 2)) - self.sprite_display_enabled = bool(value & (1 << 1)) - self.background_enabled = bool(value & (1 << 0)) + self.sprites_enabled = bool(value & (1 << 1)) + self.background.enabled = bool(value & (1 << 0)) + def get_tile_map_combined(self): + #if (self.control.read() & mask) != 0: + if self.background_and_window_lower_tile_data_selected: + return constants.VRAM_MAP_B + else: + return constants.VRAM_MAP_A + + def get_selected_tile_map_space(self): + if self.window.upper_tile_map_selected: + return constants.VRAM_DATA_A + else: + return constants.VRAM_DATA_B # ----------------------------------------------------------------------------- class InvalidModeOrderException(Exception): @@ -86,7 +100,10 @@ Exception.__init__(self, \ "Wrong Mode order! No valid transition %i => %i" \ % (previous_mode.id(), mode.id())) - + +class HandleSubclassException(Exception): + def __init__(self): + Exception.__init__(self, "") class Mode(object): @@ -499,20 +516,28 @@ def set_tile_data(self, data): pass - def get_tile_data(self): + def get_selected_tile_map_space(self): pass # ----------------------------------------------------------------------------- class Window(object): - def __init__(self): + def __init__(self, video): + self.video = video self.reset() def reset(self): self.x = 0 self.y = 0 self.line_y = 0 + self.upper_tile_map_selected = False + self.enabled = False + def update_line_y(self, data): + # don't draw window if it was not enabled and not being drawn before + if not self.enabled and (data & 0x20) != 0 and \ + self.line_y == 0 and self.video.line_y > self.y: + self.line_y = 144 class Background(object): @@ -524,6 +549,8 @@ # be displayed in the left upper corner of the screen. self.scroll_x = 0 self.scroll_y = 0 + self.enabled = True + self.upper_tile_map_selected = False # ----------------------------------------------------------------------------- @@ -534,10 +561,11 @@ self.driver = video_driver self.v_blank_interrupt_flag = interrupt.v_blank self.lcd_interrupt_flag = interrupt.lcd - self.control = ControlRegister() - self.window = Window() - self.status = StatusRegister(self) + self.window = Window(self) self.background = Background() + self.status = StatusRegister(self) + self.control = ControlRegister(self.window, + self.background) self.memory = memory self.create_tile_maps() self.create_sprites() @@ -545,8 +573,8 @@ def create_tile_maps(self): # create the maxumal possible sprites - self.tile_map_1 = [None]*32*32 - self.tile_map_2 = [None]*32*32 + self.tile_map_1 = [None] * 32 * 32 + self.tile_map_2 = [None] * 32 * 32 def update_tiles(self): pass @@ -564,7 +592,6 @@ self.oam[address + 2], self.oam[address + 3]) - def reset(self): self.control.reset() self.status.reset() @@ -625,9 +652,11 @@ self.set_window_y(data) elif address == constants.WX: self.set_window_x(data) - elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: + elif constants.OAM_ADDR <= address < \ + constants.OAM_ADDR + constants.OAM_SIZE: self.set_oam(address, data) - elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: + elif constants.VRAM_ADDR <= address < \ + constants.VRAM_ADDR + constants.VRAM_SIZE: self.set_vram(address, data) def read(self, address): @@ -655,9 +684,11 @@ return self.get_window_y() elif address == constants.WX: return self.get_window_x() - elif constants.OAM_ADDR <= address < constants.OAM_ADDR + constants.OAM_SIZE: + elif constants.OAM_ADDR <= address < \ + constants.OAM_ADDR + constants.OAM_SIZE: return self.get_oam(address) - elif constants.VRAM_ADDR <= address < constants.VRAM_ADDR + constants.VRAM_SIZE: + elif constants.VRAM_ADDR <= address < \ + constants.VRAM_ADDR + constants.VRAM_SIZE: return self.get_vram(address) return 0xFF @@ -678,10 +709,7 @@ def set_control(self, data): if self.control.lcd_enabled != bool(data & 0x80): self.reset_control(data) - # don't draw window if it was not enabled and not being drawn before - if not self.control.window_enabled and (data & 0x20) != 0 and \ - self.window.line_y == 0 and self.line_y > self.window.y: - self.window.line_y = 144 + self.window.update_line_y(data) self.control.write(data) def reset_control(self, data): @@ -861,6 +889,11 @@ self.window.x = data def set_oam(self, address, data): + """ + sets one byte of the object attribute memory. + The object attribute memory stores the position and seme other + attributes of the sprites + """ self.oam[address - constants.OAM_ADDR] = data & 0xFF #self.update_sprites(address) self.update_sprites() @@ -869,6 +902,10 @@ return self.oam[address - constants.OAM_ADDR] def set_vram(self, address, data): + """ + sets one byte of the video memory. + The video memroy contains the tiles used to disply. + """ self.vram[address - constants.VRAM_ADDR] = data & 0xFF self.update_tiles() @@ -898,14 +935,14 @@ self.driver.update_display() def draw_line(self): - if self.control.background_enabled: + if self.background.enabled: self.draw_background() else: self.draw_clean_background() - if self.control.window_enabled: + if self.window.enabled: self.draw_window() - if self.control.sprite_display_enabled: - self.draw_objects() + if self.control.sprites_enabled: + self.draw_sprites() self.draw_pixels() def draw_clean_background(self): @@ -919,9 +956,9 @@ self.draw_tiles(8 - (x & 7), tile_map, tile_data) def prepare_background_data(self, x, y): - tile_map = self.get_tile_map(0x08) + tile_map = self.control.get_selected_tile_map_space() tile_map += ((y >> 3) << 5) + (x >> 3) - tile_data = self.get_tile_data(0x10) + tile_data = self.control.get_selected_tile_map_space() tile_data += (y & 7) << 1 return tile_map, tile_data @@ -935,26 +972,15 @@ self.window.line_y += 1 def prepare_window_data(self): - tile_map = self.get_tile_map(0x40) + tile_map = self.control.get_tile_map_combined() tile_map += (self.window.line_y >> 3) << 5 - tile_data = self.get_tile_data(0x10) + tile_data = self.control.get_selected_tile_map_space() tile_data += (self.window.line_y & 7) << 1 return tile_map, tile_data; - def get_tile_map(self, mask): - if (self.control.read() & mask) != 0: - return constants.VRAM_MAP_B - else: - return constants.VRAM_MAP_A - - def get_tile_data(self, mask): - if (self.control.read() & mask) != 0: - return constants.VRAM_DATA_A - else: - return constants.VRAM_DATA_B - def draw_objects(self): - count = self.scan_objects() + def draw_sprites(self): + count = self.scan_sprites() lastx = 176 for index in range(176, count): data = self.objects[index] @@ -967,7 +993,7 @@ self.draw_overlapped_object_tile(x, address, flags) lastx = x - def scan_objects(self): + def scan_sprites(self): count = 0 # search active objects for offset in range(0, 4*40, 4): @@ -998,10 +1024,10 @@ count += 1 if count >= constants.OBJECTS_PER_LINE: break - self.sort_scan_object(count) + self.sort_scan_sprite(count) return count - def sort_scan_object(self, count): + def sort_scan_sprite(self, count): # sort objects from lower to higher priority for index in range(count): rightmost = index From pedronis at codespeak.net Fri Sep 12 11:12:56 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 12 Sep 2008 11:12:56 +0200 (CEST) Subject: [pypy-svn] r58077 - in pypy/branch/pypy-pytrunk/pypy/tool/pytest: . test Message-ID: <20080912091256.5772A16A77B@codespeak.net> Author: pedronis Date: Fri Sep 12 11:12:54 2008 New Revision: 58077 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Log: (iko, pedronis) make the test more robust by faking less, simpler but naive for now implementation of paths for items Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py Fri Sep 12 11:12:54 2008 @@ -1,6 +1,11 @@ from py.__.test.session import Session from py.__.test import event + +def generic_path(item): + names = item.listnames() + return '.'.join(names).replace('.(', '(') + class FileLogSession(Session): def __init__(self, config): @@ -14,10 +19,6 @@ def log_event_to_file(self, ev): if isinstance(ev, event.ItemTestReport): outcome = ev.outcome - metainfo = ev.colitem.repr_metainfo() - path = metainfo.fspath - modpath = metainfo.modpath - if modpath: - path += ":%s" % modpath - print >>self.logfile, "%s %s" % (outcome.shortrepr, path) + gpath = generic_path(ev.colitem) + print >>self.logfile, "%s %s" % (outcome.shortrepr, gpath) Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Fri Sep 12 11:12:54 2008 @@ -1,7 +1,11 @@ +import py + from pypy.tool.pytest import filelog import os, StringIO +from py.__.test.collect import Node, Item from py.__.test.event import ItemTestReport +from py.__.test.runner import OutcomeRepr class Fake(object): @@ -9,6 +13,23 @@ self.__dict__.update(kwds) +def test_generic_path(): + p1 = Node('a', config='dummy') + p2 = Node('B', parent=p1) + p3 = Node('()', parent = p2) + item = Item('c', parent = p3) + + res = filelog.generic_path(item) + assert res == 'a.B().c' + + +def make_item(*names): + node = None + config = "dummy" + for name in names[:-1]: + node = Node(name, parent=node, config=config) + return Item(names[-1], parent=node) + class TestFileLogSession(object): @@ -34,15 +55,15 @@ sess.logfile.close() os.unlink(logfname) - def test_item_test_passed_or_skipped(self): + def test_item_test_passed(self): option = Fake(eventlog=None) config = Fake(option=option) sess = filelog.FileLogSession(config) sess.logfile = StringIO.StringIO() - colitem = Fake(repr_metainfo=lambda: Fake(fspath='some/path', - modpath="a.b")) - outcome=Fake(shortrepr='.') + colitem = make_item('some', 'path', 'a', 'b') + + outcome=OutcomeRepr('execute', '.', '') rep_ev = ItemTestReport(colitem, outcome=outcome) sess.bus.notify(rep_ev) @@ -51,12 +72,16 @@ assert len(lines) == 1 line = lines[0] assert line.startswith(". ") - assert line[2:] == 'some/path:a.b' + assert line[2:] == 'some.path.a.b' + + def test_item_test_skipped(self): + py.test.skip("WIP: take the longrepr into account") + option = Fake(eventlog=None) + config = Fake(option=option) + sess = filelog.FileLogSession(config) sess.logfile = StringIO.StringIO() - colitem = Fake(repr_metainfo=lambda: Fake(fspath='some/path', - modpath=None)) - outcome=Fake(shortrepr='s') + outcome=OutcomeRepr('execute', 's', '') rep_ev = ItemTestReport(colitem, outcome=outcome) sess.bus.notify(rep_ev) @@ -66,7 +91,3 @@ line = lines[0] assert line.startswith("s ") - assert line[2:] == 'some/path' - - -# XXX integration tests From pedronis at codespeak.net Fri Sep 12 12:35:56 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 12 Sep 2008 12:35:56 +0200 (CEST) Subject: [pypy-svn] r58078 - in pypy/branch/pypy-pytrunk/pypy/tool/pytest: . test Message-ID: <20080912103556.5E54416A75C@codespeak.net> Author: pedronis Date: Fri Sep 12 12:35:54 2008 New Revision: 58078 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Log: (iko, pedronis) - reorganize the code by using helpers with tests - cover CollectionReport and InternalExceptions - armor longreprs by starting each line with a whitespace - all lines in the log terminates with '\n' Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py Fri Sep 12 12:35:54 2008 @@ -15,10 +15,25 @@ filelog = config.option.filelog self.logfile = open(filelog, 'w') # line buffering ? + def write_log_entry(self, shortrepr, name, longrepr): + print >>self.logfile, "%s %s" % (shortrepr, name) + for line in longrepr.splitlines(): + print >>self.logfile, " %s" % line + + def log_outcome(self, ev): + outcome = ev.outcome + gpath = generic_path(ev.colitem) + self.write_log_entry(outcome.shortrepr, gpath, str(outcome.longrepr)) def log_event_to_file(self, ev): if isinstance(ev, event.ItemTestReport): - outcome = ev.outcome - gpath = generic_path(ev.colitem) - print >>self.logfile, "%s %s" % (outcome.shortrepr, gpath) + self.log_outcome(ev) + elif isinstance(ev, event.CollectionReport): + if not ev.passed: + self.log_outcome(ev) + elif isinstance(ev, event.InternalException): + path = ev.repr.reprcrash.path # fishing :( + self.write_log_entry('!', path, str(ev.repr)) + + Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Fri Sep 12 12:35:54 2008 @@ -4,7 +4,8 @@ import os, StringIO from py.__.test.collect import Node, Item -from py.__.test.event import ItemTestReport +from py.__.test.event import ItemTestReport, CollectionReport +from py.__.test.event import InternalException from py.__.test.runner import OutcomeRepr @@ -55,6 +56,72 @@ sess.logfile.close() os.unlink(logfname) + def test_write_log_entry(self): + option = Fake(eventlog=None) + config = Fake(option=option) + sess = filelog.FileLogSession(config) + + sess.logfile = StringIO.StringIO() + sess.write_log_entry('.', 'name', '') + entry = sess.logfile.getvalue() + assert entry[-1] == '\n' + entry_lines = entry.splitlines() + assert len(entry_lines) == 1 + assert entry_lines[0] == '. name' + + sess.logfile = StringIO.StringIO() + sess.write_log_entry('s', 'name', 'Skipped') + entry = sess.logfile.getvalue() + assert entry[-1] == '\n' + entry_lines = entry.splitlines() + assert len(entry_lines) == 2 + assert entry_lines[0] == 's name' + assert entry_lines[1] == ' Skipped' + + sess.logfile = StringIO.StringIO() + sess.write_log_entry('s', 'name', 'Skipped\n') + entry = sess.logfile.getvalue() + assert entry[-1] == '\n' + entry_lines = entry.splitlines() + assert len(entry_lines) == 2 + assert entry_lines[0] == 's name' + assert entry_lines[1] == ' Skipped' + + sess.logfile = StringIO.StringIO() + longrepr = ' tb1\n tb 2\nE tb3\nSome Error' + sess.write_log_entry('F', 'name', longrepr) + entry = sess.logfile.getvalue() + assert entry[-1] == '\n' + entry_lines = entry.splitlines() + assert len(entry_lines) == 5 + assert entry_lines[0] == 'F name' + assert entry_lines[1:] == [' '+line for line in longrepr.splitlines()] + + def test_log_outcome(self): + option = Fake(eventlog=None) + config = Fake(option=option) + sess = filelog.FileLogSession(config) + + sess.logfile = StringIO.StringIO() + colitem = make_item('some', 'path', 'a', 'b') + + try: + raise ValueError + except ValueError: + the_repr = py.code.ExceptionInfo().getrepr() + + outcome=OutcomeRepr('execute', 'F', the_repr) + ev = Fake(colitem=colitem, outcome=outcome) + + sess.log_outcome(ev) + + entry = sess.logfile.getvalue() + entry_lines = entry.splitlines() + + assert entry_lines[0] == 'F some.path.a.b' + assert entry_lines[-1][0] == ' ' + assert 'ValueError' in entry + def test_item_test_passed(self): option = Fake(eventlog=None) config = Fake(option=option) @@ -64,7 +131,7 @@ colitem = make_item('some', 'path', 'a', 'b') outcome=OutcomeRepr('execute', '.', '') - rep_ev = ItemTestReport(colitem, outcome=outcome) + rep_ev = ItemTestReport(colitem, passed=outcome) sess.bus.notify(rep_ev) @@ -74,20 +141,53 @@ assert line.startswith(". ") assert line[2:] == 'some.path.a.b' - - def test_item_test_skipped(self): - py.test.skip("WIP: take the longrepr into account") + def test_collection_report(self): option = Fake(eventlog=None) config = Fake(option=option) sess = filelog.FileLogSession(config) sess.logfile = StringIO.StringIO() - outcome=OutcomeRepr('execute', 's', '') - rep_ev = ItemTestReport(colitem, outcome=outcome) + + colitem = make_item('some', 'path') + + outcome=OutcomeRepr('execute', '', '') + rep_ev = CollectionReport(colitem, object(), passed=outcome) sess.bus.notify(rep_ev) - lines = sess.logfile.getvalue().splitlines() - assert len(lines) == 1 - line = lines[0] + entry = sess.logfile.getvalue() + assert not entry + + sess.logfile = StringIO.StringIO() + outcome=OutcomeRepr('execute', 'F', 'Some Error') + rep_ev = CollectionReport(colitem, object(), failed=outcome) + + sess.bus.notify(rep_ev) + + lines = sess.logfile.getvalue().splitlines() + assert len(lines) == 2 + assert lines[0] == 'F some.path' + + def test_internal_exception(self): + # they are produced for example by a teardown failing + # at the end of the run + option = Fake(eventlog=None) + config = Fake(option=option) + sess = filelog.FileLogSession(config) + sess.logfile = StringIO.StringIO() - assert line.startswith("s ") + try: + raise ValueError + except ValueError: + excinfo = py.code.ExceptionInfo() + + internal = InternalException(excinfo) + + sess.bus.notify(internal) + + entry = sess.logfile.getvalue() + entry_lines = entry.splitlines() + + assert entry_lines[0].startswith('! ') + assert os.path.basename(__file__)[:-1] in entry_lines[0] #.py/.pyc + assert entry_lines[-1][0] == ' ' + assert 'ValueError' in entry From cfbolz at codespeak.net Fri Sep 12 13:37:17 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 12 Sep 2008 13:37:17 +0200 (CEST) Subject: [pypy-svn] r58079 - pypy/dist/pypy/lang/smalltalk Message-ID: <20080912113717.E192316A75E@codespeak.net> Author: cfbolz Date: Fri Sep 12 13:37:15 2008 New Revision: 58079 Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py Log: some comments in the squeak image loader Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/dist/pypy/lang/smalltalk/squeakimage.py Fri Sep 12 13:37:15 2008 @@ -69,21 +69,29 @@ # ____________________________________________________________ class ImageReader(object): + + def __init__(self, space, stream): self.space = space self.stream = stream + # dictionary mapping old address to chunk object self.chunks = {} self.chunklist = [] + # cache wrapper integers + self.intcache = {} def initialize(self): + # XXX should be called something like read_full_image self.read_header() self.read_body() self.init_compactclassesarray() + # until here, the chunks are generated self.init_g_objects() self.init_w_objects() self.fillin_w_objects() def read_header(self): + # 1 word version version = self.stream.peek() if version != 0x1966: self.stream.swap = True @@ -92,14 +100,20 @@ raise CorruptImageError version = self.stream.next() #------ + # 1 word headersize headersize = self.stream.next() + # 1 word size of the full image self.endofmemory = self.stream.next() # endofmemory = bodysize + # 1 word old base address self.oldbaseaddress = self.stream.next() + # 1 word pointer to special objects array self.specialobjectspointer = self.stream.next() + # 1 word last used hash lasthash = self.stream.next() savedwindowssize = self.stream.next() fullscreenflag = self.stream.next() extravmmemory = self.stream.next() + # we called 9 times next, 1 word = 4 byte self.stream.skipbytes(headersize - (9 * 4)) def read_body(self): @@ -143,10 +157,11 @@ chunk.g_object.fillin_w_object() def init_compactclassesarray(self): - """ (CompiledMethod Symbol Array PseudoContext LargePositiveInteger nil MethodDictionary Association Point Rectangle nil TranslatedMethod BlockContext MethodContext nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil ) """ + """ from the blue book (CompiledMethod Symbol Array PseudoContext LargePositiveInteger nil MethodDictionary Association Point Rectangle nil TranslatedMethod BlockContext MethodContext nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil ) """ special = self.chunks[self.specialobjectspointer] assert special.size > 24 #at least assert special.format == 2 + # squeak-specific: compact classes array chunk = self.chunks[special.data[COMPACT_CLASSES_ARRAY]] assert len(chunk.data) == 31 assert chunk.format == 2 @@ -206,6 +221,9 @@ def special(self, index): return self.special_objects[index] +# from the squeak source code: +# in squeak, the compact classes array can be found at this position +# in the special objects array COMPACT_CLASSES_ARRAY = 28 # ____________________________________________________________ @@ -216,21 +234,27 @@ GenericObject from the image chunks, and uses them as starting point for the actual create of pypy.lang.smalltalk.model classes. """ + def __init__(self, space): self.space = space - self.owner = None + self.reader = None def isinitialized(self): - return self.owner is not None + return self.reader is not None def initialize_int(self, value, reader): - self.owner = reader + self.reader = reader self.value = value self.size = -1 - self.w_object = self.space.wrap_int(value) + if value in reader.intcache: + w_int = reader.intcache[value] + else: + w_int = self.space.wrap_int(value) + reader.intcache[value] = w_int + self.w_object = w_int def initialize(self, chunk, reader): - self.owner = reader + self.reader = reader self.size = chunk.size self.hash12 = chunk.hash12 self.format = chunk.format @@ -241,10 +265,10 @@ def init_class(self, chunk): if chunk.iscompact(): - self.g_class = self.owner.compactclasses[chunk.classid + self.g_class = self.reader.compactclasses[chunk.classid - 1].g_object # Smalltalk is 1-based indexed else: - self.g_class = self.owner.chunks[chunk.classid].g_object + self.g_class = self.reader.chunks[chunk.classid].g_object def init_data(self, chunk): if not self.ispointers(): return @@ -255,10 +279,10 @@ def decode_pointer(self, pointer): if (pointer & 1) == 1: small_int = GenericObject(self.space) - small_int.initialize_int(pointer >> 1, self.owner) + small_int.initialize_int(pointer >> 1, self.reader) return small_int else: - return self.owner.chunks[pointer].g_object + return self.reader.chunks[pointer].g_object def isbytes(self): return 8 <= self.format <= 11 @@ -286,6 +310,8 @@ followed by indexable bytes (same interpretation of low 2 bits as above) """ if self.w_object is None: + # the instantiate call circumvents the constructors + # and makes empty objects if self.format < 5: # XXX self.format == 4 is weak self.w_object = objectmodel.instantiate(model.W_PointersObject) @@ -322,6 +348,7 @@ def fillin_pointersobject(self, w_pointersobject): assert self.pointers is not None + # XXX is the following needed? if w_pointersobject._shadow is not None: w_pointersobject._shadow.detach_shadow() w_pointersobject._vars = [g_object.w_object for g_object in self.pointers] @@ -345,7 +372,7 @@ w_bytesobject.hash = self.chunk.hash12 # XXX check this def get_bytes(self): bytes = [] - if self.owner.swap: + if self.reader.swap: for each in self.chunk.data: bytes.append(chr((each >> 0) & 0xff)) bytes.append(chr((each >> 8) & 0xff)) @@ -373,11 +400,14 @@ w_compiledmethod.bytes = ''.join(bbytes) class ImageChunk(object): + """ A chunk knows the information from the header, but the body of the + object is not decoded yet.""" def __init__(self, space, size, format, classid, hash12): self.size = size self.format = format self.classid = classid self.hash12 = hash12 + # list of integers forming the body of the object self.data = None self.g_object = GenericObject(space) From pedronis at codespeak.net Fri Sep 12 14:25:14 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 12 Sep 2008 14:25:14 +0200 (CEST) Subject: [pypy-svn] r58080 - in pypy/branch/pypy-pytrunk/pypy/tool/pytest: . test Message-ID: <20080912122514.BFD4916A70B@codespeak.net> Author: pedronis Date: Fri Sep 12 14:25:11 2008 New Revision: 58080 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Log: (iko, pedronis) use an information richer format for the test paths used in the log Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py Fri Sep 12 14:25:11 2008 @@ -3,8 +3,27 @@ def generic_path(item): - names = item.listnames() - return '.'.join(names).replace('.(', '(') + chain = item.listchain() + gpath = [chain[0].name] + fspath = chain[0].fspath + fspart = False + for node in chain[1:]: + newfspath = node.fspath + if newfspath == fspath: + if fspart: + gpath.append(':') + fspart = False + else: + gpath.append('.') + else: + gpath.append('/') + fspart = True + name = node.name + if name[0] in '([': + gpath.pop() + gpath.append(name) + fspath = newfspath + return ''.join(gpath) class FileLogSession(Session): Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py (original) +++ pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Fri Sep 12 14:25:11 2008 @@ -3,7 +3,7 @@ from pypy.tool.pytest import filelog import os, StringIO -from py.__.test.collect import Node, Item +from py.__.test.collect import Node, Item, FSCollector from py.__.test.event import ItemTestReport, CollectionReport from py.__.test.event import InternalException from py.__.test.runner import OutcomeRepr @@ -16,6 +16,7 @@ def test_generic_path(): p1 = Node('a', config='dummy') + assert p1.fspath is None p2 = Node('B', parent=p1) p3 = Node('()', parent = p2) item = Item('c', parent = p3) @@ -23,12 +24,27 @@ res = filelog.generic_path(item) assert res == 'a.B().c' + p0 = FSCollector('proj/test', config='dummy') + p1 = FSCollector('proj/test/a', parent=p0) + p2 = Node('B', parent=p1) + p3 = Node('()', parent = p2) + p4 = Node('c', parent=p3) + item = Item('[1]', parent = p4) + + res = filelog.generic_path(item) + assert res == 'test/a:B().c[1]' + def make_item(*names): node = None config = "dummy" for name in names[:-1]: - node = Node(name, parent=node, config=config) + if '/' in name: + node = FSCollector(name, parent=node, config=config) + else: + node = Node(name, parent=node, config=config) + if names[-1] is None: + return node return Item(names[-1], parent=node) class TestFileLogSession(object): @@ -128,7 +144,7 @@ sess = filelog.FileLogSession(config) sess.logfile = StringIO.StringIO() - colitem = make_item('some', 'path', 'a', 'b') + colitem = make_item('proj/test', 'proj/test/mod', 'a', 'b') outcome=OutcomeRepr('execute', '.', '') rep_ev = ItemTestReport(colitem, passed=outcome) @@ -139,7 +155,7 @@ assert len(lines) == 1 line = lines[0] assert line.startswith(". ") - assert line[2:] == 'some.path.a.b' + assert line[2:] == 'test/mod:a.b' def test_collection_report(self): option = Fake(eventlog=None) @@ -147,7 +163,7 @@ sess = filelog.FileLogSession(config) sess.logfile = StringIO.StringIO() - colitem = make_item('some', 'path') + colitem = make_item('proj/test', 'proj/test/mod', 'A', None) outcome=OutcomeRepr('execute', '', '') rep_ev = CollectionReport(colitem, object(), passed=outcome) @@ -165,7 +181,7 @@ lines = sess.logfile.getvalue().splitlines() assert len(lines) == 2 - assert lines[0] == 'F some.path' + assert lines[0] == 'F test/mod:A' def test_internal_exception(self): # they are produced for example by a teardown failing From exarkun at codespeak.net Fri Sep 12 16:35:09 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 12 Sep 2008 16:35:09 +0200 (CEST) Subject: [pypy-svn] r58082 - pypy/build/bot Message-ID: <20080912143509.48E8416A7C6@codespeak.net> Author: exarkun Date: Fri Sep 12 16:35:07 2008 New Revision: 58082 Modified: pypy/build/bot/pypybuilders.py Log: disable FileUpload until it works Modified: pypy/build/bot/pypybuilders.py ============================================================================== --- pypy/build/bot/pypybuilders.py (original) +++ pypy/build/bot/pypybuilders.py Fri Sep 12 16:35:07 2008 @@ -139,10 +139,10 @@ translationArgs=translationArguments, targetArgs=targetArguments) - uploadSuffix = "".join(translationArguments + targetArguments) - self.addStep(FileUpload, - slavesrc="pypy-src/pypy/translator/goal/" + self.executable, - masterdest="public_html/builds/" + self.executable + "-" + uploadSuffix) +# uploadSuffix = "".join(translationArguments + targetArguments) +# self.addStep(FileUpload, +# slavesrc="pypy-src/pypy/translator/goal/" + self.executable, +# masterdest="public_html/builds/" + self.executable + "-" + uploadSuffix) # if pytestArguments is not None: # self.addStep( From exarkun at codespeak.net Fri Sep 12 16:35:34 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 12 Sep 2008 16:35:34 +0200 (CEST) Subject: [pypy-svn] r58083 - pypy/build/bot Message-ID: <20080912143534.6912516A7D3@codespeak.net> Author: exarkun Date: Fri Sep 12 16:35:33 2008 New Revision: 58083 Added: pypy/build/bot/django-settings.py Modified: pypy/build/bot/master.cfg pypy/build/bot/netstring_conftest.py pypy/build/bot/per-directory-py-test.py Log: move slaves from charm to bigdogvm1 Added: pypy/build/bot/django-settings.py ============================================================================== --- (empty file) +++ pypy/build/bot/django-settings.py Fri Sep 12 16:35:33 2008 @@ -0,0 +1,2 @@ +ROOT_URLCONF="" +DATABASE_ENGINE="sqlite3" Modified: pypy/build/bot/master.cfg ============================================================================== --- pypy/build/bot/master.cfg (original) +++ pypy/build/bot/master.cfg Fri Sep 12 16:35:33 2008 @@ -26,8 +26,9 @@ 'change_source': [], 'schedulers': [Nightly("nightly", [ - "pypy-c-allworkingmodules-32", - "pypy-c-allworkingmodules-O3-32", + "pypy-c-32-O0", + "pypy-c-32-O2", + "pypy-c-32-O3", "pypy-c-allworkingmodules-O3-64", "pypy-c-allworkingmodules-winxp32"], hour=19)], @@ -38,30 +39,35 @@ in passwords.iteritems()], 'builders': [ - {"name": "pypy-c-allworkingmodules-32", - "slavenames": ["charm"], - "builddir": "pypy-c-allworkingmodules-32", + {"name": "pypy-c-32-O0", + "slavenames": ["bigdogvm1"], + "builddir": "pypy-c-32-O0", "factory": POSIXPyPyBuildFactory( ["--boxed"], ["-O0"], [])}, - {"name": "pypy-c-allworkingmodules-O3-32", - "slavenames": ["charm"], - "builddir": "pypy-c-allworkingmodules-faassen-32", + {"name": "pypy-c-32-O2", + "slavenames": ["bigdogvm1"], + "builddir": "pypy-c-32-O2", "factory": POSIXPyPyBuildFactory( - None, ["-O3", "--gcrootfinder=asmgcc"], - [])}, + None, ["-O2"], [], django=True)}, + + {"name": "pypy-c-32-O3", + "slavenames": ["bigdogvm1"], + "builddir": "pypy-c-32-O3", + "factory": POSIXPyPyBuildFactory( + None, ["-O3"], [])}, {"name": "pypy-c-allworkingmodules-O3-64", "slavenames": ["linux-dvs0"], "builddir": "pypy-c-allworkingmodules-faassen-64", "factory": POSIXPyPyBuildFactory( - ["--boxed"], ["-O3"], [], django=True)}, + ["--boxed"], ["-O3"], [])}, {"name": "pypy-c-allworkingmodules-winxp32", "slavenames": ["winxp32-py2.5"], "builddir": "pypy-c-allworkingmodules-faassen-winxp32", "factory": WindowsPyPyBuildFactory( - ["--boxed"], ["-O0"], [], django=True)}, + ["--boxed"], ["-O0"], [])}, ], 'buildbotURL': 'http://office.divmod.com:%d/' % (httpPortNumber,), Modified: pypy/build/bot/netstring_conftest.py ============================================================================== --- pypy/build/bot/netstring_conftest.py (original) +++ pypy/build/bot/netstring_conftest.py Fri Sep 12 16:35:33 2008 @@ -8,11 +8,11 @@ from twisted.spread.jelly import jelly import cPickle -from bot.netstring import netstring +from netstring import netstring from StringIO import StringIO def itemrepr(item): - return (item.__class__.__name__, item.listnames()) + return (item.__class__.__name__, item.listnames()) class MyReporter(AbstractReporter): def __init__(self, *args, **kwds): Modified: pypy/build/bot/per-directory-py-test.py ============================================================================== --- pypy/build/bot/per-directory-py-test.py (original) +++ pypy/build/bot/per-directory-py-test.py Fri Sep 12 16:35:33 2008 @@ -5,9 +5,12 @@ def runPyPyTests(pythonExe, rootDir, args): command = [pythonExe, r"py\bin\py.test"] + list(args) + print 'Walking', repr(rootDir) for rootDir, dirs, files in os.walk(rootDir): + print 'Found', repr(rootDir), repr(dirs), repr(files) if "test" in dirs: testDir = os.path.join(rootDir, "test") + print 'Running', repr(command + [testDir]) subprocess.call(command + [testDir]) def main(): From arigo at codespeak.net Fri Sep 12 19:07:10 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Sep 2008 19:07:10 +0200 (CEST) Subject: [pypy-svn] r58085 - pypy/build/doc Message-ID: <20080912170710.EBB2C16A173@codespeak.net> Author: arigo Date: Fri Sep 12 19:07:07 2008 New Revision: 58085 Modified: pypy/build/doc/benchmark_memory.txt Log: Look at smaps for a pypy-c-hybric. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Fri Sep 12 19:07:07 2008 @@ -118,6 +118,9 @@ XXX please review, correct, complete so we get to a shared good understanding +CPython ++++++++++++++++ + The most detailed info is provided by /proc/PID/smaps, starting from linux 2.6.14. Here is an example output of running "python2.5" on a linux 2.6.24 ubuntu machine:: @@ -192,6 +195,77 @@ For understanding how swapping and linux memory management works here is a nice read: http://sourcefrog.net/weblog/software/linux-kernel/swap.html +pypy-c +++++++++++++++++++++++++++++ + +The same with a pypy-c process (using the hybrid GC) shows many mostly +small mappings, and four big ones. They are:: + + 08048000-084d2000 r-xp 00000000 03:03 2105442 /path/to/pypy-c + Size: 4648 kB + Rss: 2076 kB + Shared_Clean: 2076 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 0 kB + +This is the code section. It is mapped read-only from the disk. We see +that the code section is 4.6MB in size, of which 2MB have been loaded so +far. These 2MB are Clean, so if the system runs out of memory they can +be simply discarded, and later reloaded on demand from the executable +(*). The 2MB are also Shared, so several pypy-c processes share them +with each other (*). + +I am using (*) to indicate "likely theories" that still need +confirmation. + +Second mapping:: + + 084d3000-088f6000 rw-p 0048a000 03:03 2105442 /path/to/pypy-c + Size: 4236 kB + Rss: 1516 kB + Shared_Clean: 1232 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 284 kB + +It's the data section of the executable. 1.5MB have been touched so far +and loaded from the disk. Out of these, 1.2MB are Clean and Shared +because they haven't been modified at all, and 0.3MB are Private and +Dirty because the data was modified in this process. + +Third mapping:: + + 088f6000-08d9b000 rw-p 088f6000 00:00 0 [heap] + Size: 4756 kB + Rss: 4680 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 4680 kB + +This one I have no clue about. It's probably malloc-ed memory but I +don't really understand where it comes from. + +Fourth mapping:: + + b6c3f000-b7d24000 rw-p b6c3f000 00:00 0 + Size: 17300 kB + Rss: 17104 kB + Shared_Clean: 14308 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 2796 kB + +This is the GC heap, for the framework GC. It is a private mmap +initialized by reading from /dev/zero. The total size is 17MB. Of +these, 14MB are Shared and Clean because they are pages full of zeroes. +These 14MB don't consume any RAM anywhere; I suspect that Linux +implements them by having only 4KB of zeroes in a corner of the kernel, +and making all the pages in these 14MB shared with this single page. +The remaining 2.8MB are Private and Dirty because they contain +GC-managed objects. + Tool to measure python interpreter mem foot print ------------------------------------------------------- From arigo at codespeak.net Fri Sep 12 19:26:19 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Sep 2008 19:26:19 +0200 (CEST) Subject: [pypy-svn] r58086 - pypy/build/doc Message-ID: <20080912172619.D5AD816A644@codespeak.net> Author: arigo Date: Fri Sep 12 19:26:18 2008 New Revision: 58086 Modified: pypy/build/doc/benchmark_memory.txt Log: Got a clue. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Fri Sep 12 19:26:18 2008 @@ -244,8 +244,10 @@ Private_Clean: 0 kB Private_Dirty: 4680 kB -This one I have no clue about. It's probably malloc-ed memory but I -don't really understand where it comes from. +This is malloc-ed memory. I am not sure but I think that it +contains mostly the long-lived objects of the hybrid GC. We +should first try to look at a pypy-c using the generation GC; +the generation GC doesn't call malloc. Fourth mapping:: From arigo at codespeak.net Fri Sep 12 19:56:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Sep 2008 19:56:40 +0200 (CEST) Subject: [pypy-svn] r58087 - pypy/build/doc Message-ID: <20080912175640.174FE16A775@codespeak.net> Author: arigo Date: Fri Sep 12 19:56:38 2008 New Revision: 58087 Modified: pypy/build/doc/benchmark_memory.txt Log: Confirmed. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Fri Sep 12 19:56:38 2008 @@ -212,12 +212,9 @@ This is the code section. It is mapped read-only from the disk. We see that the code section is 4.6MB in size, of which 2MB have been loaded so far. These 2MB are Clean, so if the system runs out of memory they can -be simply discarded, and later reloaded on demand from the executable -(*). The 2MB are also Shared, so several pypy-c processes share them -with each other (*). - -I am using (*) to indicate "likely theories" that still need -confirmation. +be simply discarded, and later reloaded on demand from the executable. +The 2MB are also Shared, so several pypy-c processes share them +with each other. Second mapping:: From arigo at codespeak.net Fri Sep 12 20:02:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Sep 2008 20:02:14 +0200 (CEST) Subject: [pypy-svn] r58088 - pypy/build/doc Message-ID: <20080912180214.AA92016A5FA@codespeak.net> Author: arigo Date: Fri Sep 12 20:02:10 2008 New Revision: 58088 Modified: pypy/build/doc/benchmark_memory.txt Log: Summarize the findings. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Fri Sep 12 20:02:10 2008 @@ -265,6 +265,15 @@ The remaining 2.8MB are Private and Dirty because they contain GC-managed objects. +In summary (and as always assuming the OS is Linux), 5 +independently-started pypy-c processes consume:: + + 1 * size(code section) + + 1 * size(data section) + + 4 * size(modified data in the data section) + + 5 * size(malloc heap) + + 5 * size(framework GC heap in use) + Tool to measure python interpreter mem foot print ------------------------------------------------------- From arigo at codespeak.net Fri Sep 12 20:04:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Sep 2008 20:04:53 +0200 (CEST) Subject: [pypy-svn] r58089 - pypy/build/doc Message-ID: <20080912180453.8313316A5FA@codespeak.net> Author: arigo Date: Fri Sep 12 20:04:47 2008 New Revision: 58089 Modified: pypy/build/doc/benchmark_memory.txt Log: Clarify. Modified: pypy/build/doc/benchmark_memory.txt ============================================================================== --- pypy/build/doc/benchmark_memory.txt (original) +++ pypy/build/doc/benchmark_memory.txt Fri Sep 12 20:04:47 2008 @@ -268,9 +268,9 @@ In summary (and as always assuming the OS is Linux), 5 independently-started pypy-c processes consume:: - 1 * size(code section) + - 1 * size(data section) + - 4 * size(modified data in the data section) + + 1 * size(code section accessed so far) + + 1 * size(data section accessed and not modified so far) + + 5 * size(data section modified) + 5 * size(malloc heap) + 5 * size(framework GC heap in use) From hpk at codespeak.net Sat Sep 13 10:02:55 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 13 Sep 2008 10:02:55 +0200 (CEST) Subject: [pypy-svn] r58098 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080913080255.47D0816A3CC@codespeak.net> Author: hpk Date: Sat Sep 13 10:02:49 2008 New Revision: 58098 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: some updates after discussion with maciej. Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 13 10:02:49 2008 @@ -31,21 +31,13 @@ * official way to have bindings to external (C) libraries for PyPy -* slow, but getting better - * can handle i.e. pysqlite-ctypes, pyglet, pymunk or Sole Scion -* ctypes is getting better as a side effect: - - * errno handling - - * bugfixes - - * helper libraries +* has better errno handling, bugfixes * part of google sponsoring -* 32bit only by now +* note: 32bit and a bit slow CTypes configure ================ @@ -104,9 +96,12 @@ * http://twistedmatrix.com/ + Other software ============== +* pure python should just work + * BitTorrent * PyPy translation toolchain @@ -115,8 +110,7 @@ * sympy -* various smaller things, templating engines, - most pure-python software +* various smaller things, templating engines Obscure details that people rely on =================================== @@ -189,6 +183,17 @@ * psyco is a nice proof that this approach would work +Threading / Stackless +=================================== + +* currently using GIL, quite robust + +* free threading? "it's work" + +* pypy-c has software threading / stackless + +* added during translation + Other backends ============== @@ -232,10 +237,9 @@ - compatible to Python 2.5.2 - well tested on win/linux 32 bit -- os/x and 64 also targets -- running some major packages unmodified +- running major packages unmodified - easy_install/distutils working -- deliver pypy-c binary installs windows +- help e.g. by writing ctypes modules Contact / Q&A ========================== From fijal at codespeak.net Sat Sep 13 10:44:51 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 13 Sep 2008 10:44:51 +0200 (CEST) Subject: [pypy-svn] r58099 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080913084451.C286816A37E@codespeak.net> Author: fijal Date: Sat Sep 13 10:44:50 2008 New Revision: 58099 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: A bit of update Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 13 10:44:50 2008 @@ -5,12 +5,38 @@ What this talk is about ======================= +* a bit of motivation + * tell what Python Interpreter can run today * what we are heading for with the 1.1 release * Questions and Answers +PyPy - motivation +================================= + +* CPython is nice, but not flexible enough + +* IronPython, Jython - bound to the specific VM + +* Separate language specification from low-level details, + such as GC or platform to run + +* Psyco and Stackless Python hard to maintain + +PyPy - user motivation +======================= + +* One should never be forced to write anything in C + for performance reasons (with some exceptions: embedded + devices etc.) + +* Just-in-time compiler should make number-crunching + and static-enough code fast enough + +* One should never care about low-level details + Getting Production ready ========================== @@ -31,9 +57,11 @@ * official way to have bindings to external (C) libraries for PyPy -* can handle i.e. pysqlite-ctypes, pyglet, pymunk or Sole Scion +* can handle i.e. pysqlite-ctypes, pyglet, pymunk or Sole Scion, + almost whatever.... -* has better errno handling, bugfixes +* contribution to original ctypes + (better errno handling, bugfixes, tests...) * part of google sponsoring From hpk at codespeak.net Sat Sep 13 11:03:13 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 13 Sep 2008 11:03:13 +0200 (CEST) Subject: [pypy-svn] r58100 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080913090313.6318A16A335@codespeak.net> Author: hpk Date: Sat Sep 13 11:03:08 2008 New Revision: 58100 Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt Log: shift a bit, add history slide Modified: pypy/extradoc/talk/pycon-uk-2008/status/status.txt ============================================================================== --- pypy/extradoc/talk/pycon-uk-2008/status/status.txt (original) +++ pypy/extradoc/talk/pycon-uk-2008/status/status.txt Sat Sep 13 11:03:08 2008 @@ -37,6 +37,16 @@ * One should never care about low-level details +Brief history of PyPy +========================== + +* first sprint 2003, about 30 more by now +* CPython/Psyco/Jython/Stackless developers participating +* MIT-License, more sprints +* EU Research project 2004-2007 +* 2007-2008 - open source project +* some google sponsoring + Getting Production ready ========================== @@ -186,6 +196,18 @@ * pypy-c has fastest Interpreter startup +Speed - JIT generator +===================== + +* not ready yet! + +* will be super fast + +* some prototypes, research ongoing + +* psyco is a nice proof that this approach + would work + Memory - comparison with CPython =================================== @@ -199,17 +221,6 @@ * GCs are semi-decent -Speed - JIT generator -===================== - -* not ready yet! - -* will be super fast - -* some prototypes, research ongoing - -* psyco is a nice proof that this approach - would work Threading / Stackless =================================== From fijal at codespeak.net Sat Sep 13 11:05:08 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 13 Sep 2008 11:05:08 +0200 (CEST) Subject: [pypy-svn] r58101 - pypy/extradoc/talk/pycon-uk-2008/status Message-ID: <20080913090508.A7D6F16A356@codespeak.net> Author: fijal Date: Sat Sep 13 11:05:06 2008 New Revision: 58101 Added: pypy/extradoc/talk/pycon-uk-2008/status/status.pdf (contents, props changed) Log: add a pdf Added: pypy/extradoc/talk/pycon-uk-2008/status/status.pdf ============================================================================== Binary file. No diff available. From nshepperd at codespeak.net Sat Sep 13 12:19:39 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Sat, 13 Sep 2008 12:19:39 +0200 (CEST) Subject: [pypy-svn] r58102 - in pypy/branch/new-platformcheck/pypy/rpython/tool: . test Message-ID: <20080913101939.AB24516A360@codespeak.net> Author: nshepperd Date: Sat Sep 13 12:19:37 2008 New Revision: 58102 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Log: Implemented beginnings of support for constants defined as external symbols. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Sat Sep 13 12:19:37 2008 @@ -1,6 +1,9 @@ #! /usr/bin/env python import os, py, sys +from subprocess import Popen, PIPE +from tempfile import NamedTemporaryFile +from string import atoi from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import llmemory @@ -169,7 +172,7 @@ for key in dir(CConfig): value = getattr(CConfig, key) if isinstance(value, CConfigEntry): - entries.append((key, value)) + entries.append((key, value)) if entries: # can be empty if there are only CConfigSingleEntries infolist = [] @@ -208,6 +211,13 @@ writer = _CWriter(CConfig) writer.write_header() res[key] = value.question(writer.ask_gcc) + + for key in dir(CConfig): + value = getattr(CConfig, key) + if isinstance(value, CConfigExternEntry): + print (key, value) + value.eci = CConfig._compilation_info_ + res[key] = value.build_result() return res @@ -235,10 +245,6 @@ ''' % {'expr' : expr, 'beginpad' : beginpad.replace('\0', '\\0'), 'sizeof_beginpad' : len(beginpad), 'id' : filter(str.isalnum, self.key+'PLATCHECK'+name)} - def dumpbool(self, name, expr): - return self.dump(name, '((char)((%s)?1:0))' % (expr,)) - def bool(self, data): - return data[0] != '\x00' def dump_by_size(self, name, expr): beginpad = "__START_PLATCHECK_%s\0%s\0" % ( c_safe_string(self.key), @@ -256,7 +262,52 @@ ''' % {'expr' : expr, 'beginpad' : beginpad.replace('\0', '\\0'), 'sizeof_beginpad' : len(beginpad), 'id' : filter(str.isalnum, self.key+'PLATCHECK'+name)} + def dumpbool(self, name, expr): + return self.dump(name, '((char)((%s)?1:0))' % (expr,)) + def bool(self, data): + return data[0] != '\x00' +class CConfigExternEntry(object): + """Abstract base class.""" + def get(self, name, CC='gcc', NM=None, OBJCOPY=None): + if not NM: + NM = CC.replace('gcc', 'nm') + if not OBJCOPY: + OBJCOPY = CC.replace('gcc', 'objcopy') + libs = self.eci.link_extra + for obj in libs: + nm = Popen(NM + ' -f sysv ' + obj, shell=True, stdout=PIPE) + nm.wait() + nm = nm.stdout.read() + for line in nm.splitlines(): + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[0] == name and line[2] != 'U': + # only if the symbol is not a 'depends on' symbol ^ + offset = atoi(line[1], 16) + minsize = atoi(line[4], 16) + section = line[6] + break + else: + continue + break + else: + return None + sec = NamedTemporaryFile() + try: + if os.system('%s -O binary -j \'%s\' %s %s' % + (OBJCOPY, section, obj, sec.name)): + raise Exception('objcopy error') + sec.seek(offset) + return sec.read(minsize) + finally: + sec.close() + +class ExternString(CConfigExternEntry): + def __init__(self, name): + self.name = name + + def build_result(self, *args, **kw): + return self.get(self.name, *args, **kw) class Struct(CConfigEntry): """An entry in a CConfig class that stands for an externally Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Sat Sep 13 12:19:37 2008 @@ -1,4 +1,4 @@ -import py, sys, struct +import py, sys, struct, os from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi @@ -234,3 +234,17 @@ a = rffi_platform.memory_alignment() print a assert a % struct.calcsize("P") == 0 + +def test_extern_string(): + from tempfile import mkdtemp + dir = mkdtemp() + srcpath = os.path.join(dir, 'test.c') + objpath = os.path.join(dir, 'test.o') + src = open(srcpath, 'w') + src.write('char stuff[] = "Success!";\n') + src.close() + os.system('gcc -c -o %s %s' % (objpath, srcpath)) + class CConfig: + _compilation_info_ = ExternalCompilationInfo(link_extra = [objpath]) + STRING = rffi_platform.ExternString('stuff') + assert rffi_platform.configure(CConfig)['STRING'] == 'Success!\0' From pedronis at codespeak.net Sat Sep 13 16:29:57 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 13 Sep 2008 16:29:57 +0200 (CEST) Subject: [pypy-svn] r58109 - in pypy/branch/pypy-pytrunk/pypy: . tool/pytest tool/pytest/test Message-ID: <20080913142957.606E316A4DC@codespeak.net> Author: pedronis Date: Sat Sep 13 16:29:56 2008 New Revision: 58109 Removed: pypy/branch/pypy-pytrunk/pypy/tool/pytest/filelog.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/test/test_filelog.py Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py Log: WIP moving filelog to py/contrib Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Sat Sep 13 16:29:56 2008 @@ -30,9 +30,9 @@ Option('--direct', action="store_true", default=False, dest="rundirect", help="run pexpect tests directly"), - Option('--filelog', action="store", - default=None, dest="filelog", - help="path for FileLogSession logging") +# Option('--filelog', action="store", +# default=None, dest="filelog", +# help="path for FileLogSession logging") ) _SPACECACHE={} @@ -498,4 +498,4 @@ return (py.test.collect.Directory.recfilter(self, path) and path.check(link=0)) -from pypy.tool.pytest.filelog import FileLogSession +#from pypy.tool.pytest.filelog import FileLogSession From pedronis at codespeak.net Sat Sep 13 16:37:41 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 13 Sep 2008 16:37:41 +0200 (CEST) Subject: [pypy-svn] r58110 - pypy/branch/pypy-pytrunk/pypy/tool/pytest Message-ID: <20080913143741.213F116A408@codespeak.net> Author: pedronis Date: Sat Sep 13 16:37:40 2008 New Revision: 58110 Modified: pypy/branch/pypy-pytrunk/pypy/tool/pytest/ (props changed) Log: filelog as an external From pedronis at codespeak.net Sat Sep 13 16:50:09 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 13 Sep 2008 16:50:09 +0200 (CEST) Subject: [pypy-svn] r58114 - pypy/branch/pypy-pytrunk/pypy Message-ID: <20080913145009.31D3C16A4B5@codespeak.net> Author: pedronis Date: Sat Sep 13 16:50:07 2008 New Revision: 58114 Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py Log: re-enable filelog experimental session Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Sat Sep 13 16:50:07 2008 @@ -30,11 +30,13 @@ Option('--direct', action="store_true", default=False, dest="rundirect", help="run pexpect tests directly"), -# Option('--filelog', action="store", -# default=None, dest="filelog", -# help="path for FileLogSession logging") ) +# glue experimental filelog session +from pypy.tool.pytest.filelog.session import add_filelog_option, FileLogSession +add_filelog_option() + + _SPACECACHE={} def gettestobjspace(name=None, **kwds): """ helper for instantiating and caching space's for testing. @@ -497,5 +499,3 @@ # disable recursion in symlinked subdirectories return (py.test.collect.Directory.recfilter(self, path) and path.check(link=0)) - -#from pypy.tool.pytest.filelog import FileLogSession From exarkun at codespeak.net Sat Sep 13 18:55:45 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 13 Sep 2008 18:55:45 +0200 (CEST) Subject: [pypy-svn] r58115 - pypy/build/bot Message-ID: <20080913165545.954B316A496@codespeak.net> Author: exarkun Date: Sat Sep 13 18:55:39 2008 New Revision: 58115 Modified: pypy/build/bot/master.cfg Log: drop -O3 translation for now Modified: pypy/build/bot/master.cfg ============================================================================== --- pypy/build/bot/master.cfg (original) +++ pypy/build/bot/master.cfg Sat Sep 13 18:55:39 2008 @@ -28,7 +28,6 @@ 'schedulers': [Nightly("nightly", [ "pypy-c-32-O0", "pypy-c-32-O2", - "pypy-c-32-O3", "pypy-c-allworkingmodules-O3-64", "pypy-c-allworkingmodules-winxp32"], hour=19)], @@ -51,12 +50,6 @@ "factory": POSIXPyPyBuildFactory( None, ["-O2"], [], django=True)}, - {"name": "pypy-c-32-O3", - "slavenames": ["bigdogvm1"], - "builddir": "pypy-c-32-O3", - "factory": POSIXPyPyBuildFactory( - None, ["-O3"], [])}, - {"name": "pypy-c-allworkingmodules-O3-64", "slavenames": ["linux-dvs0"], "builddir": "pypy-c-allworkingmodules-faassen-64", From mwh at codespeak.net Sun Sep 14 08:58:05 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 08:58:05 +0200 (CEST) Subject: [pypy-svn] r58133 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914065805.01C2016A2CC@codespeak.net> Author: mwh Date: Sun Sep 14 08:58:02 2008 New Revision: 58133 Added: pypy/extradoc/talk/osdc2008/bigidea.png (contents, props changed) Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: add the big idea picture -- from a screenshot so screen resolution only :/ Added: pypy/extradoc/talk/osdc2008/bigidea.png ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 08:58:02 2008 @@ -98,7 +98,8 @@ * Translate to a lower-level, efficient form -XXX Get copy of that image in here somehow. +.. image:: bigidea.png + :scale: 50 We chose to specify the Python language by writing an implementation of Python in a restricted subset of Python that is amenable to @@ -162,7 +163,7 @@ PyPy when translated with the C backend (often called pypy-c) and most optimizations enabled is currently a reasonably fast and extremely -conformant implementation of Python 2.4.4. +conforming implementation of Python 2.4.4. XXX is conformant a word? XXX isn't it a wider range like 0.8 to 5 times, see e.g. recent sympy feedback on pypy-dev from Ondrej @@ -173,7 +174,7 @@ still-experimental Just-In-Time compiler runs some (admittedly carefully chosen) programs 60 times faster). -Extremely conformant: pypy-c runs Django 1.0, Pylons, Twisted and +Extremely conforming: pypy-c runs Django 1.0, Pylons, Twisted and Nevow unmodified (thanks to Google's Open Source office for funding work on this). There are two main sources of incompatibilities: From mwh at codespeak.net Sun Sep 14 10:26:50 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:26:50 +0200 (CEST) Subject: [pypy-svn] r58134 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914082650.45ED516A538@codespeak.net> Author: mwh Date: Sun Sep 14 10:26:47 2008 New Revision: 58134 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: some kind of progress Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:26:47 2008 @@ -52,49 +52,44 @@ Motivation ---------- -XXX it's good to highlight the shared idea/value of -XXX focusing on implementation rather than language design -XXX (it's till the case today, i'd say). -XXX if it's meant to tell how PyPy came about it's a mispresentation though - The beginnings PyPy can be traced to the first EuroPython conference in 2002, where some of the people who ultimately became involved in the project met in person for the first time, and realized they had a -common interest: we were all interested in modifying and extending the -CPython *implementation* of the Python language, rather than the -language itself (which was and still is the usual topic of discussion -on the python-dev list). - -These people included: - - * Armin Rigo, the author of Pysco, the well-known Python accelerator. - * Christian Tismer, the author of the "Stackless" variant of CPython, - which adds coroutines and other related forms of non-traditional - control flow to the language. - * Samuele Pedroni, at the time, the maintainer of Jython. - * Myself, at the time one of the more active CPython developers. - -While in many ways there's nothing deeply wrong about CPython, which -is written in a straightforward style in clear C, there are some -inherent issues: - - * Being written in C, it is hard to port to the JVM or CLI. - * Extensions to the language like Stackless have to be painfully kept - up to date with language changes as they are made. +common interest: we were all much more interested in *implementing* +the Python language, rather than the language itself, which was and +still is the usual topic of discussion on the python-dev list. + +Most of us had done a fair amount of work on and with the default, C, +implementation of Python, which is often called CPython. There is +nothing deeply wrong with CPython, which is written in a +straightforward style in clear C, but there are some inherent issues: + + * Being written in C, it is not useful for implementing Python on + platforms such as the JVM or CLI. + + * Extensions to the language like Stackless, which adds coroutines + and other non-traditional control flow to Python, have to be + painfully kept up to date with language changes as they are made. + * Some implementation decisions, such as using reference counting for - memory management or a Global Interpreter Lock, are very hard to - change by now. + memory management or a Global Interpreter Lock for threading, are + a lot of work to change. +More or less independently, we'd all decided we wanted something more +flexible. PyPy's Big Idea And The PyPy Meta-Platform ------------------------------------------ +In broad brush terms, PyPy's idea is quite simple: + * Take a description of the Python programming language - * Analyze this description, for example: + * Analyze this description + * Take various implementation decisions, for example: * Decide whether to include stackless- or psyco-like features - * Decide which GC to use - * Decide the target platform + * Decide whether to optimize for memory footprint or performance + * Decide on the target platform * Translate to a lower-level, efficient form @@ -103,33 +98,36 @@ We chose to specify the Python language by writing an implementation of Python in a restricted subset of Python that is amenable to -analysis. This let us write unit tests for parts of our -specification/implementation before we had the whole thing done, and -also let us test the whole specification/implementation before the code -that analysed the specification/implementation was written. - -The code that does the analysis -- now revealed in its true colours to -basically be a compiler for this restricted subset of Python -- is -generally referred to as the 'translator'. It is unlike most -compilers in that takes as input live Python objects (as opposed to -source code). It abstractly interprets the bytecode of functions to -produce flow graphs. Further layers of abstract interpretation -perform more analysis and gradually reduce the level of abstraction -before finally C or other source code is generated - -XXX (cfbolz) I found this section a bit confusing, giving that it's the -technical intro. It's all a bit clunky, and I think that the fact that we are -using abstract interpretation is mostly a detail. +analysis, which had an obvious practical advantage: we could test +parts or all of the specification/implementation simply by running +it. + +This means that almost from the start, PyPy has had two major +components: + + * An implementation of Python, written in a "static enough" subset + of Python. + + * The code that analyses this, now revealed in its true colours to + basically be a compiler for this restricted subset of Python. + +The development of the two parts was heavily inter-twined of course: +for example, the definition of "static enough" was a compromise +between making writing the interpreter pleasant and what was practical +to implement in the compiler. + The *LxOxP* problem ------------------- -We'd written this compiler framework, with only one expected -non-trivial input (our Python interpreter). We eventually realized -that it would be suitable for implementations of other -dynamically-typed programming languages. Now have implementations of -Prolog, JavaScript, Smalltalk and Scheme (to varying extents). We also have a -mostly working Gameboy emulator. +After several years of work on the project, we'd written this compiler +framework, with only one expected non-trivial input (our Python +interpreter). Finally, we realized that our compiler would be +suitable for implementations of other dynamically-typed programming +languages... + +Now have implementations of Prolog, JavaScript, Smalltalk and Scheme +(to varying extents) as well as a mostly working Gameboy emulator. This leads to one of PyPy's meta-goals, ameliorating the so-called LxOxP problem: given @@ -159,48 +157,64 @@ Status ------ -XXX some words about GCs, memory usage? +PyPy's dual nature means that it does not have a single status, so we +consider various parts in turn... -PyPy when translated with the C backend (often called pypy-c) and most -optimizations enabled is currently a reasonably fast and extremely -conforming implementation of Python 2.4.4. -XXX is conformant a word? +The Python Interpreter +++++++++++++++++++++++ -XXX isn't it a wider range like 0.8 to 5 times, see e.g. recent sympy feedback on pypy-dev from Ondrej -with an emphasis on it rather being slower usually? +PyPy's Python interpreter is an extremely conforming implementation of +Python 2.4. As evidence, PyPy can run, unmodified: -Reasonably fast: depending on the code being run, pypy-c ranges from -being 20% faster to about twice as slow as CPython (the -still-experimental Just-In-Time compiler runs some (admittedly -carefully chosen) programs 60 times faster). + * Django + * Pylons + * Twisted + * Nevow + +as well as 98% of CPython's 'core' tests -- roughly speaking those +tests that do not depend on extension modules (Google's Open Source +office recently funded some work on running "real" applications on +PyPy). + +By far the commonest incompatibility is the lack of extension modules. +PyPy supports a fair selection of the commonest extension modules -- +socket, select, ctypes, zlib, struct, ... -- but by no means all. -Extremely conforming: pypy-c runs Django 1.0, Pylons, Twisted and -Nevow unmodified (thanks to Google's Open Source office for funding -work on this). There are two main sources of incompatibilities: +Compatibility with Python 2.5 is almost complete in a branch. 2.6 +shouldn't be too hard. No Python 3 yet :) - - extension modules. PyPy supports a fair selection of the standard - extension modules (socket, struct, zlib, ctypes...) but hardly any - third party modules. - - finalizer semantics, for example assuming that immediately after - executing this code:: +The Compiler +++++++++++++ - >>> open('file.txt', 'w').write('blah') +PyPy's compiler has working backends that target C + POSIX (on Linux, +Windows and OS X), the JVM and the CLI, and backends in various states +of completeness for LLVM, Common Lisp and Squeak. - that the file object is already closed. +When targeting C/POSIX, it supports a range of garbage collection +options: -The former is usually a much more significant problem, of course. + * Using the Boehm-Demers-Weiser conservative garbage collector. + * Naive refcounting (used only in tests, really). + * A mark and sweep garbage collector. + * A copying generational collector. -XXX mention sandboxing as a distinguished feature? +The copying generational collector has the best performance. + + +The Compiled Interpreter +++++++++++++++++++++++++ + +When compiled with all optimizations enabled, PyPy translated to C has +performance roughly comparable to CPython, from 20% faster to 5 times +slower, with most programs clocking in at about half CPython's speed. -Compatibility with Python 2.5 is almost complete in a branch. 2.6 -shouldn't be too hard. No Python 3 yet :) -The currently supported backends target C/POSIX (like CPython), CLI -(like IronPython) and the JVM (like Jython). +Unique Stuff +++++++++++++ + +XXX mention sandboxing as a distinguished feature? -There are incomplete backends for LLVM (the Low Level Virtual -Machine), Common Lisp, Squeak and JavaScript. About The Project ----------------- @@ -214,6 +228,7 @@ period we had sprints about every 6 weeks, but now they're happening about 2-3 times a year. + Future ------ @@ -238,6 +253,7 @@ JIT magic is more advanced, get a Just in Time compiler almost for free. + Join the fun! ------------- From mwh at codespeak.net Sun Sep 14 10:40:41 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:40:41 +0200 (CEST) Subject: [pypy-svn] r58135 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914084041.69EE016A447@codespeak.net> Author: mwh Date: Sun Sep 14 10:40:39 2008 New Revision: 58135 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: I think this is the structure I want now, just need to expand/clarify. Comments welcome. Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:40:39 2008 @@ -189,7 +189,7 @@ PyPy's compiler has working backends that target C + POSIX (on Linux, Windows and OS X), the JVM and the CLI, and backends in various states -of completeness for LLVM, Common Lisp and Squeak. +of completeness for LLVM, Common Lisp, JavaScript and Squeak. When targeting C/POSIX, it supports a range of garbage collection options: @@ -213,15 +213,22 @@ Unique Stuff ++++++++++++ -XXX mention sandboxing as a distinguished feature? +JIT! + +Transparent Proxies. + +Sandboxing. + +JS backend. About The Project ----------------- -PyPy started as a classic open-source, spare-time project, was funded -for a while, and is currently mostly a spare-time project again. -XXX not a completely adequate description, e.g. google contracts +PyPy started as a classic open-source, spare-time project. For a +while -- two and a bit years -- about ten people worked full time on +PyPy as part of the EU-funded project. Now, XXX not sure what to +write here XXX. Right from the start, however, the project was driven along by *sprints* -- focused week long coding sessions. During the funding @@ -254,8 +261,8 @@ free. -Join the fun! -------------- +Getting involved +---------------- Read documentation: @@ -263,3 +270,5 @@ Come hang out in the #pypy IRC channel on freenode, subscribe to and post to the pypy-dev mailing list. + +XXX expand this. From fijal at codespeak.net Sun Sep 14 10:44:57 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 10:44:57 +0200 (CEST) Subject: [pypy-svn] r58136 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914084457.EBD8916A558@codespeak.net> Author: fijal Date: Sun Sep 14 10:44:55 2008 New Revision: 58136 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: A proposition for a paragraph rewrite Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:44:55 2008 @@ -42,6 +42,12 @@ achieved its objectives and tech goals and has even exceeded expectations" +XXX +Nowadays pypy is both open source project that people work on in their +spare time and project seeking funding from various companies to improve +certain parts of the project. +XXX + It has now gone back to being a project that people mostly work on in their spare time, with some people making PyPy the topic of Masters' theses and similar. Google's Open Source Programs Office also From mwh at codespeak.net Sun Sep 14 10:46:36 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:46:36 +0200 (CEST) Subject: [pypy-svn] r58137 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914084636.AD19F16A561@codespeak.net> Author: mwh Date: Sun Sep 14 10:46:33 2008 New Revision: 58137 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: i think i answered this point. Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:46:33 2008 @@ -3,9 +3,6 @@ :author: Michael Hudson -XXX the focus in the paper shifts from translation framework -XXX to pypy as python interpreter. I'd recommend to up-front -XXX distinguish more clearly. Abstract -------- From mwh at codespeak.net Sun Sep 14 10:48:44 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:48:44 +0200 (CEST) Subject: [pypy-svn] r58138 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914084844.DA8BA16A565@codespeak.net> Author: mwh Date: Sun Sep 14 10:48:42 2008 New Revision: 58138 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: accept and modify fijal's suggestion Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:48:42 2008 @@ -39,17 +39,10 @@ achieved its objectives and tech goals and has even exceeded expectations" -XXX -Nowadays pypy is both open source project that people work on in their -spare time and project seeking funding from various companies to improve -certain parts of the project. -XXX - -It has now gone back to being a project that people mostly work on in -their spare time, with some people making PyPy the topic of Masters' -theses and similar. Google's Open Source Programs Office also -sponsored some work on getting real world applications running PyPy -(more on that later). +Nowadays PyPy is both an open source project that people work on in +their spare time and as part of their studies and also project seeking +funding from various companies to improve certain parts of the +project. Motivation From mwh at codespeak.net Sun Sep 14 10:49:47 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:49:47 +0200 (CEST) Subject: [pypy-svn] r58139 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914084947.5BC6A16A56D@codespeak.net> Author: mwh Date: Sun Sep 14 10:49:46 2008 New Revision: 58139 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: this is mostly redundant Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:49:46 2008 @@ -218,20 +218,6 @@ JS backend. -About The Project ------------------ - -PyPy started as a classic open-source, spare-time project. For a -while -- two and a bit years -- about ten people worked full time on -PyPy as part of the EU-funded project. Now, XXX not sure what to -write here XXX. - -Right from the start, however, the project was driven along by -*sprints* -- focused week long coding sessions. During the funding -period we had sprints about every 6 weeks, but now they're happening -about 2-3 times a year. - - Future ------ From mwh at codespeak.net Sun Sep 14 10:50:31 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 10:50:31 +0200 (CEST) Subject: [pypy-svn] r58140 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914085031.3886516A15C@codespeak.net> Author: mwh Date: Sun Sep 14 10:50:28 2008 New Revision: 58140 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: a table of contents would be nice i guess Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 10:50:28 2008 @@ -4,6 +4,8 @@ :author: Michael Hudson +.. contents:: + Abstract -------- From mwh at codespeak.net Sun Sep 14 12:25:30 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 12:25:30 +0200 (CEST) Subject: [pypy-svn] r58141 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914102530.6AA8116A5D5@codespeak.net> Author: mwh Date: Sun Sep 14 12:25:27 2008 New Revision: 58141 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: got some comments from jml Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:25:27 2008 @@ -85,9 +85,9 @@ * Analyze this description * Take various implementation decisions, for example: - * Decide whether to include stackless- or psyco-like features - * Decide whether to optimize for memory footprint or performance - * Decide on the target platform + * Whether to include stackless- or psyco-like features + * Whether to optimize for memory footprint or performance + * Select the target platform * Translate to a lower-level, efficient form @@ -96,9 +96,8 @@ We chose to specify the Python language by writing an implementation of Python in a restricted subset of Python that is amenable to -analysis, which had an obvious practical advantage: we could test -parts or all of the specification/implementation simply by running -it. +analysis, which had a practical advantage: we could test the +specification/implementation simply by running it as a Python program. This means that almost from the start, PyPy has had two major components: @@ -174,7 +173,7 @@ office recently funded some work on running "real" applications on PyPy). -By far the commonest incompatibility is the lack of extension modules. +By far the most common incompatibility is the lack of extension modules. PyPy supports a fair selection of the commonest extension modules -- socket, select, ctypes, zlib, struct, ... -- but by no means all. @@ -211,38 +210,53 @@ Unique Stuff ++++++++++++ -JIT! +PyPy has a number of unique features that hint at the flexibility it's +architecture brings. -Transparent Proxies. - -Sandboxing. - -JS backend. + * The JIT: PyPy's Just-in-time compiler is still experimental, but + can run certain carefully designed programs sixty times faster than + CPython. The *really* interesting thing about PyPy's JIT is that + it was *automatically generated* from the interpreter + implementation. + + * Sandboxed execution: it is possible to compile a version of PyPy + that has all calls to external functions replaced with + + * The JavaScript backend. This potentially allows you to write + validation for your web forms once, in Python, and execute them + both in the browser (in JavaScript) and on the server. +.. XXX Could mention transparent proxies, dynamic grammar, ? Future ------ -For all that we've achieved so far, I think that the bulk of the work -so far has being laying the groundwork for the really fun stuff. +I think that the bulk of the work on PyPy so far has being laying the +groundwork for the really fun stuff. -The area with the most exciting potential is the JIT. PyPy has -already extended the state of the art in automatically generating a -JIT compiler for an implementation of a dynamic language. - -XXX mention that PyPy provides unprecedented possibilities in - producing Interpreters for specific environments (memory-constrained, - sandboxing, gaming-engines etc.) ? (hpk) i at least think it's a - very exciting area as well. +As mentioned above, one of the basic goals of PyPy is to allow +separation of language and implementation. For example, today if you +want a language implementation for use in a low memory situation, you +might choose Lua over Ruby or Python for this reason. But maybe one +day, you will be able to chose to compile a version of, say, Ruby +using PyPy making decisions to save memory and use that instead. + +In a similar vein, the sandboxing mentioned above might widen the +choice of languages suitable for scripting a game, or running code in +a web browser. + +However, the area with the most exciting potential is probably the +JIT. PyPy has already extended the state of the art in automatically +generating a JIT compiler for an implementation of a dynamic language. Down at the more nuts and bolts implementation level, something I'm interested in myself is stealing ideas -- and maybe even code -- from garbage collectors developed by the Jikes RVM project. -Something we'd really like to see are implementations of other dynamic -languages -- Ruby being an obvious example :) -- which would, when the -JIT magic is more advanced, get a Just in Time compiler almost for -free. +Something we'd really really like to see are implementations of other +dynamic languages -- Ruby being an obvious example :) -- which would, +when the JIT magic is more advanced, get a Just in Time compiler +almost for free, be sandboxable. Getting involved @@ -256,3 +270,9 @@ post to the pypy-dev mailing list. XXX expand this. + +Acknowledgements +---------------- + +Thanks to Maciek Fijalkowski, Jonathan Lange, Holger Krekel and Carl +Friedrich Bolz for comments on drafts. From fijal at codespeak.net Sun Sep 14 12:34:01 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 12:34:01 +0200 (CEST) Subject: [pypy-svn] r58142 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914103401.31CC916A222@codespeak.net> Author: fijal Date: Sun Sep 14 12:34:00 2008 New Revision: 58142 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: an XXX and mention that rpython is a strict subset Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:34:00 2008 @@ -69,6 +69,8 @@ and other non-traditional control flow to Python, have to be painfully kept up to date with language changes as they are made. +XXX mention psyco + * Some implementation decisions, such as using reference counting for memory management or a Global Interpreter Lock for threading, are a lot of work to change. @@ -95,7 +97,7 @@ :scale: 50 We chose to specify the Python language by writing an implementation -of Python in a restricted subset of Python that is amenable to +of Python in a restricted strict subset of Python that is amenable to analysis, which had a practical advantage: we could test the specification/implementation simply by running it as a Python program. From mwh at codespeak.net Sun Sep 14 12:37:12 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 14 Sep 2008 12:37:12 +0200 (CEST) Subject: [pypy-svn] r58143 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914103712.A429416A59E@codespeak.net> Author: mwh Date: Sun Sep 14 12:37:11 2008 New Revision: 58143 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: expand a few things Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:37:11 2008 @@ -222,7 +222,11 @@ implementation. * Sandboxed execution: it is possible to compile a version of PyPy - that has all calls to external functions replaced with + that has all calls to external functions replaced with *requests* + to another process which can choose how to interpret this reqest. + In this way it's possible to build an interpreter which has a very + limited view of the file system and can only use a prescribed + amount of CPU and memory. * The JavaScript backend. This potentially allows you to write validation for your web forms once, in Python, and execute them @@ -264,14 +268,13 @@ Getting involved ---------------- -Read documentation: +Like most open source projects, PyPy has a website: http://codespeak.net/pypy/ -Come hang out in the #pypy IRC channel on freenode, subscribe to and -post to the pypy-dev mailing list. - -XXX expand this. +with, perhaps not so usually, extensive documentation. We also have +an active IRC channel, #pypy on freenode, and a reasonably low-volume +mailing list, pypy-dev. Acknowledgements ---------------- From fijal at codespeak.net Sun Sep 14 12:38:00 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 12:38:00 +0200 (CEST) Subject: [pypy-svn] r58144 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914103800.48F4E16A5A2@codespeak.net> Author: fijal Date: Sun Sep 14 12:37:58 2008 New Revision: 58144 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: these backends were killed Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:37:58 2008 @@ -188,7 +188,7 @@ PyPy's compiler has working backends that target C + POSIX (on Linux, Windows and OS X), the JVM and the CLI, and backends in various states -of completeness for LLVM, Common Lisp, JavaScript and Squeak. +of completeness for LLVM and JavaScript. When targeting C/POSIX, it supports a range of garbage collection options: From fijal at codespeak.net Sun Sep 14 12:39:17 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 12:39:17 +0200 (CEST) Subject: [pypy-svn] r58145 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914103917.303F816A5B3@codespeak.net> Author: fijal Date: Sun Sep 14 12:39:16 2008 New Revision: 58145 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: hybrid gc is the best these days Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:39:16 2008 @@ -197,8 +197,10 @@ * Naive refcounting (used only in tests, really). * A mark and sweep garbage collector. * A copying generational collector. + * Extended copying generational collection with different generations + for young and large or old objects. -The copying generational collector has the best performance. +The extended copying generational collector has the best performance. The Compiled Interpreter From fijal at codespeak.net Sun Sep 14 12:41:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 12:41:33 +0200 (CEST) Subject: [pypy-svn] r58146 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914104133.1DC0D16A5C7@codespeak.net> Author: fijal Date: Sun Sep 14 12:41:31 2008 New Revision: 58146 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: Several comments Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:41:31 2008 @@ -202,6 +202,7 @@ The extended copying generational collector has the best performance. +XXX mention various threading approaches maybe? The Compiled Interpreter ++++++++++++++++++++++++ @@ -210,6 +211,7 @@ performance roughly comparable to CPython, from 20% faster to 5 times slower, with most programs clocking in at about half CPython's speed. +XXX mention that that's without a JIT at all. Unique Stuff ++++++++++++ @@ -228,7 +230,9 @@ to another process which can choose how to interpret this reqest. In this way it's possible to build an interpreter which has a very limited view of the file system and can only use a prescribed - amount of CPU and memory. + amount of CPU and memory. The thing which differentiates it from + other sandboxed python implementations is that it doesn't restrict + language features at all. * The JavaScript backend. This potentially allows you to write validation for your web forms once, in Python, and execute them @@ -236,6 +240,8 @@ .. XXX Could mention transparent proxies, dynamic grammar, ? +XXX [fijal] mention transparent proxies, kill javascript backend + Future ------ From fijal at codespeak.net Sun Sep 14 12:43:09 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Sep 2008 12:43:09 +0200 (CEST) Subject: [pypy-svn] r58147 - pypy/extradoc/talk/osdc2008 Message-ID: <20080914104309.DBBBC16A5DA@codespeak.net> Author: fijal Date: Sun Sep 14 12:43:08 2008 New Revision: 58147 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: Various XXXs Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Sun Sep 14 12:43:08 2008 @@ -255,6 +255,11 @@ day, you will be able to chose to compile a version of, say, Ruby using PyPy making decisions to save memory and use that instead. +XXX we know that pypy uses less memory per-object. + +XXX mention maybe that pypy has a very fast interpreter startup which + is useful for say CGI or embedded + In a similar vein, the sandboxing mentioned above might widen the choice of languages suitable for scripting a game, or running code in a web browser. From cami at codespeak.net Sun Sep 14 14:31:46 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 14 Sep 2008 14:31:46 +0200 (CEST) Subject: [pypy-svn] r58148 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080914123146.A9E9D16A682@codespeak.net> Author: cami Date: Sun Sep 14 14:31:43 2008 New Revision: 58148 Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/video.py Log: refactored video. extended functionality of window and background adapted tests test_video and test_video_register according to the refactoring Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Sun Sep 14 14:31:43 2008 @@ -495,7 +495,7 @@ assert video.line == [0] * (8+160+8) video.line = range(8+160+8) - video.draw_clean_background() + video.background.draw_clean_line(video.line_y) assert video.line == [0] * (8+160+8) \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Sun Sep 14 14:31:43 2008 @@ -9,7 +9,7 @@ def get_control_register(): video = get_video() - return ControlRegister(Window(video), Background()) + return ControlRegister(Window(video), Background(video)) def get_status_register(): return StatusRegister(get_video()) Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 14 14:31:43 2008 @@ -60,14 +60,14 @@ def read(self): value = 0 - value += int(self.lcd_enabled) << 7 - value += int(self.window.upper_tile_map_selected) << 6 - value += int(self.window.enabled) << 5 + value += int(self.lcd_enabled) << 7 + value += int(self.window.upper_tile_map_selected) << 6 + value += int(self.window.enabled) << 5 value += int(self.background_and_window_lower_tile_data_selected) << 4 - value += int(self.background.upper_tile_map_selected) << 3 - value += int(self.big_sprite_size_selected) << 2 - value += int(self.sprites_enabled) << 1 - value += int(self.background.enabled) + value += int(self.background.upper_tile_map_selected) << 3 + value += int(self.big_sprite_size_selected) << 2 + value += int(self.sprites_enabled) << 1 + value += int(self.background.enabled) << 0 return value def write(self, value): @@ -81,14 +81,8 @@ self.sprites_enabled = bool(value & (1 << 1)) self.background.enabled = bool(value & (1 << 0)) - def get_tile_map_combined(self): - #if (self.control.read() & mask) != 0: - if self.background_and_window_lower_tile_data_selected: - return constants.VRAM_MAP_B - else: - return constants.VRAM_MAP_A - - def get_selected_tile_map_space(self): + + def get_selected_tile_data_space(self): if self.window.upper_tile_map_selected: return constants.VRAM_DATA_A else: @@ -527,21 +521,47 @@ self.reset() def reset(self): - self.x = 0 - self.y = 0 - self.line_y = 0 + self.x = 0 + self.y = 0 + self.line_y = 0 + self.enabled = False self.upper_tile_map_selected = False - self.enabled = False def update_line_y(self, data): # don't draw window if it was not enabled and not being drawn before if not self.enabled and (data & 0x20) != 0 and \ self.line_y == 0 and self.video.line_y > self.y: - self.line_y = 144 + self.line_y = 144 + + def get_tile_map_space(self): + #if (self.control.read() & mask) != 0: + if self.upper_tile_map_selected: + return constants.VRAM_MAP_B + else: + return constants.VRAM_MAP_A + + def draw_line(self, line_y): + if line_y < self.y or self.x >= 167 or \ + self.line_y >= 144: + return + else: + tile_map, tile_data = self.prepare_window_data() + self.video.draw_tiles(self.x + 1, tile_map, tile_data) + self.line_y += 1 + + def prepare_window_data(self): + tile_map = self.get_tile_map_space() + tile_map += (self.line_y >> 3) << 5 + tile_data = self.video.control.get_selected_tile_data_space() + tile_data += (self.line_y & 7) << 1 + return tile_map, tile_data; + +# ----------------------------------------------------------------------------- class Background(object): - def __init__(self): + def __init__(self, video): + self.video = video self.reset() def reset(self): @@ -550,7 +570,32 @@ self.scroll_x = 0 self.scroll_y = 0 self.enabled = True - self.upper_tile_map_selected = False + self.upper_tile_map_selected = False + + def get_tile_map_space(self): + #if (self.control.read() & mask) != 0: + if self.upper_tile_map_selected: + return constants.VRAM_MAP_B + else: + return constants.VRAM_MAP_A + + def draw_clean_line(self, line_y): + for x in range(8+160+8): + self.video.line[x] = 0x00 + + def draw_line(self, line_y): + y = (self.scroll_y + line_y) & 0xFF + x = self.scroll_x & 0xFF + tile_map, tile_data = self.prepare_background_data(x, y) + self.video.draw_tiles(8 - (x & 7), tile_map, tile_data) + + def prepare_background_data(self, x, y): + tile_map = self.get_tile_map_space() + tile_map += ((y >> 3) << 5) + (x >> 3) + tile_data = self.video.control.get_selected_tile_data_space() + tile_data += (y & 7) << 1 + return tile_map, tile_data + # ----------------------------------------------------------------------------- @@ -562,7 +607,7 @@ self.v_blank_interrupt_flag = interrupt.v_blank self.lcd_interrupt_flag = interrupt.lcd self.window = Window(self) - self.background = Background() + self.background = Background(self) self.status = StatusRegister(self) self.control = ControlRegister(self.window, self.background) @@ -576,7 +621,8 @@ self.tile_map_1 = [None] * 32 * 32 self.tile_map_2 = [None] * 32 * 32 - def update_tiles(self): + def update_tile(self, address, data): + # XXX to implement pass def create_sprites(self): @@ -584,7 +630,7 @@ for i in range(40): self.sprites[i] = Sprite() - def update_sprites(self): + def update_all_sprites(self): for i in range(40): address = 1 * 4 self.sprites[i].set_data(self.oam[address + 0], @@ -592,6 +638,15 @@ self.oam[address + 2], self.oam[address + 3]) + def update_sprite(self, address, data): + # address divided by 4 gives the correct sprite, each sprite has 4 + # bytes of attributes + sprite_id = floor(address / 4) + data = [None]*4 + # assign the data to the correct attribute + data[address % 4] = data + self.sprites[sprite_id].set_data(data[0], data[1], data[2], data[3]) + def reset(self): self.control.reset() self.status.reset() @@ -816,7 +871,7 @@ self.dma = data for index in range(constants.OAM_SIZE): self.oam[index] = self.memory.read((self.dma << 8) + index) - self.update_sprites() + self.update_all_sprites() def get_background_palette(self): """ see set_background_palette""" @@ -896,7 +951,7 @@ """ self.oam[address - constants.OAM_ADDR] = data & 0xFF #self.update_sprites(address) - self.update_sprites() + self.update_sprite(address, data) def get_oam(self, address): return self.oam[address - constants.OAM_ADDR] @@ -904,10 +959,10 @@ def set_vram(self, address, data): """ sets one byte of the video memory. - The video memroy contains the tiles used to disply. + The video memroy contains the tiles used to display. """ self.vram[address - constants.VRAM_ADDR] = data & 0xFF - self.update_tiles() + self.update_tile(address, data) def get_vram(self, address): return self.vram[address - constants.VRAM_ADDR] @@ -936,50 +991,16 @@ def draw_line(self): if self.background.enabled: - self.draw_background() + self.background.draw_line(self.line_y) else: - self.draw_clean_background() + self.background.draw_clean_line(self.line_y) if self.window.enabled: - self.draw_window() + self.window.draw_line(self.line_y) if self.control.sprites_enabled: - self.draw_sprites() - self.draw_pixels() - - def draw_clean_background(self): - for x in range(0, 8+160+8): - self.line[x] = 0x00 - - def draw_background(self): - y = (self.background.scroll_y + self.line_y) & 0xFF - x = self.background.scroll_x & 0xFF - tile_map, tile_data = self.prepare_background_data(x, y) - self.draw_tiles(8 - (x & 7), tile_map, tile_data) - - def prepare_background_data(self, x, y): - tile_map = self.control.get_selected_tile_map_space() - tile_map += ((y >> 3) << 5) + (x >> 3) - tile_data = self.control.get_selected_tile_map_space() - tile_data += (y & 7) << 1 - return tile_map, tile_data - - def draw_window(self): - if self.line_y < self.window.y or self.window.x >= 167 or \ - self.window.line_y >= 144: - return - else: - tile_map, tile_data = self.prepare_window_data() - self.draw_tiles(self.window.x + 1, tile_map, tile_data) - self.window.line_y += 1 + self.draw_sprites_line() + self.draw_pixels_line() - def prepare_window_data(self): - tile_map = self.control.get_tile_map_combined() - tile_map += (self.window.line_y >> 3) << 5 - tile_data = self.control.get_selected_tile_map_space() - tile_data += (self.window.line_y & 7) << 1 - return tile_map, tile_data; - - - def draw_sprites(self): + def draw_sprites_line(self): count = self.scan_sprites() lastx = 176 for index in range(176, count): @@ -1104,14 +1125,13 @@ if (color & 0x0202) != 0: caller.call(x+7, color, mask) - def draw_pixels(self): + def draw_pixels_line(self): self.update_palette() pixels = self.driver.get_pixels() offset = self.line_y * self.driver.get_width() for x in range(8, 168, 4): for i in range(0,4): - pattern = self.line[x + i] - pixels[offset + i] = self.palette[pattern] + pixels[offset + i] = self.palette[self.line[x + i]] offset += 4 def clear_pixels(self): From cami at codespeak.net Sun Sep 14 14:49:49 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 14 Sep 2008 14:49:49 +0200 (CEST) Subject: [pypy-svn] r58149 - pypy/dist/pypy/lang/gameboy Message-ID: <20080914124949.1C6F616A6E3@codespeak.net> Author: cami Date: Sun Sep 14 14:49:48 2008 New Revision: 58149 Modified: pypy/dist/pypy/lang/gameboy/video.py Log: fixed bug when accessing the wrong address for updating the sprite attribute Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 14 14:49:48 2008 @@ -2,7 +2,7 @@ PyGirl Emulator constants.LCD Video Display Processor """ - +import math from pypy.lang.gameboy import constants from pypy.lang.gameboy.ram import iMemory from pypy.lang.gameboy.cpu import process_2_complement @@ -317,7 +317,7 @@ if self.video.transfer: if self.video.display: self.video.draw_line() - print "mode 3 ", self.video.status.get_mode() + #print "mode 3 ", self.video.status.get_mode() self.set_end() else: self.video.status.set_mode(0) @@ -415,17 +415,17 @@ self.hidden = True - def set_data(self, byte0=None, byte1=None, byte2=None, byte3=None): + def set_data(self, byte0=-1, byte1=-1, byte2=-1, byte3=-1): """ extracts the sprite data from an oam entry """ - if byte0 is not None: + if byte0 is not -1: self.extract_y_position(byte0) - if byte0 is not None: + if byte0 is not -1: self.extract_x_position(byte1) - if byte0 is not None: + if byte0 is not -1: self.extract_tile_number(byte2) - if byte0 is not None: + if byte0 is not -1: self.extract_attributes_and_flags(byte3) def extract_y_position(self, data): @@ -639,13 +639,16 @@ self.oam[address + 3]) def update_sprite(self, address, data): + address -= constants.OAM_ADDR # address divided by 4 gives the correct sprite, each sprite has 4 # bytes of attributes - sprite_id = floor(address / 4) - data = [None]*4 + sprite_id = int(math.floor(address / 4)) + # XXX why cant I use None here + attribute = [-1] * 4 # assign the data to the correct attribute - data[address % 4] = data - self.sprites[sprite_id].set_data(data[0], data[1], data[2], data[3]) + attribute[address % 4] = data + self.sprites[sprite_id].set_data(attribute[0], attribute[1], + attribute[2], attribute[3]) def reset(self): self.control.reset() From exarkun at codespeak.net Sun Sep 14 17:51:46 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sun, 14 Sep 2008 17:51:46 +0200 (CEST) Subject: [pypy-svn] r58156 - pypy/build/doc Message-ID: <20080914155146.5644616A6BF@codespeak.net> Author: exarkun Date: Sun Sep 14 17:51:45 2008 New Revision: 58156 Modified: pypy/build/doc/ssh_config Log: config for bigdog's machines Modified: pypy/build/doc/ssh_config ============================================================================== --- pypy/build/doc/ssh_config (original) +++ pypy/build/doc/ssh_config Sun Sep 14 17:51:45 2008 @@ -1,6 +1,14 @@ Host bigdog2 Hostname 8.8.197.72 +Host bigdog6 + Hostname 8.8.197.72 + Port 2222 + +Host bigdog-vm1 + Hostname 8.8.197.72 + Port 3001 + Host snake HostName snake.cs.uni-duesseldorf.de User pypy From cami at codespeak.net Sun Sep 14 19:18:01 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 14 Sep 2008 19:18:01 +0200 (CEST) Subject: [pypy-svn] r58158 - pypy/dist/pypy/lang/gameboy Message-ID: <20080914171801.6190116A7F0@codespeak.net> Author: cami Date: Sun Sep 14 19:17:56 2008 New Revision: 58158 Modified: pypy/dist/pypy/lang/gameboy/video.py Log: added templates for the new sprite handling Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 14 19:17:56 2008 @@ -400,7 +400,8 @@ class Sprite(object): - def __init__(self): + def __init__(self, video): + self.video = video self.big_size = False self.reset() @@ -496,9 +497,18 @@ else: return 8 - def overlaps(self, sprite): + def overlaps_on_line(self, sprite, line): + return False + + def intersects_line(self, line): return False + def draw(self): + pass + + def draw_overlapped(self): + pass + # ----------------------------------------------------------------------------- @@ -628,7 +638,7 @@ def create_sprites(self): self.sprites = [None] * 40 for i in range(40): - self.sprites[i] = Sprite() + self.sprites[i] = Sprite(self) def update_all_sprites(self): for i in range(40): @@ -649,7 +659,8 @@ attribute[address % 4] = data self.sprites[sprite_id].set_data(attribute[0], attribute[1], attribute[2], attribute[3]) - + + def reset(self): self.control.reset() self.status.reset() @@ -1003,6 +1014,26 @@ self.draw_sprites_line() self.draw_pixels_line() + def draw_sprites_line_new(self): + sprites_on_line = self.get_active_sprites_on_line(self.line_y) + + last_sprite = sprites_on_line[0] + last_sprite.draw() + + for sprite in sprites_on_line[1:]: + if sprite.overlaps_on_line(last_sprite, self.line_y): + sprite.draw_overlapped() + else: + sprite.draw() + + def get_active_sprites_on_line(self, line_y): + found = [] + for i in range(len(self.sprites)): + if self.sprites[i].intersects_line(line_y) and \ + self.sprites[i].enabled: + found.append(self.sprites[i]) + return found + def draw_sprites_line(self): count = self.scan_sprites() lastx = 176 From cami at codespeak.net Sun Sep 14 19:34:53 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 14 Sep 2008 19:34:53 +0200 (CEST) Subject: [pypy-svn] r58159 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080914173453.7301516A7E5@codespeak.net> Author: cami Date: Sun Sep 14 19:34:51 2008 New Revision: 58159 Added: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Modified: pypy/dist/pypy/lang/gameboy/video.py Log: added sprite tests Added: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Sun Sep 14 19:34:51 2008 @@ -0,0 +1,86 @@ +from pypy.lang.gameboy import constants +from pypy.lang.gameboy.video import Sprite +from pypy.lang.gameboy.video import Video +from pypy.lang.gameboy.test.test_video import get_video +import py + + + +def get_sprite(): + return Sprite(get_video()) + + +# Sprite Class test ------------------------------------------------------------ + +def test_standard_values(sprite=None): + if sprite==None: + sprite = get_sprite() + #assert sprite.get_tile_number() == 0 + assert sprite.x == 0 + assert sprite.y == 0 + assert sprite.object_behind_background == False + assert sprite.x_flipped == False + assert sprite.y_flipped == False + assert sprite.palette_number == 0 + + +def test_reset(): + sprite = get_sprite() + + # sprite.set_tile_number(0x12) + # assert sprite.get_tile_number() == 0x12 + + # sprite.set_tile_number(0xFFF) + # assert sprite.get_tile_number() == 0xFF + + sprite.x = 10 + assert sprite.x == 10 + + sprite.y = 11 + assert sprite.y == 11 + + sprite.object_behind_background = True + assert sprite.object_behind_background == True + + sprite.x_flipped = True + assert sprite.x_flipped == True + + sprite.y_flipped = True + assert sprite.y_flipped == True + + sprite.use_object_pallette_1 = True + assert sprite.use_object_pallette_1 == True + + sprite.reset() + test_standard_values(sprite) + +def test_size(): + sprite = get_sprite() + sprite.big_size = False + assert sprite.get_width() == 8 + assert sprite.get_height() == 8 + + sprite.big_size = True + assert sprite.get_width() == 8 + assert sprite.get_height() == 16 + + +def test_intersects_line_normal_size(): + sprite = get_sprite() + sprite.big_size = False + sprite.y = 1 + line_intersection_test(sprite, 8) + +def test_intersects_line_big_size(): + sprite = get_sprite() + sprite.big_size = True + sprite.y = 1 + line_intersection_test(sprite, 16) + +def line_intersection_test(sprite, height): + assert not sprite.intersects_line(0) + for i in range(height): + assert sprite.intersects_line(i+1) + assert not sprite.intersects_line(height+2) + +# test sprite in video --------------------------------------------------------- Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 14 19:34:51 2008 @@ -468,8 +468,6 @@ Bit6 Y flip (0=Normal, 1=Vertically mirrored) Bit5 X flip (0=Normal, 1=Horizontally mirrored) Bit4 Palette number **Non CGB Mode Only** (0=OBP0, 1=OBP1) - Bit3 Tile VRAM-Bank **CGB Mode Only** (0=Bank 0, 1=Bank 1) - Bit2-0 Palette number **CGB Mode Only** (OBP0-7) """ self.object_behind_background = bool(data & (1 << 7)) self.x_flipped = bool(data & (1 << 6)) @@ -487,6 +485,9 @@ def get_tile_number(self): return self.tile.id + + def set_tile_number(self, tile_number): + self.tile = self.video.tiles[tile_number] def get_width(self): return 8 @@ -501,7 +502,7 @@ return False def intersects_line(self, line): - return False + return line >= self.y and line <= self.y + self.get_height() def draw(self): pass @@ -628,8 +629,9 @@ def create_tile_maps(self): # create the maxumal possible sprites + self.tile_map_0 = [None] * 32 * 32 self.tile_map_1 = [None] * 32 * 32 - self.tile_map_2 = [None] * 32 * 32 + self.tile_maps = [self.tile_map_0, self.tile_map_1] def update_tile(self, address, data): # XXX to implement From cami at codespeak.net Sun Sep 14 19:41:42 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 14 Sep 2008 19:41:42 +0200 (CEST) Subject: [pypy-svn] r58160 - pypy/dist/pypy/lang/gameboy/test Message-ID: <20080914174142.8290516A7F5@codespeak.net> Author: cami Date: Sun Sep 14 19:41:41 2008 New Revision: 58160 Modified: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Log: added more tests Modified: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Sun Sep 14 19:41:41 2008 @@ -65,6 +65,48 @@ assert sprite.get_height() == 16 +def test_hiddden_check(): + sprite = get_sprite() + assert sprite.hidden + + sprite.y = 1 + sprite.x = 0 + sprite.hide_check() + assert sprite.hidden + + sprite.y = 0 + sprite.x = 1 + sprite.hide_check() + assert sprite.hidden + + sprite.y = 1 + sprite.x = 1 + sprite.hide_check() + assert not sprite.hidden + + for y in range(1, 160-1): + for x in range(1, 168-1): + sprite.y = y + sprite.x = x + sprite.hide_check() + assert not sprite.hidden + + for x in range(1, 168-1): + sprite.y = 160 + sprite.x = x + sprite.hide_check() + assert sprite.hidden + + for y in range(1, 160-1): + sprite.y = y + sprite.x = 168 + sprite.hide_check() + assert sprite.hidden + +def test_set_data(): + py.test.skip("test not implemented") + sprite = get_sprite() + def test_intersects_line_normal_size(): sprite = get_sprite() sprite.big_size = False @@ -83,4 +125,11 @@ assert sprite.intersects_line(i+1) assert not sprite.intersects_line(height+2) +def test_intersects_line_normal_size_y_flipped(): + py.test.skip("not yet implemented") + + +def test_intersects_big_normal_size_y_flipped(): + py.test.skip("not yet implemented") + # test sprite in video --------------------------------------------------------- From arigo at codespeak.net Sun Sep 14 20:00:57 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 14 Sep 2008 20:00:57 +0200 (CEST) Subject: [pypy-svn] r58161 - pypy/branch/oo-jit/dotviewer Message-ID: <20080914180057.B098E16A16E@codespeak.net> Author: arigo Date: Sun Sep 14 20:00:56 2008 New Revision: 58161 Modified: pypy/branch/oo-jit/dotviewer/drawgraph.py Log: Improve the heuristic that detects unexpectedly reversed edges. It should now work also for "horizontal" edges. Modified: pypy/branch/oo-jit/dotviewer/drawgraph.py ============================================================================== --- pypy/branch/oo-jit/dotviewer/drawgraph.py (original) +++ pypy/branch/oo-jit/dotviewer/drawgraph.py Sun Sep 14 20:00:56 2008 @@ -167,8 +167,16 @@ def arrowhead(self): result = self.cachedarrowhead if result is None: - bottom_up = self.points[0][1] > self.points[-1][1] - if (self.tail.y > self.head.y) != bottom_up: # reversed edge + # we don't know if the list of points is in the right order + # or not :-( try to guess... + def dist(node, pt): + return abs(node.x - pt[0]) + abs(node.y - pt[1]) + + error_if_direct = (dist(self.head, self.points[-1]) + + dist(self.tail, self.points[0])) + error_if_reversed = (dist(self.tail, self.points[-1]) + + dist(self.head, self.points[0])) + if error_if_direct > error_if_reversed: # reversed edge head = 0 dir = 1 else: From mwh at codespeak.net Mon Sep 15 04:20:02 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 15 Sep 2008 04:20:02 +0200 (CEST) Subject: [pypy-svn] r58164 - pypy/extradoc/talk/osdc2008 Message-ID: <20080915022002.6780D16A856@codespeak.net> Author: mwh Date: Mon Sep 15 04:19:56 2008 New Revision: 58164 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: resolve, occasionally by removing, XXXs Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Mon Sep 15 04:19:56 2008 @@ -66,10 +66,9 @@ platforms such as the JVM or CLI. * Extensions to the language like Stackless, which adds coroutines - and other non-traditional control flow to Python, have to be - painfully kept up to date with language changes as they are made. - -XXX mention psyco + and other non-traditional control flow to Python, or Pysco, Armin + Rigo's specializing compiler, have to be painfully kept up to date + with language changes as they are made. * Some implementation decisions, such as using reference counting for memory management or a Global Interpreter Lock for threading, are @@ -191,27 +190,27 @@ of completeness for LLVM and JavaScript. When targeting C/POSIX, it supports a range of garbage collection -options: +options, including: * Using the Boehm-Demers-Weiser conservative garbage collector. * Naive refcounting (used only in tests, really). * A mark and sweep garbage collector. + * A copying semi-space collector. * A copying generational collector. - * Extended copying generational collection with different generations - for young and large or old objects. + * A 'hybrid' generational collector that uses a semi-space for the + nursery and mark and sweep for the oldest generation. -The extended copying generational collector has the best performance. +The hybrid collector has the best performance. -XXX mention various threading approaches maybe? +The compiler supports threading with a GIL-like model. The Compiled Interpreter ++++++++++++++++++++++++ When compiled with all optimizations enabled, PyPy translated to C has performance roughly comparable to CPython, from 20% faster to 5 times -slower, with most programs clocking in at about half CPython's speed. - -XXX mention that that's without a JIT at all. +slower, with most programs clocking in at about half CPython's speed +(without any JIT magic). Unique Stuff ++++++++++++ @@ -234,13 +233,6 @@ other sandboxed python implementations is that it doesn't restrict language features at all. - * The JavaScript backend. This potentially allows you to write - validation for your web forms once, in Python, and execute them - both in the browser (in JavaScript) and on the server. - -.. XXX Could mention transparent proxies, dynamic grammar, ? - -XXX [fijal] mention transparent proxies, kill javascript backend Future ------ @@ -255,11 +247,6 @@ day, you will be able to chose to compile a version of, say, Ruby using PyPy making decisions to save memory and use that instead. -XXX we know that pypy uses less memory per-object. - -XXX mention maybe that pypy has a very fast interpreter startup which - is useful for say CGI or embedded - In a similar vein, the sandboxing mentioned above might widen the choice of languages suitable for scripting a game, or running code in a web browser. From arigo at codespeak.net Mon Sep 15 10:24:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 15 Sep 2008 10:24:15 +0200 (CEST) Subject: [pypy-svn] r58165 - pypy/extradoc/talk/osdc2008 Message-ID: <20080915082415.B452316A83D@codespeak.net> Author: arigo Date: Mon Sep 15 10:24:13 2008 New Revision: 58165 Modified: pypy/extradoc/talk/osdc2008/paper.txt Log: Typos. Fix the description of the hybrid gc. Modified: pypy/extradoc/talk/osdc2008/paper.txt ============================================================================== --- pypy/extradoc/talk/osdc2008/paper.txt (original) +++ pypy/extradoc/talk/osdc2008/paper.txt Mon Sep 15 10:24:13 2008 @@ -42,7 +42,7 @@ expectations" Nowadays PyPy is both an open source project that people work on in -their spare time and as part of their studies and also project seeking +their spare time and as part of their studies and also a project seeking funding from various companies to improve certain parts of the project. @@ -198,7 +198,7 @@ * A copying semi-space collector. * A copying generational collector. * A 'hybrid' generational collector that uses a semi-space for the - nursery and mark and sweep for the oldest generation. + intermediate generations and mark and sweep for the oldest generation. The hybrid collector has the best performance. @@ -262,7 +262,7 @@ Something we'd really really like to see are implementations of other dynamic languages -- Ruby being an obvious example :) -- which would, when the JIT magic is more advanced, get a Just in Time compiler -almost for free, be sandboxable. +almost for free, and be sandboxable. Getting involved From arigo at codespeak.net Mon Sep 15 11:29:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 15 Sep 2008 11:29:38 +0200 (CEST) Subject: [pypy-svn] r58166 - pypy/branch/oo-jit/pypy/translator/tool Message-ID: <20080915092938.73BD516A541@codespeak.net> Author: arigo Date: Mon Sep 15 11:29:31 2008 New Revision: 58166 Modified: pypy/branch/oo-jit/pypy/translator/tool/make_dot.py Log: Allow an edge "port" specification. Modified: pypy/branch/oo-jit/pypy/translator/tool/make_dot.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/tool/make_dot.py (original) +++ pypy/branch/oo-jit/pypy/translator/tool/make_dot.py Mon Sep 15 11:29:31 2008 @@ -55,12 +55,17 @@ dir="forward", weight="5", constraint="true", + headport="", tailport="", ): + # the ports are used to force on which side of the node the + # edge should be attached; for example ":s" means south + # of (i.e. below) the node box. d = locals() attrs = [('%s="%s"' % (x, d[x].replace('"', '\\"').replace('\n', '\\n'))) for x in ['label', 'style', 'color', 'dir', 'weight', 'constraint']] self.emit('edge [%s];' % ", ".join(attrs)) - self.emit('%s -> %s' % (safename(name1), safename(name2))) + self.emit('%s%s -> %s%s' % (safename(name1), tailport, + safename(name2), headport)) def emit_node(self, name, shape="diamond", From arigo at codespeak.net Mon Sep 15 14:18:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 15 Sep 2008 14:18:40 +0200 (CEST) Subject: [pypy-svn] r58167 - pypy/dist/pypy/lib/_fakecompiler Message-ID: <20080915121840.C2F9016A2B8@codespeak.net> Author: arigo Date: Mon Sep 15 14:18:37 2008 New Revision: 58167 Removed: pypy/dist/pypy/lib/_fakecompiler/ Log: This appears to be no longer used, unless I can't use 'grep' properly. From pedronis at codespeak.net Mon Sep 15 18:12:51 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 15 Sep 2008 18:12:51 +0200 (CEST) Subject: [pypy-svn] r58168 - in pypy/build/bot2: . pypybuildbot pypybuildbot/test Message-ID: <20080915161251.2C4D816A46B@codespeak.net> Author: pedronis Date: Mon Sep 15 18:12:48 2008 New Revision: 58168 Added: pypy/build/bot2/ pypy/build/bot2/master.cfg (contents, props changed) pypy/build/bot2/pypybuildbot/ (props changed) pypy/build/bot2/pypybuildbot/__init__.py (contents, props changed) pypy/build/bot2/pypybuildbot/master.py (contents, props changed) pypy/build/bot2/pypybuildbot/summary.py (contents, props changed) pypy/build/bot2/pypybuildbot/test/ (props changed) pypy/build/bot2/pypybuildbot/test/test_summary.py (contents, props changed) pypy/build/bot2/slaveinfo.py (contents, props changed) Log: (iko, pedronis) work in progress: display of results from FileLogSession gathered through buildbot this is also the whole config of the buildbot we are running locally to experiment with this Added: pypy/build/bot2/master.cfg ============================================================================== --- (empty file) +++ pypy/build/bot2/master.cfg Mon Sep 15 18:12:48 2008 @@ -0,0 +1,11 @@ +import sys, os + +slavePortnum = "tcp:10407" +httpPortNumber = 8099 + +# slavename -> password +from slaveinfo import passwords + +# checkout bot2 in the home dir of the master +sys.path.append(os.path.expanduser('~/bot2/')) +execfile(os.path.expanduser('~/bot2/pypybuildbot/master.py')) Added: pypy/build/bot2/pypybuildbot/__init__.py ============================================================================== Added: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/master.py Mon Sep 15 18:12:48 2008 @@ -0,0 +1,62 @@ +from buildbot.scheduler import Nightly +from buildbot.buildslave import BuildSlave +from buildbot.status.html import WebStatus + +from buildbot.process import factory +from buildbot.steps import source, shell + + +# I really wanted to pass logPath to Site +from twisted.web.server import Site +class LoggingSite(Site): + def __init__(self, *a, **kw): + Site.__init__(self, logPath='httpd.log', *a, **kw) +from twisted.web import server +if server.Site.__name__ == 'Site': + server.Site = LoggingSite +# So I did. + +status = WebStatus(httpPortNumber, allowForce=True) + +pypyOwnFactory = factory.BuildFactory() +pypyOwnFactory.addStep(source.SVN("https://codespeak.net/svn/pypy/branch/pypy-pytrunk")) +pypyOwnFactory.addStep(shell.ShellCommand( + description="pytest", + command="py/bin/py.test pypy/module/__builtin__ pypy/module/operator --session=FileLogSession --filelog=pytest.log".split(), + logfiles={'pytestLog': 'pytest.log'})) + +BuildmasterConfig = { + 'slavePortnum': slavePortnum, + + 'change_source': [], + 'schedulers': [Nightly("nightly", + ["pypy-own-linux", "pypy-own-other-linux"], hour=19)], + 'status': [status], + + 'slaves': [BuildSlave(name, password) + for (name, password) + in passwords.iteritems()], + + 'builders': [ + {"name": "pypy-own-linux", + "slavenames": ["vitaly"], + "builddir": "pypy-own-linux", + "factory": pypyOwnFactory + }, + {"name": "pypy-own-other-linux", + "slavenames": ["fido"], + "builddir": "pypy-own-other-linux", + "factory": pypyOwnFactory + } + ], + + 'buildbotURL': 'http://localhost:%d/' % (httpPortNumber,), + 'projectURL': 'http://codespeak.net/pypy/', + 'projectName': 'PyPy'} + +import pypybuildbot.summary +reload(pypybuildbot.summary) +summary = pypybuildbot.summary + +# pypy test summary page +status.putChild('summary', summary.Summary()) Added: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Sep 15 18:12:48 2008 @@ -0,0 +1,95 @@ + +from buildbot.status.web.base import HtmlResource + +# xxx caching? +class RevOutcome(object): + + def __init__(self, rev): + self.rev = rev + self._outcomes = {} + self.failed = set() + self.skipped = set() + # xxx failure tracebacks + + def populate_one(self, name, shortrepr): + namekey = name.split(':', 1) + if namekey[0].endswith('.py'): + namekey[0] = namekey[0][:-3].replace('/', '.') + + namekey = tuple(namekey) + self._outcomes[namekey] = shortrepr + if shortrepr == 's': + self.skipped.add(namekey) + elif shortrepr == '.': + pass + else: + self.failed.add(namekey) + + def populate(self, log): + for line in log.readlines(): + kind = line[0] + if kind == ' ': + continue + name = line[2:].rstrip() + self.populate_one(name, kind) + + def get_outcome(self, namekey): + return self._outcomes[namekey] + + +class GatherOutcome(object): + + def __init__(self, map): + self.map = map + self._failed = None + self._skipped = None + + @property + def failed(self): + if self._failed is None: + self._failed = set() + for prefix, outcome in self.map.items(): + self._failed.update([(prefix,)+ namekey for namekey in + outcome.failed]) + return self._failed + + @property + def skipped(self): + if self._skipped is None: + self._skipped = set() + for prefix, outcome in self.map.items(): + self._skipped.update([(prefix,) + namekey for namekey in + outcome.skipped]) + return self._skipped + + def get_outcome(self, namekey): + return self.map[namekey[0]].get_outcome(namekey[1:]) + + + +N = 10 + +class Summary(HtmlResource): + + def recentRevisions(self, request): + # xxx branches + status = self.getStatus(request) + revs = {} + for builderName in status.getBuilderNames(): + builderStatus = status.getBuilder(builderName) + for build in builderStatus.generateFinishedBuilds(num_builds=N): + rev = build.getProperty("got_revision") + revBuilds = revs.setdefault(rev, {}) + if builderName not in revBuilds: # pick the most recent or ? + # xxx hack, go through the steps and make sure + # the log is there + log = [log for log in build.getLogs() + if log.getName() == "pytestLog"][0] + revBuilds[builderName] = RevOutcome(log) + revs = revs.items() + revs.sort() + return revs + + def body(self, request): + revs = self.recentRevisions(request) + return repr(len(revs)) Added: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Sep 15 18:12:48 2008 @@ -0,0 +1,75 @@ +from pypybuildbot import summary +from StringIO import StringIO + +class TestOutcomes(object): + + def test_populate(self): + rev_outcome = summary.RevOutcome(50000) + log = StringIO("""F a/b.py:test_one +. a/b.py:test_two +s a/b.py:test_three +""") + + rev_outcome.populate(log) + + assert rev_outcome.skipped == set([("a.b","test_three")]) + assert rev_outcome.failed == set([("a.b", "test_one")]) + + res = rev_outcome.get_outcome(("a.b", "test_one")) + assert res == 'F' + + res = rev_outcome.get_outcome(("a.b", "test_three")) + assert res == 's' + + res = rev_outcome.get_outcome(("a.b", "test_two")) + assert res == '.' + + + def test_GatherOutcome(self): + rev_outcome_foo = summary.RevOutcome(50000) + log = StringIO("""F a/b.py:test_one +. a/b.py:test_two +s a/b.py:test_three +""") + + rev_outcome_foo.populate(log) + + + rev_outcome_bar = summary.RevOutcome(50000) + log = StringIO(""". a/b.py:test_one +. a/b.py:test_two +s a/b.py:test_three +""") + + rev_outcome_bar.populate(log) + + d = {'foo': rev_outcome_foo, + 'bar': rev_outcome_bar} + + goutcome = summary.GatherOutcome(d) + + + assert goutcome.failed == set([('foo', 'a.b', 'test_one')]) + assert goutcome.failed == set([('foo', 'a.b', 'test_one')]) + + assert goutcome.skipped == set([('foo', 'a.b', 'test_three'), + ('bar', 'a.b', 'test_three'), + ]) + assert goutcome.skipped == set([('foo', 'a.b', 'test_three'), + ('bar', 'a.b', 'test_three'), + ]) + + for prefix in ('foo', 'bar'): + for mod, testname in (("a.b", "test_one"), ("a.b", "test_two"), + ("a.b", "test_three")): + + outcome1 = d[prefix].get_outcome((mod, testname)) + outcome2 = goutcome.get_outcome((prefix, mod, testname)) + assert outcome2 == outcome1 + + goutcome_top = summary.GatherOutcome({'sub': goutcome}) + + assert goutcome_top.failed == set([('sub', 'foo', 'a.b', 'test_one')]) + + res = goutcome_top.get_outcome(('sub', 'foo', 'a.b', 'test_one')) + assert res == 'F' Added: pypy/build/bot2/slaveinfo.py ============================================================================== --- (empty file) +++ pypy/build/bot2/slaveinfo.py Mon Sep 15 18:12:48 2008 @@ -0,0 +1,3 @@ + +# Mapping from slave name to slave password +passwords = {} From pedronis at codespeak.net Mon Sep 15 18:27:59 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 15 Sep 2008 18:27:59 +0200 (CEST) Subject: [pypy-svn] r58169 - pypy/build/bot Message-ID: <20080915162759.596A016A387@codespeak.net> Author: pedronis Date: Mon Sep 15 18:27:57 2008 New Revision: 58169 Modified: pypy/build/bot/master.cfg Log: this was reconfig friendlier for us Modified: pypy/build/bot/master.cfg ============================================================================== --- pypy/build/bot/master.cfg (original) +++ pypy/build/bot/master.cfg Mon Sep 15 18:27:57 2008 @@ -15,7 +15,8 @@ def __init__(self, *a, **kw): Site.__init__(self, logPath='httpd.log', *a, **kw) from twisted.web import server -server.Site = LoggingSite +if server.Site.__name__ == 'Site': + server.Site = LoggingSite # So I did. status = WebStatus(httpPortNumber, allowForce=True) From arigo at codespeak.net Mon Sep 15 23:09:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 15 Sep 2008 23:09:54 +0200 (CEST) Subject: [pypy-svn] r58170 - pypy/dist/pypy/doc Message-ID: <20080915210954.D1C0C16A200@codespeak.net> Author: arigo Date: Mon Sep 15 23:09:51 2008 New Revision: 58170 Modified: pypy/dist/pypy/doc/getting-started.txt Log: The default level is really 2, for now. Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Mon Sep 15 23:09:51 2008 @@ -565,8 +565,8 @@ over one hour) and extremely RAM-hungry (kill it if it starts swapping heavily). If you have less than 1.5 GB of RAM (or a slow machine) you might want to pick the - `optimization level`_ `1` in the next step. The default - level `3` gives much better results, though. + `optimization level`_ `1` in the next step. A level of + `2` or `3` gives much better results, though. 3. Run:: From pedronis at codespeak.net Tue Sep 16 09:50:32 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 16 Sep 2008 09:50:32 +0200 (CEST) Subject: [pypy-svn] r58175 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20080916075032.3020316A219@codespeak.net> Author: pedronis Date: Tue Sep 16 09:50:29 2008 New Revision: 58175 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: (iko, pedronis) longer, clearer naming Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Sep 16 09:50:29 2008 @@ -2,7 +2,7 @@ from buildbot.status.web.base import HtmlResource # xxx caching? -class RevOutcome(object): +class RevisionOutcomeSet(object): def __init__(self, rev): self.rev = rev @@ -12,7 +12,7 @@ # xxx failure tracebacks def populate_one(self, name, shortrepr): - namekey = name.split(':', 1) + namekey = name.split(':', 1) # xxx not always the correct thing if namekey[0].endswith('.py'): namekey[0] = namekey[0][:-3].replace('/', '.') @@ -37,7 +37,7 @@ return self._outcomes[namekey] -class GatherOutcome(object): +class GatherOutcomeSet(object): def __init__(self, map): self.map = map @@ -85,7 +85,7 @@ # the log is there log = [log for log in build.getLogs() if log.getName() == "pytestLog"][0] - revBuilds[builderName] = RevOutcome(log) + revBuilds[builderName] = RevisionOutcomeSet(log) revs = revs.items() revs.sort() return revs Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Tue Sep 16 09:50:29 2008 @@ -4,49 +4,49 @@ class TestOutcomes(object): def test_populate(self): - rev_outcome = summary.RevOutcome(50000) + rev_outcome_set = summary.RevisionOutcomeSet(50000) log = StringIO("""F a/b.py:test_one . a/b.py:test_two s a/b.py:test_three """) - rev_outcome.populate(log) + rev_outcome_set.populate(log) - assert rev_outcome.skipped == set([("a.b","test_three")]) - assert rev_outcome.failed == set([("a.b", "test_one")]) + assert rev_outcome_set.skipped == set([("a.b","test_three")]) + assert rev_outcome_set.failed == set([("a.b", "test_one")]) - res = rev_outcome.get_outcome(("a.b", "test_one")) + res = rev_outcome_set.get_outcome(("a.b", "test_one")) assert res == 'F' - res = rev_outcome.get_outcome(("a.b", "test_three")) + res = rev_outcome_set.get_outcome(("a.b", "test_three")) assert res == 's' - res = rev_outcome.get_outcome(("a.b", "test_two")) + res = rev_outcome_set.get_outcome(("a.b", "test_two")) assert res == '.' - def test_GatherOutcome(self): - rev_outcome_foo = summary.RevOutcome(50000) + def test_GatherOutcomeSet(self): + rev_outcome_set_foo = summary.RevisionOutcomeSet(50000) log = StringIO("""F a/b.py:test_one . a/b.py:test_two s a/b.py:test_three """) - rev_outcome_foo.populate(log) + rev_outcome_set_foo.populate(log) - rev_outcome_bar = summary.RevOutcome(50000) + rev_outcome_set_bar = summary.RevisionOutcomeSet(50000) log = StringIO(""". a/b.py:test_one . a/b.py:test_two s a/b.py:test_three """) - rev_outcome_bar.populate(log) + rev_outcome_set_bar.populate(log) - d = {'foo': rev_outcome_foo, - 'bar': rev_outcome_bar} + d = {'foo': rev_outcome_set_foo, + 'bar': rev_outcome_set_bar} - goutcome = summary.GatherOutcome(d) + goutcome = summary.GatherOutcomeSet(d) assert goutcome.failed == set([('foo', 'a.b', 'test_one')]) @@ -67,7 +67,7 @@ outcome2 = goutcome.get_outcome((prefix, mod, testname)) assert outcome2 == outcome1 - goutcome_top = summary.GatherOutcome({'sub': goutcome}) + goutcome_top = summary.GatherOutcomeSet({'sub': goutcome}) assert goutcome_top.failed == set([('sub', 'foo', 'a.b', 'test_one')]) From pedronis at codespeak.net Tue Sep 16 11:08:28 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 16 Sep 2008 11:08:28 +0200 (CEST) Subject: [pypy-svn] r58176 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20080916090828.D0A4516A32F@codespeak.net> Author: pedronis Date: Tue Sep 16 11:08:25 2008 New Revision: 58176 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: (iko, pedronis) start displaying failures formatted like our old summaries Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Sep 16 11:08:25 2008 @@ -1,3 +1,5 @@ +import py +html = py.xml.html from buildbot.status.web.base import HtmlResource @@ -5,7 +7,7 @@ class RevisionOutcomeSet(object): def __init__(self, rev): - self.rev = rev + self.revision = rev self._outcomes = {} self.failed = set() self.skipped = set() @@ -43,6 +45,8 @@ self.map = map self._failed = None self._skipped = None + self.revision = map.values()[0].revision + @property def failed(self): @@ -65,10 +69,60 @@ def get_outcome(self, namekey): return self.map[namekey[0]].get_outcome(namekey[1:]) - +# ________________________________________________________________ N = 10 +def colsizes(namekeys): + colsizes = None + for keys in namekeys: + if colsizes is None: + colsizes = [0] * len(keys) + colsizes = map(max, zip(map(len, keys), colsizes)) + + return colsizes + + +class SummaryPage(object): + + def __init__(self): + self.sections = [] + + def add_section(self, outcome_sets): + by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set + in outcome_sets) + lines = [] + def bars(): + return ' |'*len(lines) + for rev, outcome_set in by_rev: + count_failures = len(outcome_set.failed) + count_skipped = len(outcome_set.skipped) + lines.append("%s %d" % (bars(),rev)) + lines.append(bars()) + + failed = set() + for rev, outcome_set in by_rev: + failed.update(outcome_set.failed) + + colwidths = colsizes(failed) + + for failure in sorted(failed): + line = [] + for rev, outcome_set in by_rev: + letter = outcome_set.get_outcome(failure) + line.append(" %s" % letter) + for width, key in zip(colwidths, failure): + line.append(" %-*s" % (width, key)) + lines.append(''.join(line)) + + section = html.pre('\n'.join(lines)) + self.sections.append(section) + + def render(self): + body_html = html.div(self.sections) + return body_html.unicode() + + class Summary(HtmlResource): def recentRevisions(self, request): @@ -78,18 +132,23 @@ for builderName in status.getBuilderNames(): builderStatus = status.getBuilder(builderName) for build in builderStatus.generateFinishedBuilds(num_builds=N): - rev = build.getProperty("got_revision") + rev = int(build.getProperty("got_revision")) revBuilds = revs.setdefault(rev, {}) if builderName not in revBuilds: # pick the most recent or ? # xxx hack, go through the steps and make sure # the log is there log = [log for log in build.getLogs() if log.getName() == "pytestLog"][0] - revBuilds[builderName] = RevisionOutcomeSet(log) - revs = revs.items() - revs.sort() + outcome_set = RevisionOutcomeSet(rev) + outcome_set.populate(log) + revBuilds[builderName] = outcome_set return revs def body(self, request): revs = self.recentRevisions(request) - return repr(len(revs)) + outcome_sets = [] + for rev, by_build in revs.items(): + outcome_sets.append(GatherOutcomeSet(by_build)) + page = SummaryPage() + page.add_section(outcome_sets) + return page.render() Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Tue Sep 16 11:08:25 2008 @@ -5,6 +5,9 @@ def test_populate(self): rev_outcome_set = summary.RevisionOutcomeSet(50000) + + assert rev_outcome_set.revision == 50000 + log = StringIO("""F a/b.py:test_one . a/b.py:test_two s a/b.py:test_three @@ -24,7 +27,6 @@ res = rev_outcome_set.get_outcome(("a.b", "test_two")) assert res == '.' - def test_GatherOutcomeSet(self): rev_outcome_set_foo = summary.RevisionOutcomeSet(50000) log = StringIO("""F a/b.py:test_one @@ -48,6 +50,7 @@ goutcome = summary.GatherOutcomeSet(d) + assert goutcome.revision == 50000 assert goutcome.failed == set([('foo', 'a.b', 'test_one')]) assert goutcome.failed == set([('foo', 'a.b', 'test_one')]) @@ -73,3 +76,12 @@ res = goutcome_top.get_outcome(('sub', 'foo', 'a.b', 'test_one')) assert res == 'F' + + def test_colsizes(self): + failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'), + ('ab', '', 'cd')] + + res = summary.colsizes(failed) + + assert res == [2,3,2] + From witulski at codespeak.net Tue Sep 16 14:08:21 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Tue, 16 Sep 2008 14:08:21 +0200 (CEST) Subject: [pypy-svn] r58177 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080916120821.A290016A541@codespeak.net> Author: witulski Date: Tue Sep 16 14:08:16 2008 New Revision: 58177 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: added some compares(greather, less, equal...) but no test passes with neg. nums Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Tue Sep 16 14:08:16 2008 @@ -1,6 +1,5 @@ from pypy.jit.codegen.x86_64.objmodel import Register8, Register64, Immediate8, Immediate32, Immediate64 - #Mapping from 64Bit-Register to coding (Rex.W or Rex.B , ModRM) REGISTER_MAP = { "rax": (0, 0), @@ -102,7 +101,7 @@ # rexW(1) = 64bitMode self.write_rex_byte(rexW, rexR, rexX, rexB) self.write(opcode) - if not tttn == None: + if not tttn == None: #write 0F9X byte = (9 << 4) | tttn self.write(chr(byte)) self.write_modRM_byte(mod, modrm2, modrm1) @@ -173,7 +172,12 @@ _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", 3, None, 0) _PUSH_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 6) + _SETE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,4) _SETG_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,15) + _SETGE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,13) + _SETL_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,12) + _SETLE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,14) + _SETNE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,5) _SUB_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x28", 3, None, None) _SUB_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 5) @@ -206,7 +210,8 @@ def JNE(self,op1): self.write("\x0F") self.write("\x85") - self.writeImm32(op1) + self.writeImm32(op1) + print self.tell(),": JNE to",op1 def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) @@ -236,6 +241,26 @@ method = getattr(self, "_SETG"+op1.to_string()) method(op1) + def SETL(self, op1): + method = getattr(self, "_SETL"+op1.to_string()) + method(op1) + + def SETGE(self, op1): + method = getattr(self, "_SETGE"+op1.to_string()) + method(op1) + + def SETLE(self, op1): + method = getattr(self, "_SETLE"+op1.to_string()) + method(op1) + + def SETE(self, op1): + method = getattr(self, "_SETE"+op1.to_string()) + method(op1) + + def SETNE(self, op1): + method = getattr(self, "_SETNE"+op1.to_string()) + method(op1) + def SUB(self, op1, op2): method = getattr(self, "_SUB"+op1.to_string()+op2.to_string()) method(op1, op2) @@ -252,7 +277,7 @@ def writeImm32(self, imm32): x = hex(imm32) if x[0]=='-': - x = self.cast_to_neg_hex(x) + x = self.cast_to_neg_hex(int(x,16)) # parse to string and cut "0x" off # fill with zeros if to short y = "0"*(10-len(x))+x[2:len(x)] @@ -263,8 +288,8 @@ self.write(chr(int(y[0:2],16))) - # TODO: sign extention? - # Parse the integervalue to an charakter + # TODO: sign extension? + # Parse the integervalue to an character # and write it def writeImm64(self, imm64): x = hex(imm64) @@ -293,5 +318,8 @@ # TODO: write me def cast_to_neg_hex(self,a_hex): - return a_hex + #FIXME: segfault to compliment + x = hex(18446744073709551616 +a_hex) + y = x[16:len(x)-1] + return y Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Tue Sep 16 14:08:16 2008 @@ -88,11 +88,13 @@ @specialize.arg(1) def genop1(self, opname, gv_arg): genmethod = getattr(self, 'op_' + opname) + print self.mc.tell(),":",opname return genmethod(gv_arg) @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): genmethod = getattr(self, 'op_' + opname) + print self.mc.tell(),":",opname return genmethod(gv_arg1, gv_arg2) op_int_add = make_two_argument_method("ADD") @@ -122,6 +124,36 @@ self.mc.SETG(Register8("al")) return Register64("rax") + def op_int_lt(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + gv_z = self.allocate_register("rax") + self.mc.SETL(Register8("al")) + return Register64("rax") + + def op_int_le(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + gv_z = self.allocate_register("rax") + self.mc.SETLE(Register8("al")) + return Register64("rax") + + def op_int_eq(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + gv_z = self.allocate_register("rax") + self.mc.SETE(Register8("al")) + return Register64("rax") + + def op_int_ne(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + gv_z = self.allocate_register("rax") + self.mc.SETNE(Register8("al")) + return Register64("rax") + + def op_int_ge(self, gv_x, gv_y): + self.mc.CMP(gv_x, gv_y) + gv_z = self.allocate_register("rax") + self.mc.SETGE(Register8("al")) + return Register64("rax") + def finish_and_return(self, sigtoken, gv_returnvar): #self.mc.write("\xB8\x0F\x00\x00\x00") self._open() @@ -136,6 +168,7 @@ gv_x = self.allocate_register() self.mc.MOV(gv_x,Immediate64(target.startaddr)) self.mc.JMP(gv_x) + print self.mc.tell(),": JMP to",target.startaddr self._close() def allocate_register(self, register=None): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Tue Sep 16 14:08:16 2008 @@ -10,12 +10,12 @@ def skip(self): py.test.skip("not implemented yet") -def make_cmp_gt(rgenop): +def make_cmp(rgenop, which_cmp): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) - builder, gv_cmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") + builder, gv_cmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "cmp") builder.start_writing() - gv_result = builder.genop2("int_gt", gv_x, gv_y) + gv_result = builder.genop2(which_cmp, gv_x, gv_y) builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_cmp @@ -117,7 +117,7 @@ def test_greater(self): rgenop = self.RGenOp() - cmp_function = make_cmp_gt(rgenop) + cmp_function = make_cmp(rgenop, "int_gt") fnptr = self.cast(cmp_function,2) res = fnptr(3,4) # 3>4? assert res == 0 # false @@ -126,6 +126,83 @@ res = fnptr(4,4) assert res == 0 res = fnptr(4,0) + assert res == 1 + res = fnptr(-4,0) + assert res == 0 + + def test_less(self): + rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_lt") + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3<4? + assert res == 1 # true + res = fnptr(4,3) + assert res == 0 + res = fnptr(4,4) + assert res == 0 + res = fnptr(4,0) + assert res == 0 + res = fnptr(-4,0) + assert res == 1 + + def test_less_or_equal(self): + rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_le") + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3<=4? + assert res == 1 # true + res = fnptr(4,3) + assert res == 0 + res = fnptr(4,4) + assert res == 1 + res = fnptr(4,0) + assert res == 0 + res = fnptr(-4,0) + assert res == 1 + + def test_greater_or_equal(self): + rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_ge") + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3>=4? + assert res == 0 # false + res = fnptr(4,3) + assert res == 1 + res = fnptr(4,4) + assert res == 1 + res = fnptr(4,0) + assert res == 1 + res = fnptr(-4,0) + assert res == 0 + + def test__equal(self): + rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_eq") + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3==4? + assert res == 0 # false + res = fnptr(4,3) + assert res == 0 + res = fnptr(4,4) + assert res == 1 + res = fnptr(4,0) + assert res == 0 + res = fnptr(-4,0) + assert res == 0 + + def test_not_equal(self): + rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_ne") + fnptr = self.cast(cmp_function,2) + res = fnptr(3,4) # 3!=4? + assert res == 1 # true + res = fnptr(4,3) + assert res == 1 + res = fnptr(4,4) + assert res == 0 + res = fnptr(4,0) + assert res == 1 + res = fnptr(-4,0) assert res == 1 # def test_push_and_pop(self): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Tue Sep 16 14:08:16 2008 @@ -51,6 +51,12 @@ ten = fp(-4, -6) assert ten == -10 print ten + four = fp(-4,0) + assert four == -4 + print four + four = fp(0,-4) + assert four == -4 + print four two = fp(-4, 6) assert two == 2 print two From pedronis at codespeak.net Tue Sep 16 16:14:23 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 16 Sep 2008 16:14:23 +0200 (CEST) Subject: [pypy-svn] r58178 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20080916141423.C859716A280@codespeak.net> Author: pedronis Date: Tue Sep 16 16:14:21 2008 New Revision: 58178 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: (iko, pedronis) basic summary with links to tracebacks is now working Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Sep 16 16:14:21 2008 @@ -1,22 +1,29 @@ +import urllib + import py html = py.xml.html from buildbot.status.web.base import HtmlResource -# xxx caching? class RevisionOutcomeSet(object): - def __init__(self, rev): + def __init__(self, rev, key=None): self.revision = rev + self.key = key self._outcomes = {} self.failed = set() self.skipped = set() - # xxx failure tracebacks + self.longreprs = {} - def populate_one(self, name, shortrepr): - namekey = name.split(':', 1) # xxx not always the correct thing - if namekey[0].endswith('.py'): - namekey[0] = namekey[0][:-3].replace('/', '.') + def populate_one(self, name, shortrepr, longrepr=None): + if shortrepr == '!': + namekey = [name, ''] + else: + namekey = name.split(':', 1) + if namekey[0].endswith('.py'): + namekey[0] = namekey[0][:-3].replace('/', '.') + if len(namekey) == 1: + namekey.append('') namekey = tuple(namekey) self._outcomes[namekey] = shortrepr @@ -27,17 +34,80 @@ else: self.failed.add(namekey) + if longrepr: + self.longreprs[namekey] = longrepr + def populate(self, log): + kind = None + def add_one(): + if kind is not None: + self.populate_one(name, kind, ''.join(longrepr)) for line in log.readlines(): - kind = line[0] - if kind == ' ': + first = line[0] + if first == ' ': + longrepr.append(line[1:]) continue + add_one() + kind = first name = line[2:].rstrip() - self.populate_one(name, kind) + longrepr = [] + add_one() def get_outcome(self, namekey): return self._outcomes[namekey] + def get_key_namekey(self, namekey): + return (self.key, namekey) + + def get_longrepr(self, namekey): + return self.longreprs.get(namekey, '') + +class RevisionOutcomeSetCache(object): + CACHESIZE = 10 + + def __init__(self): + self._outcome_sets = {} + self._lru = [] + + def _load_outcome_set(self, status, key): + builderName, buildNumber = key + builderStatus = status.getBuilder(builderName) + build = builderStatus.getBuild(buildNumber) + + rev = int(build.getProperty("got_revision")) + log = None + for step in build.getSteps(): + candLogs = [log for log in step.getLogs() + if log.getName() == "pytestLog"] + if candLogs: + log = candLogs[0] + break + + outcome_set = RevisionOutcomeSet(rev, key) + if log is None or not log.hasContents(): + outcome_set.populate_one('', '!', "no log from the test run") + else: + outcome_set.populate(log) + return outcome_set + + def get(self, status, key): + try: + self._lru.remove(key) + except ValueError: + pass + self._lru.append(key) + try: + return self._outcome_sets[key] + except KeyError: + pass + if len(self._lru) > self.CACHESIZE: + dead_key = self._lru.pop(0) + self._outcome_sets.pop(dead_key, None) + outcome_set = self._load_outcome_set(status, key) + self._outcome_sets[key] = outcome_set + return outcome_set + +outcome_set_cache = RevisionOutcomeSetCache() class GatherOutcomeSet(object): @@ -47,7 +117,6 @@ self._skipped = None self.revision = map.values()[0].revision - @property def failed(self): if self._failed is None: @@ -68,6 +137,12 @@ def get_outcome(self, namekey): return self.map[namekey[0]].get_outcome(namekey[1:]) + + def get_key_namekey(self, namekey): + return self.map[namekey[0]].get_key_namekey(namekey[1:]) + + def get_longrepr(self, namekey): + return self.map[namekey[0]].get_longrepr(namekey[1:]) # ________________________________________________________________ @@ -88,6 +163,17 @@ def __init__(self): self.sections = [] + def make_longrepr_url_for(self, outcome_set, namekey): + cachekey, namekey = outcome_set.get_key_namekey(namekey) + parms={ + 'builder': cachekey[0], + 'build': cachekey[1], + 'mod': namekey[0], + 'testname': namekey[1] + } + qs = urllib.urlencode(parms) + return "/summary/longrepr?" + qs + def add_section(self, outcome_sets): by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set in outcome_sets) @@ -97,9 +183,9 @@ for rev, outcome_set in by_rev: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - lines.append("%s %d" % (bars(),rev)) - lines.append(bars()) - + lines.append(["%s %d" % (bars(),rev), "\n"]) + lines.append([bars(), "\n"]) + failed = set() for rev, outcome_set in by_rev: failed.update(outcome_set.failed) @@ -110,20 +196,72 @@ line = [] for rev, outcome_set in by_rev: letter = outcome_set.get_outcome(failure) - line.append(" %s" % letter) + if outcome_set.get_longrepr(failure): + longrepr_url = self.make_longrepr_url_for(outcome_set, + failure) + line.append([" ",html.a(letter, href=longrepr_url)]) + else: + line.append(" %s" % letter) for width, key in zip(colwidths, failure): line.append(" %-*s" % (width, key)) - lines.append(''.join(line)) + lines.append(line) + lines.append("\n") - section = html.pre('\n'.join(lines)) + section = html.pre(lines) self.sections.append(section) def render(self): body_html = html.div(self.sections) return body_html.unicode() +class LongRepr(HtmlResource): + + def get_namekey(self, request): + mod = request.args.get('mod', []) + if not mod: + mod = None + else: + mod = mod[0] + + testname = request.args.get('testname', []) + if testname: + testname = testname[0] + else: + testname = '' + + return (mod, testname) + + def getTitle(self, request): + mod, testname = self.get_namekey(request) + if mod is None: + return "no such test" + return "%s %s" % (mod, testname) + + def body(self, request): + builder = request.args.get('builder', []) + build = request.args.get('build', []) + if not builder or not build: + return "no such build" + builderName = builder[0] + buildNumber = int(build[0]) + + outcome_set = outcome_set_cache.get(self.getStatus(request), + (builderName, + buildNumber)) + + namekey = self.get_namekey(request) + + longrepr = outcome_set.get_longrepr(namekey) + + return html.pre(longrepr).unicode() + class Summary(HtmlResource): + title="Summary" # xxx + + def __init__(self): + HtmlResource.__init__(self) + self.putChild('longrepr', LongRepr()) def recentRevisions(self, request): # xxx branches @@ -135,13 +273,10 @@ rev = int(build.getProperty("got_revision")) revBuilds = revs.setdefault(rev, {}) if builderName not in revBuilds: # pick the most recent or ? - # xxx hack, go through the steps and make sure - # the log is there - log = [log for log in build.getLogs() - if log.getName() == "pytestLog"][0] - outcome_set = RevisionOutcomeSet(rev) - outcome_set.populate(log) + key = (builderName, build.getNumber()) + outcome_set = outcome_set_cache.get(status, key) revBuilds[builderName] = outcome_set + return revs def body(self, request): Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Tue Sep 16 16:14:21 2008 @@ -4,9 +4,10 @@ class TestOutcomes(object): def test_populate(self): - rev_outcome_set = summary.RevisionOutcomeSet(50000) + rev_outcome_set = summary.RevisionOutcomeSet(50000, ('foo', 40)) assert rev_outcome_set.revision == 50000 + assert rev_outcome_set.key == ('foo', 40) log = StringIO("""F a/b.py:test_one . a/b.py:test_two @@ -20,24 +21,119 @@ res = rev_outcome_set.get_outcome(("a.b", "test_one")) assert res == 'F' + key_namekey = rev_outcome_set.get_key_namekey(("a.b", "test_one")) + assert key_namekey == (('foo', 40), ("a.b", "test_one")) res = rev_outcome_set.get_outcome(("a.b", "test_three")) assert res == 's' + key_namekey = rev_outcome_set.get_key_namekey(("a.b", "test_three")) + assert key_namekey == (('foo', 40), ("a.b", "test_three")) res = rev_outcome_set.get_outcome(("a.b", "test_two")) assert res == '.' + def test_populate_from_empty(self): + rev_outcome_set = summary.RevisionOutcomeSet(0) + log = StringIO("") + rev_outcome_set.populate(log) + + def test_populate_longrepr(self): + rev_outcome_set = summary.RevisionOutcomeSet(50000) + log = StringIO("""F a/b.py:test_one + some + traceback +. a/b.py:test_two +s a/b.py:test_three + some skip +""") + + rev_outcome_set.populate(log) + + assert len(rev_outcome_set.skipped) == 1 + assert len(rev_outcome_set.failed) == 1 + + assert rev_outcome_set.longreprs == { +("a.b","test_three"): "some skip\n", +("a.b", "test_one"): "some\ntraceback\n" + } + + res = rev_outcome_set.get_longrepr(("a.b", "test_two")) + assert res == '' + + res = rev_outcome_set.get_longrepr(("a.b", "test_one")) + assert res == "some\ntraceback\n" + + def test_populate_special(self): + rev_outcome_set = summary.RevisionOutcomeSet(50000) + log = StringIO("""F a/b.py +s a/c.py +! +! /a/b/c.py:92 +""") + + rev_outcome_set.populate(log) + + assert rev_outcome_set.failed == set([ + ("a.b", ''), + ("", ''), + ("/a/b/c.py:92", '')]) + + assert rev_outcome_set.skipped == set([ + ("a.c", '')]) + + + def test_RevisionOutcomeSetCache(self): + cache = summary.RevisionOutcomeSetCache() + cache.CACHESIZE = 3 + calls = [] + def load(x, y): + calls.append(y) + return y + + cache._load_outcome_set = load + + res = cache.get('status', 'a') + assert res == 'a' + cache.get('status', 'b') + cache.get('status', 'c') + + assert calls == ['a', 'b', 'c'] + + cache.get('status', 'a') + cache.get('status', 'b') + res = cache.get('status', 'c') + assert res == 'c' + + assert calls == ['a', 'b', 'c'] + + calls = [] + res = cache.get('status', 'd') + assert res == 'd' + assert cache.get('status', 'c') == 'c' + assert cache.get('status', 'b') == 'b' + assert calls == ['d'] + + res = cache.get('status', 'a') + assert res == 'a' + + assert calls == ['d', 'a'] + def test_GatherOutcomeSet(self): - rev_outcome_set_foo = summary.RevisionOutcomeSet(50000) + key_foo = ('foo', 3) + rev_outcome_set_foo = summary.RevisionOutcomeSet(50000, key_foo) log = StringIO("""F a/b.py:test_one + some + traceback . a/b.py:test_two s a/b.py:test_three """) rev_outcome_set_foo.populate(log) - - rev_outcome_set_bar = summary.RevisionOutcomeSet(50000) + + key_bar = ('bar', 7) + rev_outcome_set_bar = summary.RevisionOutcomeSet(50000, + key_bar) log = StringIO(""". a/b.py:test_one . a/b.py:test_two s a/b.py:test_three @@ -70,6 +166,13 @@ outcome2 = goutcome.get_outcome((prefix, mod, testname)) assert outcome2 == outcome1 + key_namekey1 = d[prefix].get_key_namekey((mod, testname)) + key_namekey2 = goutcome.get_key_namekey((prefix, mod, + testname)) + assert key_namekey1 == key_namekey2 + + + goutcome_top = summary.GatherOutcomeSet({'sub': goutcome}) assert goutcome_top.failed == set([('sub', 'foo', 'a.b', 'test_one')]) @@ -77,6 +180,12 @@ res = goutcome_top.get_outcome(('sub', 'foo', 'a.b', 'test_one')) assert res == 'F' + res = goutcome_top.get_key_namekey(('sub', 'foo', 'a.b', 'test_one')) + assert res == (key_foo, ('a.b', 'test_one')) + + res = goutcome_top.get_longrepr(('sub', 'foo', 'a.b', 'test_one')) + assert res == "some\ntraceback\n" + def test_colsizes(self): failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'), ('ab', '', 'cd')] From arigo at codespeak.net Tue Sep 16 18:31:39 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 16 Sep 2008 18:31:39 +0200 (CEST) Subject: [pypy-svn] r58180 - pypy/dist/pypy/lang/gameboy Message-ID: <20080916163139.9EE8016A0F6@codespeak.net> Author: arigo Date: Tue Sep 16 18:31:38 2008 New Revision: 58180 Modified: pypy/dist/pypy/lang/gameboy/video.py Log: untabbify. Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 16 18:31:38 2008 @@ -298,7 +298,7 @@ def activate(self, previous_mode): #if previous_mode.id() == 2: self.set_begin() - #else: + #else: # pass # #raise InvalidModeOrderException(self, previous_mode) @@ -338,14 +338,14 @@ 3: During Transfering Data to LCD Driver """ def __init__(self, video): - self.create_modes(video) + self.create_modes(video) self.reset() def create_modes(self, video): - self.mode0 = Mode0(video) - self.mode1 = Mode1(video) - self.mode2 = Mode2(video) - self.mode3 = Mode3(video) + self.mode0 = Mode0(video) + self.mode1 = Mode1(video) + self.mode2 = Mode2(video) + self.mode3 = Mode3(video) self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] From arigo at codespeak.net Tue Sep 16 18:57:37 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 16 Sep 2008 18:57:37 +0200 (CEST) Subject: [pypy-svn] r58181 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20080916165737.661C516A1D4@codespeak.net> Author: arigo Date: Tue Sep 16 18:57:35 2008 New Revision: 58181 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py Log: Fix for sandboxing. Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Tue Sep 16 18:57:35 2008 @@ -22,7 +22,7 @@ mh = mallocHelpers() mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size) c_realloc = rffi.llexternal('GC_REALLOC', [rffi.VOIDP, rffi.INT], - rffi.VOIDP) + rffi.VOIDP, sandboxsafe=True) def _realloc(ptr, size): return llmemory.cast_ptr_to_adr(c_realloc(rffi.cast(rffi.VOIDP, ptr), size)) mh.realloc = _realloc From arigo at codespeak.net Tue Sep 16 19:16:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 16 Sep 2008 19:16:53 +0200 (CEST) Subject: [pypy-svn] r58182 - in pypy/dist/pypy/interpreter/test: . demomixedmod mixedmodule Message-ID: <20080916171653.0B8BB16A0AB@codespeak.net> Author: arigo Date: Tue Sep 16 19:16:52 2008 New Revision: 58182 Added: pypy/dist/pypy/interpreter/test/demomixedmod/ - copied from r58164, pypy/dist/pypy/interpreter/test/mixedmodule/ Removed: pypy/dist/pypy/interpreter/test/mixedmodule/ Modified: pypy/dist/pypy/interpreter/test/test_appinterp.py Log: Rename this to avoid confusion with the pypy.mixedmodule module. Modified: pypy/dist/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/dist/pypy/interpreter/test/test_appinterp.py Tue Sep 16 19:16:52 2008 @@ -111,9 +111,9 @@ class TestMixedModule: def test_accesses(self): - space = self.space - import mixedmodule - w_module = mixedmodule.Module(space, space.wrap('mixedmodule')) + space = self.space + import demomixedmod + w_module = demomixedmod.Module(space, space.wrap('mixedmodule')) space.appexec([w_module], """ (module): assert module.value is None From arigo at codespeak.net Tue Sep 16 19:28:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 16 Sep 2008 19:28:54 +0200 (CEST) Subject: [pypy-svn] r58183 - in pypy/dist/pypy/interpreter: . test Message-ID: <20080916172854.5524916A0C9@codespeak.net> Author: arigo Date: Tue Sep 16 19:28:53 2008 New Revision: 58183 Modified: pypy/dist/pypy/interpreter/mixedmodule.py pypy/dist/pypy/interpreter/test/test_appinterp.py Log: Test and fix. Modified: pypy/dist/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/dist/pypy/interpreter/mixedmodule.py (original) +++ pypy/dist/pypy/interpreter/mixedmodule.py Tue Sep 16 19:28:53 2008 @@ -19,6 +19,7 @@ Module.__init__(self, space, w_name) self.lazy = True self.__class__.buildloaders() + self.loaders = self.loaders.copy() # copy from the class to the inst def get_applevel_name(cls): """ NOT_RPYTHON """ Modified: pypy/dist/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/dist/pypy/interpreter/test/test_appinterp.py Tue Sep 16 19:28:53 2008 @@ -132,3 +132,30 @@ assert name in module.__dict__ """) assert space.is_true(w_module.call('somefunc')) + + def test_whacking_at_loaders(self): + """Some MixedModules change 'self.loaders' in __init__(), but doing + so they incorrectly mutated a class attribute. 'loaders' is now a + per-instance attribute, holding a fresh copy of the dictionary. + """ + from pypy.interpreter.mixedmodule import MixedModule + from pypy.conftest import maketestobjspace + + class MyModule(MixedModule): + interpleveldefs = {} + appleveldefs = {} + def __init__(self, space, w_name): + def loader(myspace): + assert myspace is space + return myspace.wrap("hello") + MixedModule.__init__(self, space, w_name) + self.loaders["hi"] = loader + + space1 = self.space + w_mymod1 = MyModule(space1, space1.wrap('mymod')) + + space2 = maketestobjspace() + w_mymod2 = MyModule(space2, space2.wrap('mymod')) + + w_str = space1.getattr(w_mymod1, space1.wrap("hi")) + assert space1.str_w(w_str) == "hello" From pedronis at codespeak.net Tue Sep 16 22:35:23 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 16 Sep 2008 22:35:23 +0200 (CEST) Subject: [pypy-svn] r58184 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080916203523.18929169ECC@codespeak.net> Author: pedronis Date: Tue Sep 16 22:35:21 2008 New Revision: 58184 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: info for iko, pedronis Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Tue Sep 16 22:35:21 2008 @@ -16,6 +16,8 @@ Samuele Pedroni Carl Friedrich Bolz 3rd Oct - ??? my flat Alexander Schremmer 5-15 possible +Samuele Pedroni 5-13 Hotel Blaettler +Anders Hammarqvist 5-13 Hotel Blaettler ==================== ============== ============================ People on the following list were present at previous sprints: From pedronis at codespeak.net Wed Sep 17 10:02:12 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 17 Sep 2008 10:02:12 +0200 (CEST) Subject: [pypy-svn] r58189 - pypy/build/bot2/pypybuildbot Message-ID: <20080917080212.84310169FAB@codespeak.net> Author: pedronis Date: Wed Sep 17 10:02:09 2008 New Revision: 58189 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: (iko, pedronis) provide some links to the per builder runs stdio logs Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Sep 17 10:02:09 2008 @@ -7,13 +7,14 @@ class RevisionOutcomeSet(object): - def __init__(self, rev, key=None): + def __init__(self, rev, key=None, run_stdio=None): self.revision = rev self.key = key self._outcomes = {} self.failed = set() self.skipped = set() self.longreprs = {} + self._run_stdio = run_stdio def populate_one(self, name, shortrepr, longrepr=None): if shortrepr == '!': @@ -62,6 +63,9 @@ def get_longrepr(self, namekey): return self.longreprs.get(namekey, '') + def get_run_stdios(self): + return {self.key: self._run_stdio} + class RevisionOutcomeSetCache(object): CACHESIZE = 10 @@ -75,19 +79,22 @@ build = builderStatus.getBuild(buildNumber) rev = int(build.getProperty("got_revision")) - log = None + pytest_log = None + stdio_url = "no_log" for step in build.getSteps(): - candLogs = [log for log in step.getLogs() - if log.getName() == "pytestLog"] - if candLogs: - log = candLogs[0] + logs = dict((log.getName(), log) for log in step.getLogs()) + if 'pytestLog' in logs: + pytest_log = logs['pytestLog'] + stdio_url = status.getURLForThing(logs['stdio']) + # builbot is broken in this :( + stdio_url = stdio_url[:-1]+"stdio" break - outcome_set = RevisionOutcomeSet(rev, key) - if log is None or not log.hasContents(): + outcome_set = RevisionOutcomeSet(rev, key, stdio_url) + if pytest_log is None or not pytest_log.hasContents(): outcome_set.populate_one('', '!', "no log from the test run") else: - outcome_set.populate(log) + outcome_set.populate(pytest_log) return outcome_set def get(self, status, key): @@ -143,6 +150,12 @@ def get_longrepr(self, namekey): return self.map[namekey[0]].get_longrepr(namekey[1:]) + + def get_run_stdios(self): + all = {} + for outcome_set in self.map.itervalues(): + all.update(outcome_set.get_run_stdios()) + return all # ________________________________________________________________ @@ -174,6 +187,15 @@ qs = urllib.urlencode(parms) return "/summary/longrepr?" + qs + def make_stdio_anchors_for(self, outcome_set): + anchors = [] + stdios = sorted(outcome_set.get_run_stdios().items()) + for cachekey, url in stdios: + builder = cachekey[0] + anchors.append(' ') + anchors.append(html.a(builder, href=url)) + return anchors + def add_section(self, outcome_sets): by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set in outcome_sets) @@ -183,7 +205,10 @@ for rev, outcome_set in by_rev: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - lines.append(["%s %d" % (bars(),rev), "\n"]) + line = ["%s %d" % (bars(),rev)] + line.append(self.make_stdio_anchors_for(outcome_set)) + line.append('\n') + lines.append(line) lines.append([bars(), "\n"]) failed = set() From arigo at codespeak.net Wed Sep 17 11:53:48 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 17 Sep 2008 11:53:48 +0200 (CEST) Subject: [pypy-svn] r58194 - pypy/dist/pypy/doc/jit Message-ID: <20080917095348.172C31683F0@codespeak.net> Author: arigo Date: Wed Sep 17 11:53:46 2008 New Revision: 58194 Modified: pypy/dist/pypy/doc/jit/overview.txt Log: Typo. Modified: pypy/dist/pypy/doc/jit/overview.txt ============================================================================== --- pypy/dist/pypy/doc/jit/overview.txt (original) +++ pypy/dist/pypy/doc/jit/overview.txt Wed Sep 17 11:53:46 2008 @@ -146,7 +146,7 @@ instance, Sun Microsystems is investing in JRuby, which aims to use the Java Hotspot JIT to improve the performance of Ruby. However, this requires a lot of hand crafting and will only provide speedups for one language on one platform. -Some issues are delicate, e.g., how to remove the overhead of constant boxing +Some issues are delicate, e.g., how to remove the overhead of constantly boxing and unboxing, typical in dynamic languages. An advantage compared to PyPy is that there are some hand optimizations that can be performed, that do not fit in the metaprogramming approach. But metaprogramming makes the PyPy JIT From arigo at codespeak.net Wed Sep 17 11:55:12 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 17 Sep 2008 11:55:12 +0200 (CEST) Subject: [pypy-svn] r58195 - pypy/dist/pypy/doc/jit Message-ID: <20080917095512.EC267169E1C@codespeak.net> Author: arigo Date: Wed Sep 17 11:55:10 2008 New Revision: 58195 Modified: pypy/dist/pypy/doc/jit/overview.txt Log: Another typo. Modified: pypy/dist/pypy/doc/jit/overview.txt ============================================================================== --- pypy/dist/pypy/doc/jit/overview.txt (original) +++ pypy/dist/pypy/doc/jit/overview.txt Wed Sep 17 11:55:10 2008 @@ -57,8 +57,8 @@ re-implementations are not faster (and cannot run all existing Python applications). -Partical results ----------------- +Practical results +----------------- The JIT compilers that we generate use some techniques that are not in widespread use so far, but they are not exactly new either. The point From pedronis at codespeak.net Wed Sep 17 19:07:41 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 17 Sep 2008 19:07:41 +0200 (CEST) Subject: [pypy-svn] r58208 - in pypy/build/bot2: . pypybuildbot Message-ID: <20080917170741.58677169F66@codespeak.net> Author: pedronis Date: Wed Sep 17 19:07:39 2008 New Revision: 58208 Added: pypy/build/bot2/pypybuildbot/steps.py (contents, props changed) Modified: pypy/build/bot2/master.cfg pypy/build/bot2/pypybuildbot/master.py Log: (iko, pedronis) - start of supporting windows setup - svnwcrevert invocation Modified: pypy/build/bot2/master.cfg ============================================================================== --- pypy/build/bot2/master.cfg (original) +++ pypy/build/bot2/master.cfg Wed Sep 17 19:07:39 2008 @@ -4,7 +4,9 @@ httpPortNumber = 8099 # slavename -> password -from slaveinfo import passwords +import slaveinfo +reload(slaveinfo) +passwords = slaveinfo.passwords # checkout bot2 in the home dir of the master sys.path.append(os.path.expanduser('~/bot2/')) Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Sep 17 19:07:39 2008 @@ -2,9 +2,6 @@ from buildbot.buildslave import BuildSlave from buildbot.status.html import WebStatus -from buildbot.process import factory -from buildbot.steps import source, shell - # I really wanted to pass logPath to Site from twisted.web.server import Site @@ -18,19 +15,20 @@ status = WebStatus(httpPortNumber, allowForce=True) -pypyOwnFactory = factory.BuildFactory() -pypyOwnFactory.addStep(source.SVN("https://codespeak.net/svn/pypy/branch/pypy-pytrunk")) -pypyOwnFactory.addStep(shell.ShellCommand( - description="pytest", - command="py/bin/py.test pypy/module/__builtin__ pypy/module/operator --session=FileLogSession --filelog=pytest.log".split(), - logfiles={'pytestLog': 'pytest.log'})) +import pypybuildbot.steps +reload(pypybuildbot.steps) +pypysteps = pypybuildbot.steps + +pypyOwnTestFactory = pypysteps.PyPyOwnTestFactory() +pypyOwnTestFactoryWin = pypysteps.PyPyOwnTestFactory(platform="win32") BuildmasterConfig = { 'slavePortnum': slavePortnum, 'change_source': [], 'schedulers': [Nightly("nightly", - ["pypy-own-linux", "pypy-own-other-linux"], hour=19)], + ["pypy-own-linux", "pypy-own-other-linux", + "pypy-own-win"], hour=19)], 'status': [status], 'slaves': [BuildSlave(name, password) @@ -41,13 +39,17 @@ {"name": "pypy-own-linux", "slavenames": ["vitaly"], "builddir": "pypy-own-linux", - "factory": pypyOwnFactory + "factory": pypyOwnTestFactory }, {"name": "pypy-own-other-linux", "slavenames": ["fido"], "builddir": "pypy-own-other-linux", - "factory": pypyOwnFactory - } + "factory": pypyOwnTestFactory + }, + {"name": "pypy-own-win", + "slavenames": ['ebgoc'], + "builddir": "pypy-own-win", + "factory": pypyOwnTestFactoryWin} ], 'buildbotURL': 'http://localhost:%d/' % (httpPortNumber,), Added: pypy/build/bot2/pypybuildbot/steps.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/steps.py Wed Sep 17 19:07:39 2008 @@ -0,0 +1,65 @@ +from buildbot.process import factory +from buildbot.steps import source, shell +from buildbot.status.builder import SUCCESS + + +class FirstTime(shell.SetProperty): + + def __init__(self, **kwds): + shell.SetProperty.__init__(self, description="first-time", + property="first-time") + + +class PosixFirstTime(FirstTime): + command = "test -d pypy || echo yes" + +class WindowsFirstTime(FirstTime): + command = "if not exist pypy echo yes" + + +class CondShellCommand(shell.ShellCommand): + + def __init__(self, **kwds): + shell.ShellCommand.__init__(self, **kwds) + self.cond = kwds.get('cond', lambda props: True) + + def start(self): + props = self.build.getProperties() + yes = self.cond(props) + if yes: + shell.ShellCommand.start(self) + else: + self.setStatus(None, SUCCESS) + self.finished(SUCCESS) + +# ________________________________________________________________ + +def not_first_time(props): + first_time = props.getProperty("first-time") + return not first_time + +class PyPyOwnTestFactory(factory.BuildFactory): + + def __init__(self, *a, **kw): + platform = kw.pop('platform', 'linux') + factory.BuildFactory.__init__(self, *a, **kw) + + if platform == "win32": + first_time_check = WindowsFirstTime() + else: + first_time_check = PosixFirstTime() + + self.addStep(first_time_check) + self.addStep(CondShellCommand( + description="wcrevert", + cond=not_first_time, + command = ["python", "py/bin/py.svnwcrevert", "."], + haltOnFailure=True)) + self.addStep(source.SVN("https://codespeak.net/svn/pypy/" + "branch/pypy-pytrunk")) + self.addStep(shell.ShellCommand( + description="pytest", + command=["python", "py/bin/py.test", + "pypy/module/__builtin__", "pypy/module/operator", + "--session=FileLogSession", "--filelog=pytest.log"], + logfiles={'pytestLog': 'pytest.log'})) From hpk at codespeak.net Wed Sep 17 21:37:12 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 17 Sep 2008 21:37:12 +0200 (CEST) Subject: [pypy-svn] r58209 - pypy/build/benchmem Message-ID: <20080917193712.46BCE16840F@codespeak.net> Author: hpk Date: Wed Sep 17 21:37:10 2008 New Revision: 58209 Added: pypy/build/benchmem/ pypy/build/benchmem/benchtool.py (contents, props changed) pypy/build/benchmem/microbench.py - copied, changed from r58176, pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Log: beginnings of memory benchmark tool and benchmarks (motivated but not yet fully incorporating what fijal did in the cross-compilation branch) Added: pypy/build/benchmem/benchtool.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchtool.py Wed Sep 17 21:37:10 2008 @@ -0,0 +1,93 @@ +import py +import os, sys +import microbench +class BenchRunner(object): + def __init__(self, executable, resultdir): + self.resultdir = py.path.local(resultdir) + self.executable = executable + + def log(self, *args): + print " ".join(map(str, args)) + + def _preparebench(self, name, args): + arglist = ",".join(map(str, args)) + source = py.code.Source(microbench, """ + def checkpoint(): + sys.stdout.write(".") + sys.stdin.read(1) + if __name__ == "__main__": + import os, sys + pid = os.getpid() + sys.stdout.write(str(pid)) + sys.stdout.write("\\n") + %s(checkpoint, %s) + """ %(name, arglist)) + p = self.resultdir.join("%s.py" %(name,)) + p.write(source) + return p + + def run_checkpointed_bench(self, name, args): + benchpyfile = self._preparebench(name, args) + self.log("created", benchpyfile) + cmd = "%s -u %s" %(self.executable, benchpyfile) + self.log("exec", cmd) + stdout, stdin = os.popen2(cmd) + pid = int(stdin.readline()) + logpath = self.resultdir.join("%s.smaps" %(name,)) + smaps = SmapsRecorder(pid, logpath) + stdout.write(".") + stdout.flush() + while not stdin.closed: + c = stdin.read(1) + if not c: + break + smaps.snapshot() + stdout.write(".") + stdout.flush() + self.log("finished", cmd) + +class SmapsRecorder: + SEPLINE = "="*80 + "\n" + + def __init__(self, pid, logpath): + self.logpath = py.path.local(logpath) + self._file = self.logpath.open("a") + self.pid = pid + self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) + assert self.smapspath.check() + self.snapshot() + + def snapshot(self): + s = self.smapspath.read() + self._file.write(s) + self._file.write("\n") + self._file.write(self.SEPLINE) + self._file.flush() + +if __name__ == '__main__': + pass + +# tests + +def setup_module(mod): + if sys.platform.find("linux") == -1: + py.test.skip("linux required") + mod.tmpdir = py.test.ensuretemp(mod.__name__) + +def test_smapsrecorder(): + tmpdir = py.test.ensuretemp("smapsrecorder") + logpath = tmpdir.join("logfile") + pid = os.getpid() + rec = SmapsRecorder(pid=pid, logpath=logpath) + s = logpath.read() + assert s.count(SmapsRecorder.SEPLINE) == 1 + rec.snapshot() + rec.snapshot() + del rec + s = logpath.read() + assert s.count(SmapsRecorder.SEPLINE) == 3 + +def test_benchrunner(): + tmpdir = py.test.ensuretemp("benchrunner") + runner = BenchRunner(executable="python2.5", resultdir=tmpdir) + smapsfile = runner.run_checkpointed_bench("create_recursive_tuples", (10, 10)) Copied: pypy/build/benchmem/microbench.py (from r58176, pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py) ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py (original) +++ pypy/build/benchmem/microbench.py Wed Sep 17 21:37:10 2008 @@ -1,59 +1,34 @@ -#!/usr/bin/env python -import autopath -""" This file attempts to measure how much memory is taken by various objects. +""" + this file contains microbenchmarks. """ -from pypy.translator.benchmark.bench_mem import measure, smaps_measure_func -import py - -def measure_func(num, pid): - res = smaps_measure_func(pid) - print num, res.private - -def tuples(read, write, coeff): - import gc +def create_recursive_tuples(checkpoint, iter1, iter2): x = () - for i in range(1000 * coeff): - x = (x,) - gc.collect() - write('x') - read() - write('e') - -def linked_list(read, write, coeff): - import gc + for i in range(iter1): + checkpoint() + for j in range(iter2): + x = (x,) + +class A(object): + def __init__(self, other): + self.other = other +def linked_list(checkpoint, iter1, iter2): class A(object): def __init__(self, other): self.other = other x = None - for i in range(1000 * coeff): - x = A(x) - gc.collect() - write('x') - read() - write('e') - -def list_of_instances_with_int(read, write, coeff): - import gc + for i in range(iter1): + checkpoint() + for j in range(iter2): + x = A(x) +def list_of_instances_with_int(checkpoint, iter1, iter2): class A(object): def __init__(self, x): self.x = x - - x = [A(i) for i in range(1000 * coeff)] - gc.collect() - write('x') - read() - write('e') - -if __name__ == '__main__': - coeff = 1 - i = 0 - funcs = [] - while i < 10: - funcs.append(lambda r, w, coeff=coeff: list_of_instances_with_int(r, w, coeff)) - coeff *= 2 - i += 1 - res = measure(measure_func, funcs) + for i in range(iter1): + checkpoint() + for j in range(iter2): + x = A(x) From hpk at codespeak.net Wed Sep 17 22:21:16 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 17 Sep 2008 22:21:16 +0200 (CEST) Subject: [pypy-svn] r58210 - in pypy/build/benchmem: . benchmark testing Message-ID: <20080917202116.97C2E16A041@codespeak.net> Author: hpk Date: Wed Sep 17 22:21:14 2008 New Revision: 58210 Added: pypy/build/benchmem/benchmark/ pypy/build/benchmem/benchmark/create_recursive_tuples.py - copied, changed from r58209, pypy/build/benchmem/microbench.py pypy/build/benchmem/benchmark/list_of_instances_with_ints.py - copied, changed from r58209, pypy/build/benchmem/microbench.py pypy/build/benchmem/benchmark/simple_linked_instances.py - copied, changed from r58209, pypy/build/benchmem/microbench.py pypy/build/benchmem/testing/ pypy/build/benchmem/testing/__init__.py pypy/build/benchmem/testing/test_benchtool.py - copied, changed from r58209, pypy/build/benchmem/benchtool.py Removed: pypy/build/benchmem/microbench.py Modified: pypy/build/benchmem/benchtool.py Log: cleanups, move benchmark to own directory, create testing dir Copied: pypy/build/benchmem/benchmark/create_recursive_tuples.py (from r58209, pypy/build/benchmem/microbench.py) ============================================================================== --- pypy/build/benchmem/microbench.py (original) +++ pypy/build/benchmem/benchmark/create_recursive_tuples.py Wed Sep 17 22:21:14 2008 @@ -1,34 +1,8 @@ -""" - this file contains microbenchmarks. -""" -def create_recursive_tuples(checkpoint, iter1, iter2): +def bench_create_recursive_tuples(checkpoint, iter1, iter2): x = () for i in range(iter1): checkpoint() for j in range(iter2): x = (x,) -class A(object): - def __init__(self, other): - self.other = other - -def linked_list(checkpoint, iter1, iter2): - class A(object): - def __init__(self, other): - self.other = other - - x = None - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = A(x) - -def list_of_instances_with_int(checkpoint, iter1, iter2): - class A(object): - def __init__(self, x): - self.x = x - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = A(x) Copied: pypy/build/benchmem/benchmark/list_of_instances_with_ints.py (from r58209, pypy/build/benchmem/microbench.py) ============================================================================== --- pypy/build/benchmem/microbench.py (original) +++ pypy/build/benchmem/benchmark/list_of_instances_with_ints.py Wed Sep 17 22:21:14 2008 @@ -1,34 +1,12 @@ -""" - this file contains microbenchmarks. -""" - -def create_recursive_tuples(checkpoint, iter1, iter2): - x = () - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = (x,) class A(object): - def __init__(self, other): - self.other = other - -def linked_list(checkpoint, iter1, iter2): - class A(object): - def __init__(self, other): - self.other = other - - x = None - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = A(x) + def __init__(self, x, y): + self.x = x + self.y = y -def list_of_instances_with_int(checkpoint, iter1, iter2): - class A(object): - def __init__(self, x): - self.x = x +def bench_list_of_instances_with_ints(checkpoint, iter1, iter2): + l = [] for i in range(iter1): checkpoint() for j in range(iter2): - x = A(x) + l.append(A(i, j)) Copied: pypy/build/benchmem/benchmark/simple_linked_instances.py (from r58209, pypy/build/benchmem/microbench.py) ============================================================================== --- pypy/build/benchmem/microbench.py (original) +++ pypy/build/benchmem/benchmark/simple_linked_instances.py Wed Sep 17 22:21:14 2008 @@ -1,34 +1,10 @@ -""" - this file contains microbenchmarks. -""" - -def create_recursive_tuples(checkpoint, iter1, iter2): - x = () - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = (x,) - class A(object): def __init__(self, other): self.other = other -def linked_list(checkpoint, iter1, iter2): - class A(object): - def __init__(self, other): - self.other = other - +def bench_linked_list(checkpoint, iter1, iter2): x = None for i in range(iter1): checkpoint() for j in range(iter2): x = A(x) - -def list_of_instances_with_int(checkpoint, iter1, iter2): - class A(object): - def __init__(self, x): - self.x = x - for i in range(iter1): - checkpoint() - for j in range(iter2): - x = A(x) Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Wed Sep 17 22:21:14 2008 @@ -1,6 +1,8 @@ import py import os, sys -import microbench +mydir = py.magic.autopath().dirpath() +benchmarkdir = mydir.join("benchmark") + class BenchRunner(object): def __init__(self, executable, resultdir): self.resultdir = py.path.local(resultdir) @@ -9,9 +11,19 @@ def log(self, *args): print " ".join(map(str, args)) - def _preparebench(self, name, args): + def getbenchpath(self, filename): + return benchmarkdir.join(filename) + + def _preparebench(self, filename, args): + path = self.getbenchpath(filename) + for name, obj in vars(path.pyimport()).items(): + if name.startswith("bench") and callable(obj): + break + else: + raise LookupError("no benchmark found in %s" %(path,)) + arglist = ",".join(map(str, args)) - source = py.code.Source(microbench, """ + source = py.code.Source(path.read(), """ def checkpoint(): sys.stdout.write(".") sys.stdin.read(1) @@ -26,14 +38,14 @@ p.write(source) return p - def run_checkpointed_bench(self, name, args): - benchpyfile = self._preparebench(name, args) + def run_checkpointed_bench(self, filename, args): + benchpyfile = self._preparebench(filename, args) self.log("created", benchpyfile) cmd = "%s -u %s" %(self.executable, benchpyfile) self.log("exec", cmd) stdout, stdin = os.popen2(cmd) pid = int(stdin.readline()) - logpath = self.resultdir.join("%s.smaps" %(name,)) + logpath = benchpyfile.new(ext="smaps") smaps = SmapsRecorder(pid, logpath) stdout.write(".") stdout.flush() @@ -45,6 +57,7 @@ stdout.write(".") stdout.flush() self.log("finished", cmd) + return logpath class SmapsRecorder: SEPLINE = "="*80 + "\n" @@ -67,27 +80,3 @@ if __name__ == '__main__': pass -# tests - -def setup_module(mod): - if sys.platform.find("linux") == -1: - py.test.skip("linux required") - mod.tmpdir = py.test.ensuretemp(mod.__name__) - -def test_smapsrecorder(): - tmpdir = py.test.ensuretemp("smapsrecorder") - logpath = tmpdir.join("logfile") - pid = os.getpid() - rec = SmapsRecorder(pid=pid, logpath=logpath) - s = logpath.read() - assert s.count(SmapsRecorder.SEPLINE) == 1 - rec.snapshot() - rec.snapshot() - del rec - s = logpath.read() - assert s.count(SmapsRecorder.SEPLINE) == 3 - -def test_benchrunner(): - tmpdir = py.test.ensuretemp("benchrunner") - runner = BenchRunner(executable="python2.5", resultdir=tmpdir) - smapsfile = runner.run_checkpointed_bench("create_recursive_tuples", (10, 10)) Added: pypy/build/benchmem/testing/__init__.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/testing/__init__.py Wed Sep 17 22:21:14 2008 @@ -0,0 +1 @@ +# Copied: pypy/build/benchmem/testing/test_benchtool.py (from r58209, pypy/build/benchmem/benchtool.py) ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Wed Sep 17 22:21:14 2008 @@ -1,78 +1,11 @@ import py import os, sys -import microbench -class BenchRunner(object): - def __init__(self, executable, resultdir): - self.resultdir = py.path.local(resultdir) - self.executable = executable - def log(self, *args): - print " ".join(map(str, args)) - - def _preparebench(self, name, args): - arglist = ",".join(map(str, args)) - source = py.code.Source(microbench, """ - def checkpoint(): - sys.stdout.write(".") - sys.stdin.read(1) - if __name__ == "__main__": - import os, sys - pid = os.getpid() - sys.stdout.write(str(pid)) - sys.stdout.write("\\n") - %s(checkpoint, %s) - """ %(name, arglist)) - p = self.resultdir.join("%s.py" %(name,)) - p.write(source) - return p - - def run_checkpointed_bench(self, name, args): - benchpyfile = self._preparebench(name, args) - self.log("created", benchpyfile) - cmd = "%s -u %s" %(self.executable, benchpyfile) - self.log("exec", cmd) - stdout, stdin = os.popen2(cmd) - pid = int(stdin.readline()) - logpath = self.resultdir.join("%s.smaps" %(name,)) - smaps = SmapsRecorder(pid, logpath) - stdout.write(".") - stdout.flush() - while not stdin.closed: - c = stdin.read(1) - if not c: - break - smaps.snapshot() - stdout.write(".") - stdout.flush() - self.log("finished", cmd) - -class SmapsRecorder: - SEPLINE = "="*80 + "\n" - - def __init__(self, pid, logpath): - self.logpath = py.path.local(logpath) - self._file = self.logpath.open("a") - self.pid = pid - self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) - assert self.smapspath.check() - self.snapshot() - - def snapshot(self): - s = self.smapspath.read() - self._file.write(s) - self._file.write("\n") - self._file.write(self.SEPLINE) - self._file.flush() - -if __name__ == '__main__': - pass - -# tests +from benchtool import SmapsRecorder, BenchRunner, benchmarkdir def setup_module(mod): if sys.platform.find("linux") == -1: py.test.skip("linux required") - mod.tmpdir = py.test.ensuretemp(mod.__name__) def test_smapsrecorder(): tmpdir = py.test.ensuretemp("smapsrecorder") @@ -90,4 +23,11 @@ def test_benchrunner(): tmpdir = py.test.ensuretemp("benchrunner") runner = BenchRunner(executable="python2.5", resultdir=tmpdir) - smapsfile = runner.run_checkpointed_bench("create_recursive_tuples", (10, 10)) + def checker(name, *args): + smapsfile = runner.run_checkpointed_bench(name, args) + assert smapsfile.check() + + for name in benchmarkdir.listdir("*.py"): + if name.basename[0] != "_": + yield checker, name.basename, 10, 10 + From hpk at codespeak.net Wed Sep 17 22:30:05 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 17 Sep 2008 22:30:05 +0200 (CEST) Subject: [pypy-svn] r58212 - in pypy/build/benchmem: . benchmark testing Message-ID: <20080917203005.08A72169FD2@codespeak.net> Author: hpk Date: Wed Sep 17 22:30:05 2008 New Revision: 58212 Modified: pypy/build/benchmem/ (props changed) pypy/build/benchmem/benchmark/ (props changed) pypy/build/benchmem/testing/ (props changed) pypy/build/benchmem/testing/__init__.py (props changed) Log: FIXEOL From pedronis at codespeak.net Wed Sep 17 23:32:50 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 17 Sep 2008 23:32:50 +0200 (CEST) Subject: [pypy-svn] r58213 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20080917213250.E515C16A0B4@codespeak.net> Author: pedronis Date: Wed Sep 17 23:32:49 2008 New Revision: 58213 Modified: pypy/build/bot2/pypybuildbot/steps.py pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: fixes: - preserve .buildbot-sourcedata across svnwcrevert becaue otherwise a full checkout will happen - fix problems with absent results - fix problems with builds that even didn't get as far as checking out (they have no revision) Modified: pypy/build/bot2/pypybuildbot/steps.py ============================================================================== --- pypy/build/bot2/pypybuildbot/steps.py (original) +++ pypy/build/bot2/pypybuildbot/steps.py Wed Sep 17 23:32:49 2008 @@ -53,7 +53,8 @@ self.addStep(CondShellCommand( description="wcrevert", cond=not_first_time, - command = ["python", "py/bin/py.svnwcrevert", "."], + command = ["python", "py/bin/py.svnwcrevert", ".", + ".buildbot-sourcedata"], haltOnFailure=True)) self.addStep(source.SVN("https://codespeak.net/svn/pypy/" "branch/pypy-pytrunk")) Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Sep 17 23:32:49 2008 @@ -55,14 +55,14 @@ add_one() def get_outcome(self, namekey): - return self._outcomes[namekey] - - def get_key_namekey(self, namekey): - return (self.key, namekey) + return self._outcomes.get(namekey, ' ') def get_longrepr(self, namekey): return self.longreprs.get(namekey, '') + def get_key_namekey(self, namekey): + return (self.key, namekey) + def get_run_stdios(self): return {self.key: self._run_stdio} @@ -143,14 +143,20 @@ return self._skipped def get_outcome(self, namekey): - return self.map[namekey[0]].get_outcome(namekey[1:]) + which = namekey[0] + if which not in self.map: + return ' ' + return self.map[which].get_outcome(namekey[1:]) + + def get_longrepr(self, namekey): + which = namekey[0] + if which not in self.map: + return '' + return self.map[which].get_longrepr(namekey[1:]) def get_key_namekey(self, namekey): return self.map[namekey[0]].get_key_namekey(namekey[1:]) - def get_longrepr(self, namekey): - return self.map[namekey[0]].get_longrepr(namekey[1:]) - def get_run_stdios(self): all = {} for outcome_set in self.map.itervalues(): @@ -235,6 +241,18 @@ section = html.pre(lines) self.sections.append(section) + def add_no_revision_builds(self, status, no_revision_builds): + section = html.div(html.p("builds aborted without getting a revision")) + + for build in no_revision_builds: + builderName = build.getBuilder().getName() + num = build.getNumber() + descr = "%s #%d" % (builderName, num) + url = status.getURLForThing(build) + section.append(html.a(descr, href=url)) + section.append(html.br()) + self.sections.append(section) + def render(self): body_html = html.div(self.sections) return body_html.unicode() @@ -280,7 +298,12 @@ return html.pre(longrepr).unicode() - +def getProp(obj, name, default=None): + try: + return obj.getProperty(name) + except KeyError: + return default + class Summary(HtmlResource): title="Summary" # xxx @@ -288,27 +311,34 @@ HtmlResource.__init__(self) self.putChild('longrepr', LongRepr()) - def recentRevisions(self, request): + def recentRevisions(self, status): # xxx branches - status = self.getStatus(request) revs = {} + no_revision_builds = [] for builderName in status.getBuilderNames(): builderStatus = status.getBuilder(builderName) for build in builderStatus.generateFinishedBuilds(num_builds=N): - rev = int(build.getProperty("got_revision")) - revBuilds = revs.setdefault(rev, {}) - if builderName not in revBuilds: # pick the most recent or ? - key = (builderName, build.getNumber()) - outcome_set = outcome_set_cache.get(status, key) - revBuilds[builderName] = outcome_set + got_rev = getProp(build, 'got_revision', None) + if got_rev is None: + no_revision_builds.append(build) + else: + rev = int(got_rev) + revBuilds = revs.setdefault(rev, {}) + if builderName not in revBuilds: # pick the most recent or ? + key = (builderName, build.getNumber()) + outcome_set = outcome_set_cache.get(status, key) + revBuilds[builderName] = outcome_set - return revs + return revs, no_revision_builds def body(self, request): - revs = self.recentRevisions(request) + status = self.getStatus(request) + + revs, no_revision_builds = self.recentRevisions(status) outcome_sets = [] for rev, by_build in revs.items(): outcome_sets.append(GatherOutcomeSet(by_build)) page = SummaryPage() - page.add_section(outcome_sets) + page.add_section(outcome_sets) + page.add_no_revision_builds(status, no_revision_builds) return page.render() Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Wed Sep 17 23:32:49 2008 @@ -81,6 +81,11 @@ assert rev_outcome_set.skipped == set([ ("a.c", '')]) + def test_absent_outcome(self): + rev_outcome_set = summary.RevisionOutcomeSet(50000) + + res = rev_outcome_set.get_outcome(('a', 'b')) + assert res == ' ' def test_RevisionOutcomeSetCache(self): cache = summary.RevisionOutcomeSetCache() @@ -186,6 +191,13 @@ res = goutcome_top.get_longrepr(('sub', 'foo', 'a.b', 'test_one')) assert res == "some\ntraceback\n" + # absent + res = goutcome_top.get_outcome(('what', 'foo', 'a.b', 'test_one')) + assert res == ' ' + + res = goutcome_top.get_longrepr(('what', 'foo', 'a.b', 'test_one')) + assert res == '' + def test_colsizes(self): failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'), ('ab', '', 'cd')] From fijal at codespeak.net Thu Sep 18 09:45:59 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 09:45:59 +0200 (CEST) Subject: [pypy-svn] r58215 - pypy/build/doc Message-ID: <20080918074559.F2D9616A0B0@codespeak.net> Author: fijal Date: Thu Sep 18 09:45:56 2008 New Revision: 58215 Modified: pypy/build/doc/benchmark_memory_gcnotes.txt Log: Some notes about gc and paging Modified: pypy/build/doc/benchmark_memory_gcnotes.txt ============================================================================== --- pypy/build/doc/benchmark_memory_gcnotes.txt (original) +++ pypy/build/doc/benchmark_memory_gcnotes.txt Thu Sep 18 09:45:56 2008 @@ -18,3 +18,7 @@ * deferred refcounting with a nursery +Some references +=============== + +* Garbage collector without paging: http://www.cs.umass.edu/~emery/pubs/f034-hertz.pdf, some discussion: http://lambda-the-ultimate.org/node/2391 From fijal at codespeak.net Thu Sep 18 09:52:49 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 09:52:49 +0200 (CEST) Subject: [pypy-svn] r58216 - pypy/build/doc Message-ID: <20080918075249.F1B7C16A04A@codespeak.net> Author: fijal Date: Thu Sep 18 09:52:49 2008 New Revision: 58216 Modified: pypy/build/doc/benchmark_memory_gcnotes.txt Log: Some other paper about measurments Modified: pypy/build/doc/benchmark_memory_gcnotes.txt ============================================================================== --- pypy/build/doc/benchmark_memory_gcnotes.txt (original) +++ pypy/build/doc/benchmark_memory_gcnotes.txt Thu Sep 18 09:52:49 2008 @@ -22,3 +22,5 @@ =============== * Garbage collector without paging: http://www.cs.umass.edu/~emery/pubs/f034-hertz.pdf, some discussion: http://lambda-the-ultimate.org/node/2391 + +* paper about measurments http://www-cs.canisius.edu/~hertzm/thesis.pdf From witulski at codespeak.net Thu Sep 18 10:13:50 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Thu, 18 Sep 2008 10:13:50 +0200 (CEST) Subject: [pypy-svn] r58217 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080918081350.E24E4169E7C@codespeak.net> Author: witulski Date: Thu Sep 18 10:13:48 2008 New Revision: 58217 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: Added some boolfunc. (XOR,OR,AND) New Bool-Tests pass Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Thu Sep 18 10:13:48 2008 @@ -149,6 +149,8 @@ _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 2) _ADD_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x00", 3, None, None) + _AND_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x21", 3, None, None) + # FIXME: rexB is set _CMP_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 1, "\x81", 3, None, 7) _CMP_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x39", 3, None, None) @@ -167,6 +169,8 @@ _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") _JMP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 4) + + _OR_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x09", 3, None, None) # FIXME: rexW is set _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", 3, None, 0) @@ -182,11 +186,17 @@ _SUB_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x28", 3, None, None) _SUB_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 5) + _XOR_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x31", 3, None, None) + # TODO: maybe a problem with more ore less than two arg. def ADD(self, op1, op2): method = getattr(self, "_ADD"+op1.to_string()+op2.to_string()) method(op1, op2) + def AND(self, op1, op2): + method = getattr(self, "_AND"+op1.to_string()+op2.to_string()) + method(op1, op2) + def CMP(self, op1, op2): method = getattr(self, "_CMP"+op1.to_string()+op2.to_string()) method(op1, op2) @@ -213,6 +223,10 @@ self.writeImm32(op1) print self.tell(),": JNE to",op1 + def OR(self, op1, op2): + method = getattr(self, "_OR"+op1.to_string()+op2.to_string()) + method(op1, op2) + def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) method(op1) @@ -265,6 +279,10 @@ method = getattr(self, "_SUB"+op1.to_string()+op2.to_string()) method(op1, op2) + def XOR(self, op1, op2): + method = getattr(self, "_XOR"+op1.to_string()+op2.to_string()) + method(op1, op2) + def get_register_bits(self, register): return REGISTER_MAP[register] Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Thu Sep 18 10:13:48 2008 @@ -98,12 +98,15 @@ return genmethod(gv_arg1, gv_arg2) op_int_add = make_two_argument_method("ADD") + op_int_and = make_two_argument_method("AND") op_int_dec = make_one_argument_method("DEC") op_int_inc = make_one_argument_method("INC") op_int_mul = make_two_argument_method("IMUL") + op_int_or = make_two_argument_method("OR") op_int_push = make_one_argument_method("PUSH") op_int_pop = make_one_argument_method("POP") op_int_sub = make_two_argument_method("SUB") + op_int_xor = make_two_argument_method("XOR") #FIXME: can only jump 32bit def jump_if_true(self, gv_condition, args_for_jump_gv): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Thu Sep 18 10:13:48 2008 @@ -10,6 +10,16 @@ def skip(self): py.test.skip("not implemented yet") +def make_bool_op(rgenop, which_bool_op): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_bool_op, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "bool_op") + builder.start_writing() + + gv_result = builder.genop2(which_bool_op, gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_bool_op + def make_cmp(rgenop, which_cmp): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_cmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "cmp") @@ -205,6 +215,61 @@ res = fnptr(-4,0) assert res == 1 + def test_int_and(self): + rgenop = self.RGenOp() + bool_function = make_bool_op(rgenop,"int_and") + fnptr = self.cast(bool_function,2) + result = fnptr(1,1) + assert result == 1 + result = fnptr(1,0) + assert result == 0 + result = fnptr(0,1) + assert result == 0 + result = fnptr(0,0) + assert result == 0 + # AND 010101 + # 101010 + # = 000000 + result = fnptr(42,21) + assert result == 0 + + def test_int_or(self): + rgenop = self.RGenOp() + bool_function = make_bool_op(rgenop,"int_or") + fnptr = self.cast(bool_function,2) + result = fnptr(1,1) + assert result == 1 + result = fnptr(1,0) + assert result == 1 + result = fnptr(0,1) + assert result == 1 + result = fnptr(0,0) + assert result == 0 + # or 010101 + # 101010 + # = 111111 + result = fnptr(42,21) + assert result == 63 + + def test_int_xor(self): + rgenop = self.RGenOp() + bool_function = make_bool_op(rgenop,"int_xor") + fnptr = self.cast(bool_function,2) + result = fnptr(1,1) + assert result == 0 + result = fnptr(1,0) + assert result == 1 + result = fnptr(0,1) + assert result == 1 + result = fnptr(0,0) + assert result == 0 + # xor 010101 + # 101010 + # = 111111 + result = fnptr(42,21) + assert result == 63 + + # def test_push_and_pop(self): # rgenop = self.RGenOp() # push_result = make_push(rgenop) From witulski at codespeak.net Thu Sep 18 10:54:19 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Thu, 18 Sep 2008 10:54:19 +0200 (CEST) Subject: [pypy-svn] r58218 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080918085419.5801A16A0F9@codespeak.net> Author: witulski Date: Thu Sep 18 10:54:17 2008 New Revision: 58218 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: litle Test clean up Added NEG and NOT Operations + Tests(pass) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Thu Sep 18 10:54:17 2008 @@ -168,6 +168,10 @@ _IMUL_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x0F", 3, None, None, None, "\xAF") _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") + _NEG_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xF7", 3, None, 3) + + _NOT_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xF7", 3, None, 2) + _JMP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 4) _OR_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x09", 3, None, None) @@ -247,6 +251,14 @@ method(op2, op1) else: method(op1, op2) + + def NEG(self, op1): + method = getattr(self, "_NEG"+op1.to_string()) + method(op1) + + def NOT(self, op1): + method = getattr(self, "_NOT"+op1.to_string()) + method(op1) def RET(self): self.write("\xC3") Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Thu Sep 18 10:54:17 2008 @@ -88,13 +88,13 @@ @specialize.arg(1) def genop1(self, opname, gv_arg): genmethod = getattr(self, 'op_' + opname) - print self.mc.tell(),":",opname + # print self.mc.tell(),":",opname return genmethod(gv_arg) @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): genmethod = getattr(self, 'op_' + opname) - print self.mc.tell(),":",opname + # print self.mc.tell(),":",opname return genmethod(gv_arg1, gv_arg2) op_int_add = make_two_argument_method("ADD") @@ -102,6 +102,8 @@ op_int_dec = make_one_argument_method("DEC") op_int_inc = make_one_argument_method("INC") op_int_mul = make_two_argument_method("IMUL") + op_int_neg = make_one_argument_method("NEG") + op_int_not = make_one_argument_method("NOT") op_int_or = make_two_argument_method("OR") op_int_push = make_one_argument_method("PUSH") op_int_pop = make_one_argument_method("POP") Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Thu Sep 18 10:54:17 2008 @@ -10,6 +10,16 @@ def skip(self): py.test.skip("not implemented yet") +def make_one_op_instr(rgenop, instr_name): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) + builder, gv_one_op_instr, [gv_x] = rgenop.newgraph(sigtoken, "one_op_instr") + builder.start_writing() + + gv_result = builder.genop1(instr_name, gv_x) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_one_op_instr + def make_bool_op(rgenop, which_bool_op): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_bool_op, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "bool_op") @@ -46,58 +56,19 @@ gv_result = builder.genop2("int_mul", gv_x, rgenop.genconst(num)) builder.finish_and_return(sigtoken, gv_result) builder.end() - return gv_mul - -def make_inc(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) - builder, gv_inc, gv_x = rgenop.newgraph(sigtoken, "inc") - builder.start_writing() - gv_result = builder.genop1("int_inc", gv_x[0]) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_inc - -def make_dec(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) - builder, gv_dec, gv_x = rgenop.newgraph(sigtoken, "dec") - builder.start_writing() - gv_result = builder.genop1("int_dec", gv_x[0]) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_dec - -def make_push(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) - builder, gv_push, gv_x = rgenop.newgraph(sigtoken, "push") - builder.start_writing() - gv_result = builder.genop1("int_push", gv_x[0]) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_push - -def make_pop(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) - builder, gv_pop, gv_x = rgenop.newgraph(sigtoken, "pop") - builder.start_writing() - gv_result = builder.genop1("int_pop", gv_x[0]) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_pop - + return gv_mul class TestRGenopDirect(AbstractRGenOpTestsDirect): RGenOp = RX86_64GenOp def test_inc(self): - rgenop = self.RGenOp() - inc_function = make_inc(rgenop) + inc_function = make_one_op_instr(self.RGenOp(),"int_inc") fnptr = self.cast(inc_function,1) res = fnptr(0) assert res == 1 def test_dec(self): - rgenop = self.RGenOp() - dec_function = make_dec(rgenop) + dec_function = make_one_op_instr(self.RGenOp(),"int_dec") fnptr = self.cast(dec_function,1) res = fnptr(2) assert res == 1 @@ -268,17 +239,33 @@ # = 111111 result = fnptr(42,21) assert result == 63 - - # def test_push_and_pop(self): - # rgenop = self.RGenOp() - # push_result = make_push(rgenop) - # fnptr = self.cast(push_result,1) - # pop_result = make_pop(rgenop) - # fnptr = self.cast(pop_result,1) - # res = fnptr(42) - #assert res == 1 + def test_neg(self): + neg_function = make_one_op_instr(self.RGenOp(),"int_neg") + fnptr = self.cast(neg_function,1) + result = fnptr(1) + assert result == -1 + result = fnptr(-1) + assert result == 1 + result = fnptr(255) + assert result == -255 + result = fnptr(0) + assert result == 0 + result = fnptr(-123456789) + assert result == 123456789 + def test_not(self): + not_function = make_one_op_instr(self.RGenOp(),"int_not") + fnptr = self.cast(not_function,1) + result = fnptr(1) + assert result == -2 + result = fnptr(0) + assert result == -1 + result = fnptr(-43) + assert result == 42 + + #TODO: Test push/pop + test_directtesthelper_direct = skip test_dummy_compile = skip test_cast_raising = skip From witulski at codespeak.net Thu Sep 18 11:33:54 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Thu, 18 Sep 2008 11:33:54 +0200 (CEST) Subject: [pypy-svn] r58219 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080918093354.01B5F169ED3@codespeak.net> Author: witulski Date: Thu Sep 18 11:33:53 2008 New Revision: 58219 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: Added a new Test which adds two nums, seams the return is the cause of all broken tests(neg num bug) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Thu Sep 18 11:33:53 2008 @@ -143,7 +143,8 @@ # The opcodes differs depending on the operands # Params: - # W (64bit Operands), R, X, B, Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode + # W (64bit Operands), R (permits R8-R15), X (extend Index field), B (extend r/m field), + # Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode # FIXME: rexB is set _ADD_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 2) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Thu Sep 18 11:33:53 2008 @@ -24,7 +24,7 @@ def test_add_big_num(self): builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) @@ -32,10 +32,29 @@ assert num == 2280 print num - def test_add(self): + def test_add_twice(self): builder, fp, inputargs_gv, token = make_testbuilder(2) genv0 = inputargs_gv[0] #the first argument "place" genv1 = inputargs_gv[1] + genv2 = builder.genop2("int_add", genv0, genv1) + genv_result = builder.genop2("int_add", genv2, genv1) + builder.finish_and_return(token, genv_result) + result = fp(4, 6) # 4+6+6= 16 + assert result == 16 + result = fp(2,12) # 2+12+12= 26 + assert result == 26 + result = fp(10,-2) # 10+(-2)+(-2) = 6 + assert result == 6 + result = fp(-4,0) # -4 +0+0 = -4 + assert result == -4 + result = fp(0,-4) # 0+(-4)+(-4) = -8 + assert result == -8 + + + def test_add(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] #the first argument "location" + genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) ten = fp(4, 6) # 4+6= 10 @@ -44,7 +63,7 @@ def test_add_neg(self): builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) @@ -66,7 +85,7 @@ def test_add_imm32(self): builder, fp, inputargs_gv, token = make_testbuilder(1) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) num = fp(1111) # 1111+1000 = 2111 @@ -83,17 +102,21 @@ def test_sub(self): builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv1 = inputargs_gv[1] genv_result = builder.genop2("int_sub", genv0, genv1) #creates the subtraction and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) four = fp(10, 6) # 10 - 6 = 4 assert four == 4 + ten = fp(-2, 8) + assert ten == -10 + ten = fp(-2,-12) + assert ten == 10 print four def test_sub_imm32(self): builder, fp, inputargs_gv, token = make_testbuilder(1) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv_result = builder.genop2("int_sub", genv0, rgenop.genconst(2)) #creates the subtraction and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) eight = fp(10) # 10-2 = 8 From nshepperd at codespeak.net Thu Sep 18 12:53:23 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Thu, 18 Sep 2008 12:53:23 +0200 (CEST) Subject: [pypy-svn] r58220 - in pypy/branch/new-platformcheck/pypy/rpython/tool: . test Message-ID: <20080918105323.463D816A101@codespeak.net> Author: nshepperd Date: Thu Sep 18 12:53:21 2008 New Revision: 58220 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Log: Tentative support for obtaining constants from extrnal libraries (that is, using -lstuff). Expect refactoring. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Thu Sep 18 12:53:21 2008 @@ -2,7 +2,7 @@ import os, py, sys from subprocess import Popen, PIPE -from tempfile import NamedTemporaryFile +from tempfile import NamedTemporaryFile, mkdtemp from string import atoi from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi @@ -215,9 +215,8 @@ for key in dir(CConfig): value = getattr(CConfig, key) if isinstance(value, CConfigExternEntry): - print (key, value) value.eci = CConfig._compilation_info_ - res[key] = value.build_result() + res[key] = value.build_result(locals().get('result')) return res @@ -267,47 +266,120 @@ def bool(self, data): return data[0] != '\x00' -class CConfigExternEntry(object): - """Abstract base class.""" - def get(self, name, CC='gcc', NM=None, OBJCOPY=None): - if not NM: - NM = CC.replace('gcc', 'nm') - if not OBJCOPY: - OBJCOPY = CC.replace('gcc', 'objcopy') - libs = self.eci.link_extra - for obj in libs: - nm = Popen(NM + ' -f sysv ' + obj, shell=True, stdout=PIPE) - nm.wait() - nm = nm.stdout.read() - for line in nm.splitlines(): - line = [part.strip() for part in line.split('|')] - if len(line) == 7 and line[0] == name and line[2] != 'U': - # only if the symbol is not a 'depends on' symbol ^ - offset = atoi(line[1], 16) - minsize = atoi(line[4], 16) - section = line[6] - break - else: +def get_symbol_data(eci, name, NM, OBJCOPY): + link_extra = ' '.join(list(eci.link_extra) + ['-static', '-Wl,--trace', '-Wl,--whole-archive']) + libraries = ' '.join(['-l' + lib for lib in eci.libraries]) + libdirs = ' '.join(['-L' + _dir for _dir in eci.library_dirs]) + dir = mkdtemp() + srcpath = os.path.join(dir, 'main.c') + objpath = os.path.join(dir, 'main.o') + exepath = os.path.join(dir, 'main.exe') + src = open(srcpath, 'w') + src.write('int main() {}\n') + src.close() + if os.system('gcc -c -o %s %s' % (objpath, srcpath)): + raise CompilationError('gcc does not work') + linkcmd = ' '.join(['gcc -o ', exepath, objpath, link_extra, libdirs, libraries]) + link = Popen(linkcmd, shell=True, stdout=PIPE, stderr=PIPE) + + linked = [] + for line in link.stdout.readlines(): + line = line.strip() + if os.path.exists(line): + linked.append(line) + sub = line.split('(') + if len(sub) >= 2: + sub = sub[1].split(')')[0].strip() + if os.path.exists(sub) and sub not in linked: + linked.append(sub) + + for obj in linked: + nm = Popen(NM + ' -f sysv ' + obj, shell=True, stdout=PIPE, stderr=PIPE) + nm = list(nm.stdout.readlines()) + is_archive = False + for line in nm: + if '[' in line and ']' in line: + is_archive = True + member = line.partition('[')[2].partition(']')[0] continue - break + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[0] == name and line[2] != 'U': + # only if the symbol is a real symbol .....^ + offset = atoi(line[1], 16) + maxsize = atoi(line[4], 16) + section = line[6] + break else: - return None - sec = NamedTemporaryFile() - try: - if os.system('%s -O binary -j \'%s\' %s %s' % - (OBJCOPY, section, obj, sec.name)): - raise Exception('objcopy error') - sec.seek(offset) - return sec.read(minsize) - finally: - sec.close() + continue + base_offset = offset + this_member = None + for line in nm: + if is_archive: + if '[' in line and ']' in line: + this_member = line.partition('[')[2].partition(']')[0] + continue + if not is_archive or this_member == member: + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[6] == section and line[2] != 'U': + base_offset = min(base_offset, atoi(line[1], 16)) + offset = offset - base_offset + break + else: + return None + sec = NamedTemporaryFile() + if is_archive: + obj2 = NamedTemporaryFile() + ar = Popen('ar p %s %s' % (obj, member), shell=True, stdout=PIPE) + data = ar.stdout.read() + obj2.write(data) + obj2.flush() + obj = obj2.name + cmd = '%s -O binary -j \'%s\' %s %s' % (OBJCOPY, section, obj, sec.name) + if os.system(cmd): + raise CompilationError('objcopy error') + sec.seek(offset) + return sec.read(maxsize) + +class CConfigExternEntry(object): + """Abstract base class.""" + def get(self, name): + return get_symbol_data(self.eci, name, 'nm', 'objcopy') class ExternString(CConfigExternEntry): def __init__(self, name): self.name = name - def build_result(self, *args, **kw): - return self.get(self.name, *args, **kw) + def build_result(self, config_result): + data = self.get(self.name) + if data: + return data.partition('\0')[0] + else: + return None + +class ExternStruct(CConfigExternEntry): + def __init__(self, name, cconfig_entry=None, rffi_struct=None): + if not (cconfig_entry or rffi_struct): + raise TypeError('ExternStruct takes 3 arguments') + self.entry = cconfig_entry + self.rffi_struct = rffi_struct + self.name = name + + def build_result(self, config_result): + if not self.rffi_struct: + rffi_struct = config_result.get_entry_result(self.entry) + else: + rffi_struct = self.rffi_struct + data = self.get(self.name) + if not data: + return None + class StructResult(object): pass + res = StructResult() + for (fld_name, fld_offset) in zip( + rffi_struct._names, rffi_struct._hints['fieldoffsets']): + fld_type = rffi_struct._flds[fld_name] + fld_data = data[fld_offset:fld_offset+rffi.sizeof(fld_type)] + setattr(res, fld_name, rffi.cast(fld_type, fld_data[0])) + return res class Struct(CConfigEntry): """An entry in a CConfig class that stands for an externally Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Thu Sep 18 12:53:21 2008 @@ -245,6 +245,37 @@ src.close() os.system('gcc -c -o %s %s' % (objpath, srcpath)) class CConfig: - _compilation_info_ = ExternalCompilationInfo(link_extra = [objpath]) + _compilation_info_ = ExternalCompilationInfo(link_extra=[objpath]) STRING = rffi_platform.ExternString('stuff') - assert rffi_platform.configure(CConfig)['STRING'] == 'Success!\0' + assert rffi_platform.configure(CConfig)['STRING'] == 'Success!' + +def test_extern_fail(): + class CConfig: + _compilation_info_ = ExternalCompilationInfo() + STRING = rffi_platform.ExternString('stuff') + assert rffi_platform.configure(CConfig)['STRING'] is None + +def test_extern_struct(): + from tempfile import mkdtemp + dir = mkdtemp() + srcpath = os.path.join(dir, 'test.c') + objpath = os.path.join(dir, 'test.o') + hpath = os.path.join(dir, 'test.h') + h = open(hpath, 'w') + print >> h, 'typedef struct Stuff {' + print >> h, ' int a;' + print >> h, ' int b;' + print >> h, '} Stuff;' + h.close() + src = open(srcpath, 'w') + print >> src, '#include "' + hpath + '"' + print >> src, 'struct Stuff stuff = { 1, 2 };' + src.close() + os.system('gcc -c -o %s %s' % (objpath, srcpath)) + class CConfig: + _compilation_info_ = ExternalCompilationInfo(link_extra=[objpath], includes=[hpath]) + STRUCT = rffi_platform.Struct('Stuff', [('a', rffi.INT), ('b', rffi.INT)]) + STUFF = rffi_platform.ExternStruct('stuff', STRUCT) + assert rffi_platform.configure(CConfig)['STUFF'].c_a == 1 + assert rffi_platform.configure(CConfig)['STUFF'].c_b == 2 + From hpk at codespeak.net Thu Sep 18 14:23:20 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 18 Sep 2008 14:23:20 +0200 (CEST) Subject: [pypy-svn] r58221 - in pypy/build/benchmem: . testing Message-ID: <20080918122320.1B499169F36@codespeak.net> Author: hpk Date: Thu Sep 18 14:23:18 2008 New Revision: 58221 Modified: pypy/build/benchmem/benchtool.py pypy/build/benchmem/testing/test_benchtool.py Log: parse all data from smaps files, with tests. Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Thu Sep 18 14:23:18 2008 @@ -1,5 +1,5 @@ import py -import os, sys +import os, sys, re mydir = py.magic.autopath().dirpath() benchmarkdir = mydir.join("benchmark") @@ -76,6 +76,61 @@ self._file.write("\n") self._file.write(self.SEPLINE) self._file.flush() + +class SmapsMapping: + def __init__(self, headvars, bodyvars): + self.__dict__.update(headvars) + self.__dict__.update(bodyvars) + self.inode = int(self.inode) + +class SmapsReader(object): + # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' + headrex = re.compile(r""" + (?P\w+-\w+)\s+ + (?P[rwxps-]+)\s+ + (?P\w+)\s+ + (?P\w\w:\w\w)\s+ + (?P\w+)\s+ + (?P\S*)""", + re.VERBOSE) + # example 'Size: 4648 kB' + linerex = re.compile(r"(\S+):\s+(\d+)\skB") + + def __init__(self, mappings): + self.mappings = mappings + for name in ("size rss shared_clean shared_dirty " + "private_clean private_dirty".split()): + setattr(self, name, sum([getattr(x, name) for x in mappings])) + + @classmethod + def parse(cls, string): + lines = string.split("\n") + mappings = [] + while lines: + line = lines.pop(0) + if not line.strip(): + continue + m = cls.headrex.match(line) + if m is None: + print "ignoring", line + continue + headvars = m.groupdict() + bodyvars = {} + while lines: + line = lines.pop(0) + m = cls.linerex.match(line) + if m is None: + lines.insert(0, line) + mappings.append(SmapsMapping(headvars, bodyvars)) + break + name, num = m.groups() + name = name.lower() + num = int(num) * 1024 # xxx get constant from somewhere? + assert name not in bodyvars + bodyvars[name] = num + else: + mappings.append(SmapsMapping(headvars, bodyvars)) + return cls(mappings) if __name__ == '__main__': pass Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Thu Sep 18 14:23:18 2008 @@ -1,7 +1,7 @@ import py import os, sys -from benchtool import SmapsRecorder, BenchRunner, benchmarkdir +import benchtool def setup_module(mod): if sys.platform.find("linux") == -1: @@ -11,23 +11,147 @@ tmpdir = py.test.ensuretemp("smapsrecorder") logpath = tmpdir.join("logfile") pid = os.getpid() - rec = SmapsRecorder(pid=pid, logpath=logpath) + rec = benchtool.SmapsRecorder(pid=pid, logpath=logpath) s = logpath.read() - assert s.count(SmapsRecorder.SEPLINE) == 1 + assert s.count(benchtool.SmapsRecorder.SEPLINE) == 1 rec.snapshot() rec.snapshot() del rec s = logpath.read() - assert s.count(SmapsRecorder.SEPLINE) == 3 + assert s.count(benchtool.SmapsRecorder.SEPLINE) == 3 def test_benchrunner(): tmpdir = py.test.ensuretemp("benchrunner") - runner = BenchRunner(executable="python2.5", resultdir=tmpdir) + runner = benchtool.BenchRunner(executable="python2.5", resultdir=tmpdir) def checker(name, *args): smapsfile = runner.run_checkpointed_bench(name, args) assert smapsfile.check() - for name in benchmarkdir.listdir("*.py"): + for name in benchtool.benchmarkdir.listdir("*.py"): if name.basename[0] != "_": yield checker, name.basename, 10, 10 +class TestSmapsReader: + def setup_class(cls): + cls.s = py.std.textwrap.dedent("""\ + 08048000-0813f000 r-xp 00000000 fd:00 75457 sometext + Size: 988 kB + Rss: 796 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 796 kB + Private_Dirty: 0 kB + 402c2000-402c4000 rw-p 402c2000 00:00 0 + Size: 16 kB + Rss: 8 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 8 kB""") + + def test_parse_mappings(self): + smapsreader = benchtool.SmapsReader.parse(self.s) + mappings = smapsreader.mappings + assert len(mappings) == 2 + mapping = mappings[0] + assert mapping.virtualaddress == "08048000-0813f000" + assert mapping.mode == "r-xp" + assert mapping.dev == "fd:00" + assert mapping.inode == 75457 + assert mapping.filename == "sometext" + assert mapping.size == 988 * 1024 + assert mapping.rss == 796 * 1024 + assert mapping.shared_clean == 0 + assert mapping.shared_dirty == 0 + assert mapping.private_clean == 796 * 1024 + assert mapping.private_dirty == 0 + #assert mapping.referenced is None + + mapping = mappings[1] + assert mapping.virtualaddress == "402c2000-402c4000" + assert mapping.mode == "rw-p" + assert mapping.dev == "00:00" + assert mapping.inode == 0 + assert not mapping.filename + assert mapping.size == 16 * 1024 + assert mapping.rss == 8 * 1024 + assert mapping.shared_clean == 0 + assert mapping.shared_dirty == 0 + assert mapping.private_clean == 0 + assert mapping.private_dirty == 8 * 1024 + #assert mapping.referenced == 12 * 1024 + + def test_summing(self): + smaps = benchtool.SmapsReader.parse(self.s) + for name in ('size rss shared_clean shared_dirty ' + 'private_clean private_dirty').split(): + sumval = getattr(smaps, name) + val1 = getattr(smaps.mappings[0], name) + val2 = getattr(smaps.mappings[1], name) + assert sumval == val1 + val2 + +example_data2 = ''' +08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 +Size: 988 kB +Rss: 796 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 796 kB +Private_Dirty: 0 kB +0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 +Size: 148 kB +Rss: 120 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 12 kB +Private_Dirty: 108 kB +08164000-0825c000 rw-p 08164000 00:00 0 [heap] +Size: 992 kB +Rss: 924 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 924 kB +b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 +Size: 240 kB +Rss: 60 kB +Shared_Clean: 60 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 0 kB +''' + +example_data = ''' +08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 +Size: 988 kB +Rss: 796 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 796 kB +Private_Dirty: 0 kB +Referenced: 796 kB +0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 +Size: 148 kB +Rss: 120 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 12 kB +Private_Dirty: 108 kB +Referenced: 120 kB +08164000-0825c000 rw-p 08164000 00:00 0 [heap] +Size: 992 kB +Rss: 924 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 924 kB +Referenced: 924 kB +b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 +Size: 240 kB +Rss: 60 kB +Shared_Clean: 60 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 0 kB +Referenced: 60 kB +''' From pedronis at codespeak.net Thu Sep 18 14:23:27 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 18 Sep 2008 14:23:27 +0200 (CEST) Subject: [pypy-svn] r58222 - pypy/build/bot2 Message-ID: <20080918122327.8893016A062@codespeak.net> Author: pedronis Date: Thu Sep 18 14:23:26 2008 New Revision: 58222 Added: pypy/build/bot2/TODO (contents, props changed) Log: TODO file Added: pypy/build/bot2/TODO ============================================================================== --- (empty file) +++ pypy/build/bot2/TODO Thu Sep 18 14:23:26 2008 @@ -0,0 +1,17 @@ +- runner for running tests per directory and optionally in parallel + (this should be inspired by the current autotest.py running strategy minus the htmlconftest + details) + +- buildbot Nightly scheduler that can pick a uniform revision server side and with support + for specifying possibly branches + +- improve summary page (defined in pypybuildbot/summary.page), support querying/slicing + by builders/revisions(/branches), formatting and css + +- build/support for running cpython on tests, adaptations to produce structured logs and be compatible + with the py.lib trunk are probably necessary for lib-python/conftest.py + +- pypy-c py.test -A support + +- fix pypy conftests not to use deprecated interfaces + From hpk at codespeak.net Thu Sep 18 15:17:21 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 18 Sep 2008 15:17:21 +0200 (CEST) Subject: [pypy-svn] r58223 - in pypy/build/benchmem: . testing Message-ID: <20080918131721.D55CF16A08C@codespeak.net> Author: hpk Date: Thu Sep 18 15:17:19 2008 New Revision: 58223 Modified: pypy/build/benchmem/benchtool.py pypy/build/benchmem/testing/test_benchtool.py Log: (intermediate) advancing the tool to run commandlines like "python benchtool.py python2.5 create_recursive_tuples.py" Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Thu Sep 18 15:17:19 2008 @@ -32,7 +32,9 @@ pid = os.getpid() sys.stdout.write(str(pid)) sys.stdout.write("\\n") + checkpoint() %s(checkpoint, %s) + checkpoint() """ %(name, arglist)) p = self.resultdir.join("%s.py" %(name,)) p.write(source) @@ -68,7 +70,6 @@ self.pid = pid self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) assert self.smapspath.check() - self.snapshot() def snapshot(self): s = self.smapspath.read() @@ -77,13 +78,33 @@ self._file.write(self.SEPLINE) self._file.flush() +class SmapsReader: + def __init__(self, snapshots): + self.snapshots = snapshots + + @classmethod + def parse(cls, path): + """ parse a file previously recorded through SmapsRecorder. """ + s = path.read() + parts = filter(None, map(str.strip, s.split(SmapsRecorder.SEPLINE))) + snapshots = [SmapsProcess.parse(s) for s in parts] + return cls(snapshots) + + def _verify_integrity(self): + for snap in self.snapshots: + for mapping in snap.mappings: + clean = mapping.shared_clean + mapping.private_clean + dirty = mapping.private_dirty + mapping.shared_dirty + assert mapping.rss == dirty + clean + + class SmapsMapping: def __init__(self, headvars, bodyvars): self.__dict__.update(headvars) self.__dict__.update(bodyvars) self.inode = int(self.inode) -class SmapsReader(object): +class SmapsProcess(object): # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' headrex = re.compile(r""" (?P\w+-\w+)\s+ @@ -98,10 +119,14 @@ def __init__(self, mappings): self.mappings = mappings + self.UNIT = "kB" for name in ("size rss shared_clean shared_dirty " "private_clean private_dirty".split()): setattr(self, name, sum([getattr(x, name) for x in mappings])) - + + def memusage(self): + return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) + @classmethod def parse(cls, string): lines = string.split("\n") @@ -125,7 +150,7 @@ break name, num = m.groups() name = name.lower() - num = int(num) * 1024 # xxx get constant from somewhere? + num = int(num) assert name not in bodyvars bodyvars[name] = num else: @@ -133,5 +158,23 @@ return cls(mappings) if __name__ == '__main__': - pass - + executable = sys.argv[1] + name = sys.argv[2] + resultdir = py.path.local("resultdir").ensure(dir=1) + print "using", resultdir + benchrunner = BenchRunner(executable, resultdir) + logpath = benchrunner.run_checkpointed_bench(name, (100, 1000)) + reader = SmapsReader.parse(logpath) + last = None + for i, snap in enumerate(reader.snapshots): + if last is None: + last = snap + print i, snap.memusage() + continue + if (last.private_dirty != snap.private_dirty or + last.shared_dirty != snap.shared_dirty or + len(last.mappings) != snap.mappings): + print i, snap.memusage()#, snap.memdiff(last) + else: + print i, snap.memusage()#, snap.memdiff(last) + Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Thu Sep 18 15:17:19 2008 @@ -13,25 +13,28 @@ pid = os.getpid() rec = benchtool.SmapsRecorder(pid=pid, logpath=logpath) s = logpath.read() - assert s.count(benchtool.SmapsRecorder.SEPLINE) == 1 + assert s.count(benchtool.SmapsRecorder.SEPLINE) == 0 rec.snapshot() rec.snapshot() del rec s = logpath.read() - assert s.count(benchtool.SmapsRecorder.SEPLINE) == 3 + assert s.count(benchtool.SmapsRecorder.SEPLINE) == 2 -def test_benchrunner(): +def test_benchrunner_functional(): tmpdir = py.test.ensuretemp("benchrunner") runner = benchtool.BenchRunner(executable="python2.5", resultdir=tmpdir) def checker(name, *args): smapsfile = runner.run_checkpointed_bench(name, args) assert smapsfile.check() + reader = benchtool.SmapsReader.parse(smapsfile) + assert len(reader.snapshots) == 10 + 2 + reader._verify_integrity() for name in benchtool.benchmarkdir.listdir("*.py"): if name.basename[0] != "_": yield checker, name.basename, 10, 10 -class TestSmapsReader: +class TestSmapsProcess: def setup_class(cls): cls.s = py.std.textwrap.dedent("""\ 08048000-0813f000 r-xp 00000000 fd:00 75457 sometext @@ -50,7 +53,7 @@ Private_Dirty: 8 kB""") def test_parse_mappings(self): - smapsreader = benchtool.SmapsReader.parse(self.s) + smapsreader = benchtool.SmapsProcess.parse(self.s) mappings = smapsreader.mappings assert len(mappings) == 2 mapping = mappings[0] @@ -59,11 +62,11 @@ assert mapping.dev == "fd:00" assert mapping.inode == 75457 assert mapping.filename == "sometext" - assert mapping.size == 988 * 1024 - assert mapping.rss == 796 * 1024 + assert mapping.size == 988 + assert mapping.rss == 796 assert mapping.shared_clean == 0 assert mapping.shared_dirty == 0 - assert mapping.private_clean == 796 * 1024 + assert mapping.private_clean == 796 assert mapping.private_dirty == 0 #assert mapping.referenced is None @@ -73,16 +76,16 @@ assert mapping.dev == "00:00" assert mapping.inode == 0 assert not mapping.filename - assert mapping.size == 16 * 1024 - assert mapping.rss == 8 * 1024 + assert mapping.size == 16 + assert mapping.rss == 8 assert mapping.shared_clean == 0 assert mapping.shared_dirty == 0 assert mapping.private_clean == 0 - assert mapping.private_dirty == 8 * 1024 - #assert mapping.referenced == 12 * 1024 + assert mapping.private_dirty == 8 + #assert mapping.referenced == 12 def test_summing(self): - smaps = benchtool.SmapsReader.parse(self.s) + smaps = benchtool.SmapsProcess.parse(self.s) for name in ('size rss shared_clean shared_dirty ' 'private_clean private_dirty').split(): sumval = getattr(smaps, name) From fijal at codespeak.net Thu Sep 18 18:14:52 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 18:14:52 +0200 (CEST) Subject: [pypy-svn] r58226 - pypy/branch/gc-experiments Message-ID: <20080918161452.D1A9F169EED@codespeak.net> Author: fijal Date: Thu Sep 18 18:14:52 2008 New Revision: 58226 Added: pypy/branch/gc-experiments/ - copied from r58225, pypy/dist/ Log: A branch to experiment with different gcs From fijal at codespeak.net Thu Sep 18 18:16:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 18:16:43 +0200 (CEST) Subject: [pypy-svn] r58227 - pypy/build/doc Message-ID: <20080918161643.7A010169EFB@codespeak.net> Author: fijal Date: Thu Sep 18 18:16:42 2008 New Revision: 58227 Modified: pypy/build/doc/benchmark_memory_gcnotes.txt Log: A couple of notes and FAQ why M&S is so bad. Modified: pypy/build/doc/benchmark_memory_gcnotes.txt ============================================================================== --- pypy/build/doc/benchmark_memory_gcnotes.txt (original) +++ pypy/build/doc/benchmark_memory_gcnotes.txt Thu Sep 18 18:16:42 2008 @@ -23,4 +23,29 @@ * Garbage collector without paging: http://www.cs.umass.edu/~emery/pubs/f034-hertz.pdf, some discussion: http://lambda-the-ultimate.org/node/2391 +NOTE: this gc requires modest (600 LOC) support from operating system + *very* unlikely we can make such assumptions + * paper about measurments http://www-cs.canisius.edu/~hertzm/thesis.pdf + +FAQ +=== + +Why mark&sweep is consuming so much memory on some benchmarks? +-------------------------------------------------------------- + +It seems that some allocation schemes (unfortunately ones happen to be +found in pypy during normal operation) create very long free lists for small +objects. Consider example: + + [(i,) for i in range(amount)] + +in CPython, any immediate object created during list construction is immediately +discarded. On the other hand, while creating the same thing in RPython, for +each allocated space, it gets stored in free list of obmalloc or C library +until collection happens. This means that since collection happens not +very often (due to high costs), this gets accumulated (also for different +python-level operations, different freelists will be populated, if requested +size is different enough). If we invoke gc.collect +every couple iterations (say 100 or 1000), this effect is gone. On the other +hand it's becoming very very slow as a side effect. From fijal at codespeak.net Thu Sep 18 18:23:24 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 18:23:24 +0200 (CEST) Subject: [pypy-svn] r58228 - in pypy/branch/gc-experiments/pypy/rpython/memory/gc: . test Message-ID: <20080918162324.12D10169F24@codespeak.net> Author: fijal Date: Thu Sep 18 18:23:23 2008 New Revision: 58228 Added: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (contents, props changed) Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py Log: A first try at mark&compact collector, a bit of shuffling around to avoid repetition. *IN-PROGRESS* doesn't work so far. Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Thu Sep 18 18:23:23 2008 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.debug import ll_assert class GCBase(object): @@ -182,12 +182,55 @@ pass +TYPEID_MASK = 0xffff +first_gcflag = 1 << 16 +GCFLAG_FORWARDED = first_gcflag +# GCFLAG_EXTERNAL is set on objects not living in the semispace: +# either immortal objects or (for HybridGC) externally raw_malloc'ed +GCFLAG_EXTERNAL = first_gcflag << 1 +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 + class MovingGCBase(GCBase): moving_gc = True + first_unused_gcflag = first_gcflag << 3 def can_move(self, addr): return True + def header(self, addr): + addr -= self.gcheaderbuilder.size_gc_header + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + + def get_type_id(self, addr): + tid = self.header(addr).tid + ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, + "get_type_id on forwarded obj") + # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. + # Although calling get_type_id() on a forwarded object works by itself, + # we catch it as an error because it's likely that what is then + # done with the typeid is bogus. + return tid & TYPEID_MASK + + def init_gc_object(self, addr, typeid, flags=0): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + hdr.tid = typeid | flags + + def init_gc_object_immortal(self, addr, typeid, flags=0): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED + # immortal objects always have GCFLAG_FORWARDED set; + # see get_forwarding_address(). + + def get_size(self, obj): + typeid = self.get_type_id(obj) + size = self.fixed_size(typeid) + if self.is_varsize(typeid): + lenaddr = obj + self.varsize_offset_to_length(typeid) + length = lenaddr.signed[0] + size += length * self.varsize_item_sizes(typeid) + size = llarena.round_up_for_allocation(size) + return size + def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/generation.py Thu Sep 18 18:23:23 2008 @@ -1,6 +1,6 @@ import sys from pypy.rpython.memory.gc.semispace import SemiSpaceGC -from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED +from pypy.rpython.memory.gc.base import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import DEBUG_PRINT from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/hybrid.py Thu Sep 18 18:23:23 2008 @@ -1,8 +1,8 @@ import sys from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import DEBUG_PRINT -from pypy.rpython.memory.gc.generation import GenerationGC, GCFLAG_FORWARDED -from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL +from pypy.rpython.memory.gc.generation import GenerationGC +from pypy.rpython.memory.gc.base import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.generation import GCFLAG_NO_YOUNG_PTRS from pypy.rpython.memory.gc.generation import GCFLAG_NO_HEAP_PTRS from pypy.rpython.lltypesystem import lltype, llmemory, llarena Added: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- (empty file) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Thu Sep 18 18:23:23 2008 @@ -0,0 +1,151 @@ + +from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gcheader import GCHeaderBuilder +from pypy.rlib.debug import ll_assert +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import get_address_stack, get_address_deque +from pypy.rpython.memory.support import AddressDict +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +from pypy.rlib.rarithmetic import ovfcheck +from pypy.rpython.lltypesystem.lloperation import llop + +GCFLAG_MARKBIT = MovingGCBase.first_unused_gcflag + +memoryError = MemoryError() + +class MarkCompactGC(MovingGCBase): + HDR = lltype.Struct('header', ('tid', lltype.Signed)) + + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=16*(1024**2)): + # space_size should be maximal available virtual memory. + # this way we'll never need to copy anything nor implement + # paging on our own + self.space_size = space_size + self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + self.AddressStack = get_address_stack(chunk_size) + self.AddressDeque = get_address_deque(chunk_size) + self.AddressDict = AddressDict + self.counter = 0 + + def setup(self): + self.space = llarena.arena_malloc(self.space_size, True) + ll_assert(bool(self.space), "couldn't allocate arena") + self.spaceptr = self.space + + def init_gc_object(self, addr, typeid, flags=0): + hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) + hdr.tid = typeid | flags + + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): + assert can_collect + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + self.eventually_collect() + result = self.spaceptr + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + self.spaceptr += totalsize + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): + assert can_collect + size_gc_header = self.gcheaderbuilder.size_gc_header + nonvarsize = size_gc_header + size + try: + varsize = ovfcheck(itemsize * length) + totalsize = ovfcheck(nonvarsize + varsize) + except OverflowError: + raise memoryError + self.eventually_collect() + result = self.spaceptr + llarena.arena_reserve(result, totalsize) + self.init_gc_object(result, typeid) + (result + size_gc_header + offset_to_length).signed[0] = length + self.spaceptr = result + llarena.round_up_for_allocation(totalsize) + # XXX has_finalizer etc. + return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + + def eventually_collect(self): + # XXX this is a very bad idea, come up with better heuristics + # XXX it also does not check can_collect flag + self.counter += 1 + if self.counter == 1000: + self.collect() + self.counter = 0 + + def collect(self): + self.debug_check_consistency() + self.mark() + self.compact() + self.update_forward_refs() + self.debug_check_consistency() + + def compact(self): + self.forwardrefs = self.AddressDict() + fromaddr = self.space + toaddr = self.space + size_gc_header = self.gcheaderbuilder.size_gc_header + while fromaddr < self.spaceptr: + hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) + obj = fromaddr + size_gc_header + objsize = self.get_size(obj) + totalsize = size_gc_header + objsize + if not hdr.tid & GCFLAG_MARKBIT: + # this object dies, clear arena + llarena.arena_reset(fromaddr, totalsize, True) + else: + # this objects survives, clear MARKBIT + hdr.tid &= ~GCFLAG_MARKBIT + if fromaddr != toaddr: + # this object needs to be copied somewhere + + # first approach - copy object if possible, otherwise + # copy it somewhere else and keep track of that + self.forwardrefs.setitem(fromaddr, toaddr) + if toaddr + totalsize > fromaddr: + # this is the worst possible scenario: object does + # not fit inside space to copy + xxx + else: + # object fits there: copy + llop.debug_print(lltype.Void, fromaddr, "copied to", toaddr, + "tid", self.header(obj).tid, + "size", totalsize) + llarena.arena_reserve(toaddr, totalsize) + llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize) + llarena.arena_reset(fromaddr, totalsize, True) + toaddr += size_gc_header + objsize + fromaddr += size_gc_header + objsize + self.spaceptr = toaddr + + def update_forward_refs(self): + ptr = self.space + size_gc_header = self.gcheaderbuilder.size_gc_header + while ptr < self.spaceptr: + obj = ptr + size_gc_header + objsize = self.get_size(obj) + totalsize = size_gc_header + objsize + self.trace(obj, self._trace_copy, None) + ptr += totalsize + self.forwardrefs = None + + def _trace_copy(self, pointer, ignored): + addr = pointer.address[0] + if addr != NULL: + pointer.address[0] = self.forwardrefs.get(addr, addr) + + def mark(self): + self.root_walker.walk_roots( + MarkCompactGC._mark_object, # stack roots + MarkCompactGC._mark_object, # static in prebuilt non-gc structures + MarkCompactGC._mark_object) # static in prebuilt gc objects + + def _mark_object(self, pointer, ignored=None): + obj = pointer.address[0] + if obj != NULL: + self.header(obj).tid |= GCFLAG_MARKBIT + self.trace(obj, self._mark_object, None) Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/marksweep.py Thu Sep 18 18:23:23 2008 @@ -22,8 +22,6 @@ DEBUG_PRINT = False memoryError = MemoryError() class MarkSweepGC(GCBase): - _alloc_flavor_ = "raw" - HDR = lltype.ForwardReference() HDRPTR = lltype.Ptr(HDR) # need to maintain a linked list of malloced objects, since we used the Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py Thu Sep 18 18:23:23 2008 @@ -10,18 +10,12 @@ from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc.base import MovingGCBase, GCFLAG_EXTERNAL +from pypy.rpython.memory.gc.base import GCFLAG_FORWARDED +from pypy.rpython.memory.gc.base import GCFLAG_FINALIZATION_ORDERING import sys, os -TYPEID_MASK = 0xffff -first_gcflag = 1 << 16 -GCFLAG_FORWARDED = first_gcflag -# GCFLAG_EXTERNAL is set on objects not living in the semispace: -# either immortal objects or (for HybridGC) externally raw_malloc'ed -GCFLAG_EXTERNAL = first_gcflag << 1 -GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 - DEBUG_PRINT = False memoryError = MemoryError() @@ -30,7 +24,7 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True malloc_zero_filled = True - first_unused_gcflag = first_gcflag << 3 + total_collection_time = 0.0 total_collection_count = 0 @@ -388,40 +382,6 @@ stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj - def get_size(self, obj): - typeid = self.get_type_id(obj) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - lenaddr = obj + self.varsize_offset_to_length(typeid) - length = lenaddr.signed[0] - size += length * self.varsize_item_sizes(typeid) - size = llarena.round_up_for_allocation(size) - return size - - def header(self, addr): - addr -= self.gcheaderbuilder.size_gc_header - return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - - def get_type_id(self, addr): - tid = self.header(addr).tid - ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, - "get_type_id on forwarded obj") - # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. - # Although calling get_type_id() on a forwarded object works by itself, - # we catch it as an error because it's likely that what is then - # done with the typeid is bogus. - return tid & TYPEID_MASK - - def init_gc_object(self, addr, typeid, flags=0): - hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags - - def init_gc_object_immortal(self, addr, typeid, flags=0): - hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED - # immortal objects always have GCFLAG_FORWARDED set; - # see get_forwarding_address(). - def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/test/test_direct.py Thu Sep 18 18:23:23 2008 @@ -279,3 +279,7 @@ 'large_object_gcptrs': 12, 'generation3_collect_threshold': 5, } + +class TestMarkCompactGC(DirectGCTest): + from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + From hpk at codespeak.net Thu Sep 18 18:51:28 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 18 Sep 2008 18:51:28 +0200 (CEST) Subject: [pypy-svn] r58229 - in pypy/build/benchmem: . testing Message-ID: <20080918165128.30B4316A021@codespeak.net> Author: hpk Date: Thu Sep 18 18:51:26 2008 New Revision: 58229 Added: pypy/build/benchmem/smaps.py Modified: pypy/build/benchmem/benchtool.py pypy/build/benchmem/testing/test_benchtool.py Log: another intermediate checkin, factoring out smaps reading, logging to a file instead of a dir and logging some initial header info for each benchmark run. Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Thu Sep 18 18:51:26 2008 @@ -1,31 +1,38 @@ +""" + benchtool.py [--benchlog=logfile] [--executable=path1] [benchname1.py] [benchname2.py] + + record memory usage for given benchmarks (or all if none specified). + +""" import py import os, sys, re +import smaps +optparse = py.compat.optparse mydir = py.magic.autopath().dirpath() benchmarkdir = mydir.join("benchmark") + class BenchRunner(object): - def __init__(self, executable, resultdir): - self.resultdir = py.path.local(resultdir) + def __init__(self, executable, benchlog): + self.benchlog = py.path.local(benchlog) self.executable = executable - + self.tmpdir = py.path.local.make_numbered_dir(prefix="bench") + def log(self, *args): print " ".join(map(str, args)) - def getbenchpath(self, filename): - return benchmarkdir.join(filename) - - def _preparebench(self, filename, args): - path = self.getbenchpath(filename) + def _preparebench(self, path, args): + path = py.path.local(path) for name, obj in vars(path.pyimport()).items(): if name.startswith("bench") and callable(obj): - break + break # xxx only first bench function is considered else: raise LookupError("no benchmark found in %s" %(path,)) arglist = ",".join(map(str, args)) source = py.code.Source(path.read(), """ def checkpoint(): - sys.stdout.write(".") + sys.stdout.write("c") sys.stdin.read(1) if __name__ == "__main__": import os, sys @@ -35,11 +42,21 @@ checkpoint() %s(checkpoint, %s) checkpoint() + sys.stdout.write("F") + sys.stdout.close() """ %(name, arglist)) - p = self.resultdir.join("%s.py" %(name,)) + p = self.tmpdir.join(path.basename) p.write(source) return p + def writeexecinfo(self, benchname): + f = self.benchlog.open("a") + print >>f, "=" * 80 + print >>f, "#executable=%r" %(str(self.executable ),) + print >>f, "#benchname=%r" %(benchname,) + f.close() + # xxx add more + def run_checkpointed_bench(self, filename, args): benchpyfile = self._preparebench(filename, args) self.log("created", benchpyfile) @@ -47,134 +64,85 @@ self.log("exec", cmd) stdout, stdin = os.popen2(cmd) pid = int(stdin.readline()) - logpath = benchpyfile.new(ext="smaps") - smaps = SmapsRecorder(pid, logpath) + + self.writeexecinfo(benchpyfile.basename) + rec = smaps.SmapsRecorder(pid, self.benchlog) stdout.write(".") stdout.flush() while not stdin.closed: c = stdin.read(1) - if not c: + if c == "F": break - smaps.snapshot() + rec.snapshot() stdout.write(".") stdout.flush() + #sys.stdout.write(".") + #sys.stdout.flush() + rec.finish() self.log("finished", cmd) - return logpath -class SmapsRecorder: - SEPLINE = "="*80 + "\n" +class BenchResult: + pass + +class BenchlogReader(object): + def __init__(self, logpath): + self.logpath = logpath + + def getname2results(self): + name2results = {} + for result in self.readresults(): + l = name2results.setdefault(result.benchname, []) + l.append(result) + return name2results - def __init__(self, pid, logpath): - self.logpath = py.path.local(logpath) - self._file = self.logpath.open("a") - self.pid = pid - self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) - assert self.smapspath.check() - - def snapshot(self): - s = self.smapspath.read() - self._file.write(s) - self._file.write("\n") - self._file.write(self.SEPLINE) - self._file.flush() - -class SmapsReader: - def __init__(self, snapshots): - self.snapshots = snapshots - - @classmethod - def parse(cls, path): - """ parse a file previously recorded through SmapsRecorder. """ - s = path.read() - parts = filter(None, map(str.strip, s.split(SmapsRecorder.SEPLINE))) - snapshots = [SmapsProcess.parse(s) for s in parts] - return cls(snapshots) - - def _verify_integrity(self): - for snap in self.snapshots: - for mapping in snap.mappings: - clean = mapping.shared_clean + mapping.private_clean - dirty = mapping.private_dirty + mapping.shared_dirty - assert mapping.rss == dirty + clean + def readresults(self): + xxx -class SmapsMapping: - def __init__(self, headvars, bodyvars): - self.__dict__.update(headvars) - self.__dict__.update(bodyvars) - self.inode = int(self.inode) - -class SmapsProcess(object): - # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' - headrex = re.compile(r""" - (?P\w+-\w+)\s+ - (?P[rwxps-]+)\s+ - (?P\w+)\s+ - (?P\w\w:\w\w)\s+ - (?P\w+)\s+ - (?P\S*)""", - re.VERBOSE) - # example 'Size: 4648 kB' - linerex = re.compile(r"(\S+):\s+(\d+)\skB") - - def __init__(self, mappings): - self.mappings = mappings - self.UNIT = "kB" - for name in ("size rss shared_clean shared_dirty " - "private_clean private_dirty".split()): - setattr(self, name, sum([getattr(x, name) for x in mappings])) - - def memusage(self): - return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) - - @classmethod - def parse(cls, string): - lines = string.split("\n") - mappings = [] - while lines: - line = lines.pop(0) - if not line.strip(): - continue - m = cls.headrex.match(line) - if m is None: - print "ignoring", line - continue - headvars = m.groupdict() - bodyvars = {} - while lines: - line = lines.pop(0) - m = cls.linerex.match(line) - if m is None: - lines.insert(0, line) - mappings.append(SmapsMapping(headvars, bodyvars)) - break - name, num = m.groups() - name = name.lower() - num = int(num) - assert name not in bodyvars - bodyvars[name] = num - else: - mappings.append(SmapsMapping(headvars, bodyvars)) - return cls(mappings) - -if __name__ == '__main__': - executable = sys.argv[1] - name = sys.argv[2] - resultdir = py.path.local("resultdir").ensure(dir=1) - print "using", resultdir - benchrunner = BenchRunner(executable, resultdir) - logpath = benchrunner.run_checkpointed_bench(name, (100, 1000)) - reader = SmapsReader.parse(logpath) - last = None - for i, snap in enumerate(reader.snapshots): - if last is None: - last = snap - print i, snap.memusage() - continue - if (last.private_dirty != snap.private_dirty or - last.shared_dirty != snap.shared_dirty or - len(last.mappings) != snap.mappings): - print i, snap.memusage()#, snap.memdiff(last) +# +# ============================================================================== +# Option Handling + +parser = optparse.OptionParser(usage=__doc__) +parser.add_option("-e", "--executable", action="store", dest="executable", default="python2.5", + help="Python executable to use for recording benchmarks") +parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", + help="logfile for recording benchmark measurements") + +def getexecutable(options): + executable = options.executable + if os.sep not in executable: + executable = py.path.local.sysfind("python2.5") + assert executable, "%s not found" %(executable) + executable = py.path.local(executable) + assert executable.check(), "%s does not exist" % executable + return executable + +def getbenchfiles(options, args): + if args: + benchfiles = [py.path.local(x) for x in args] else: - print i, snap.memusage()#, snap.memdiff(last) - + benchfiles = [x for x in benchmarkdir.listdir('*.py') + if x.basename[0] != "_"] + for x in benchfiles: + if not x.check(): + raise LookupError("%s does not exist" %(x,)) + assert benchfiles + return benchfiles + +def getbenchlog(options): + benchlog = options.benchlog + if benchlog is None: + benchlog = "bench.log" + return py.path.local(benchlog) + +if __name__ == '__main__': + (options, args) = parser.parse_args() + + executable = getexecutable(options) + names = getbenchfiles(options, args) + benchlog = getbenchlog(options) + + runner = BenchRunner(executable, benchlog) + for name in names: + runner.run_checkpointed_bench(name, (100, 100)) Added: pypy/build/benchmem/smaps.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/smaps.py Thu Sep 18 18:51:26 2008 @@ -0,0 +1,100 @@ +import os, sys, re +import py + +class SmapsRecorder: + SEPLINE = "-"*80 + "\n" + + def __init__(self, pid, logpath): + self.logpath = py.path.local(logpath) + self._file = self.logpath.open("a") + self.pid = pid + self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) + assert self.smapspath.check() + + def snapshot(self): + s = self.smapspath.read() + self._file.write(s) + self._file.write(self.SEPLINE) + self._file.flush() + + def finish(self): + self._file.close() + +class SmapsReader: + def __init__(self, snapshots): + self.snapshots = snapshots + + @classmethod + def parse(cls, path): + """ parse a file previously recorded through SmapsRecorder. """ + s = path.read() + parts = filter(None, map(str.strip, s.split(SmapsRecorder.SEPLINE))) + snapshots = [SmapsSnapshot.parse(s) for s in parts] + return cls(snapshots) + + def _verify_integrity(self): + for snap in self.snapshots: + for mapping in snap.mappings: + clean = mapping.shared_clean + mapping.private_clean + dirty = mapping.private_dirty + mapping.shared_dirty + assert mapping.rss == dirty + clean + + +class SmapsMapping: + def __init__(self, headvars, bodyvars): + self.__dict__.update(headvars) + self.__dict__.update(bodyvars) + self.inode = int(self.inode) + +class SmapsSnapshot(object): + # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' + headrex = re.compile(r""" + (?P\w+-\w+)\s+ + (?P[rwxps-]+)\s+ + (?P\w+)\s+ + (?P\w\w:\w\w)\s+ + (?P\w+)\s+ + (?P\S*)""", + re.VERBOSE) + # example 'Size: 4648 kB' + linerex = re.compile(r"(\S+):\s+(\d+)\skB") + + def __init__(self, mappings): + self.mappings = mappings + self.UNIT = "kB" + for name in ("size rss shared_clean shared_dirty " + "private_clean private_dirty".split()): + setattr(self, name, sum([getattr(x, name) for x in mappings])) + + def memusage(self): + return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) + + @classmethod + def parse(cls, string): + lines = string.split("\n") + mappings = [] + while lines: + line = lines.pop(0) + if not line.strip(): + continue + m = cls.headrex.match(line) + if m is None: + print "ignoring", line + continue + headvars = m.groupdict() + bodyvars = {} + while lines: + line = lines.pop(0) + m = cls.linerex.match(line) + if m is None: + lines.insert(0, line) + mappings.append(SmapsMapping(headvars, bodyvars)) + break + name, num = m.groups() + name = name.lower() + num = int(num) + assert name not in bodyvars + bodyvars[name] = num + else: + mappings.append(SmapsMapping(headvars, bodyvars)) + return cls(mappings) Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Thu Sep 18 18:51:26 2008 @@ -2,6 +2,7 @@ import os, sys import benchtool +import smaps def setup_module(mod): if sys.platform.find("linux") == -1: @@ -11,30 +12,35 @@ tmpdir = py.test.ensuretemp("smapsrecorder") logpath = tmpdir.join("logfile") pid = os.getpid() - rec = benchtool.SmapsRecorder(pid=pid, logpath=logpath) + rec = smaps.SmapsRecorder(pid=pid, logpath=logpath) s = logpath.read() - assert s.count(benchtool.SmapsRecorder.SEPLINE) == 0 + assert s.count(smaps.SmapsRecorder.SEPLINE) == 0 rec.snapshot() rec.snapshot() del rec s = logpath.read() - assert s.count(benchtool.SmapsRecorder.SEPLINE) == 2 + assert s.count(smaps.SmapsRecorder.SEPLINE) == 2 def test_benchrunner_functional(): tmpdir = py.test.ensuretemp("benchrunner") - runner = benchtool.BenchRunner(executable="python2.5", resultdir=tmpdir) - def checker(name, *args): - smapsfile = runner.run_checkpointed_bench(name, args) - assert smapsfile.check() - reader = benchtool.SmapsReader.parse(smapsfile) + log=tmpdir.join("log") + runner = benchtool.BenchRunner(executable="python2.5", benchlog=log) + def checker(path, *args): + if log.check(): + log.remove() + runner.run_checkpointed_bench(path, args) + assert log.check() + reader = smaps.SmapsReader.parse(log) + #assert reader.executable + #assert reader.executable assert len(reader.snapshots) == 10 + 2 reader._verify_integrity() - for name in benchtool.benchmarkdir.listdir("*.py"): - if name.basename[0] != "_": - yield checker, name.basename, 10, 10 + for path in benchtool.benchmarkdir.listdir("*.py"): + if path.basename[0] != "_": + yield checker, path, 10, 10 -class TestSmapsProcess: +class TestSmapsSnapshot: def setup_class(cls): cls.s = py.std.textwrap.dedent("""\ 08048000-0813f000 r-xp 00000000 fd:00 75457 sometext @@ -53,7 +59,7 @@ Private_Dirty: 8 kB""") def test_parse_mappings(self): - smapsreader = benchtool.SmapsProcess.parse(self.s) + smapsreader = smaps.SmapsSnapshot.parse(self.s) mappings = smapsreader.mappings assert len(mappings) == 2 mapping = mappings[0] @@ -85,12 +91,12 @@ #assert mapping.referenced == 12 def test_summing(self): - smaps = benchtool.SmapsProcess.parse(self.s) + snap = smaps.SmapsSnapshot.parse(self.s) for name in ('size rss shared_clean shared_dirty ' 'private_clean private_dirty').split(): - sumval = getattr(smaps, name) - val1 = getattr(smaps.mappings[0], name) - val2 = getattr(smaps.mappings[1], name) + sumval = getattr(snap, name) + val1 = getattr(snap.mappings[0], name) + val2 = getattr(snap.mappings[1], name) assert sumval == val1 + val2 example_data2 = ''' From fijal at codespeak.net Thu Sep 18 22:37:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 18 Sep 2008 22:37:45 +0200 (CEST) Subject: [pypy-svn] r58230 - pypy/extradoc/talk/maemo-summit-2008 Message-ID: <20080918203745.8B91B168545@codespeak.net> Author: fijal Date: Thu Sep 18 22:37:43 2008 New Revision: 58230 Added: pypy/extradoc/talk/maemo-summit-2008/ pypy/extradoc/talk/maemo-summit-2008/lightning.txt (contents, props changed) Log: (very draftish) Draft of some slides for tomorrow Added: pypy/extradoc/talk/maemo-summit-2008/lightning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/maemo-summit-2008/lightning.txt Thu Sep 18 22:37:43 2008 @@ -0,0 +1,37 @@ +Why yet another python interpreter? +=================================== + +* PyPy - very compatible to CPython + +* Interpreter written in a static subset of Python + +* Waaaay easier to experiment with + +Interpreter startup time +======================== + +XXX + +Memory usage +============ + +* Pluggable garbage collectors - why not come with something + embedded friendly? + +* Less per-object overhead + +* Impossible in CPython optimizations + +Multiple interpreters +===================== + +* Again, pluggable GCs + +* Share some static data (pyc files, modules, etc.) + +JIT +=== + +* In progress for x86 + +* Why not run on ARM as well? From nshepperd at codespeak.net Fri Sep 19 08:15:42 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Fri, 19 Sep 2008 08:15:42 +0200 (CEST) Subject: [pypy-svn] r58232 - pypy/branch/new-platformcheck/pypy/rpython/tool Message-ID: <20080919061542.401A516A095@codespeak.net> Author: nshepperd Date: Fri Sep 19 08:15:40 2008 New Revision: 58232 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Log: Use eci for platcheck compilation again. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Fri Sep 19 08:15:40 2008 @@ -10,6 +10,7 @@ from pypy.tool.gcc_cache import build_executable_cache_read, try_compile_cache from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.tool.cbuild import CompilationError +from pypy.translator.tool.cbuild import build_executable from pypy.tool.udir import udir import distutils @@ -267,31 +268,29 @@ return data[0] != '\x00' def get_symbol_data(eci, name, NM, OBJCOPY): - link_extra = ' '.join(list(eci.link_extra) + ['-static', '-Wl,--trace', '-Wl,--whole-archive']) - libraries = ' '.join(['-l' + lib for lib in eci.libraries]) - libdirs = ' '.join(['-L' + _dir for _dir in eci.library_dirs]) - dir = mkdtemp() - srcpath = os.path.join(dir, 'main.c') - objpath = os.path.join(dir, 'main.o') - exepath = os.path.join(dir, 'main.exe') - src = open(srcpath, 'w') - src.write('int main() {}\n') - src.close() - if os.system('gcc -c -o %s %s' % (objpath, srcpath)): - raise CompilationError('gcc does not work') - linkcmd = ' '.join(['gcc -o ', exepath, objpath, link_extra, libdirs, libraries]) - link = Popen(linkcmd, shell=True, stdout=PIPE, stderr=PIPE) + eci = ExternalCompilationInfo(**eci._copy_attributes()) + eci.link_extra = list(eci.link_extra) + ['-static', '-Wl,--trace', '-Wl,--whole-archive'] + eci = eci.convert_sources_to_files(being_main=True) + root = mkdtemp() + srcpath = os.path.join(root, 'main.c') + src = open(srcpath, 'w') ; src.close() + try: + path = build_executable([srcpath], eci) + '.errors' + data = open(path, 'r').read() + except CompilationError, e: + data = e.message - linked = [] - for line in link.stdout.readlines(): + linked = set() + for line in data.splitlines(): line = line.strip() if os.path.exists(line): - linked.append(line) - sub = line.split('(') - if len(sub) >= 2: - sub = sub[1].split(')')[0].strip() - if os.path.exists(sub) and sub not in linked: - linked.append(sub) + linked.add(line) + else: + sub = line.split('(') + if len(sub) >= 2: + sub = sub[1].split(')')[0].strip() + if os.path.exists(sub): + linked.add(sub) for obj in linked: nm = Popen(NM + ' -f sysv ' + obj, shell=True, stdout=PIPE, stderr=PIPE) @@ -312,13 +311,18 @@ else: continue base_offset = offset - this_member = None - for line in nm: - if is_archive: + if is_archive: + this_member = None + for line in nm: if '[' in line and ']' in line: this_member = line.partition('[')[2].partition(']')[0] continue - if not is_archive or this_member == member: + if this_member == member: + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[6] == section and line[2] != 'U': + base_offset = min(base_offset, atoi(line[1], 16)) + else: + for line in nm: line = [part.strip() for part in line.split('|')] if len(line) == 7 and line[6] == section and line[2] != 'U': base_offset = min(base_offset, atoi(line[1], 16)) From hpk at codespeak.net Fri Sep 19 08:17:37 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 19 Sep 2008 08:17:37 +0200 (CEST) Subject: [pypy-svn] r58233 - pypy/extradoc/talk/maemo-summit-2008 Message-ID: <20080919061737.23F7016A095@codespeak.net> Author: hpk Date: Fri Sep 19 08:17:36 2008 New Revision: 58233 Modified: pypy/extradoc/talk/maemo-summit-2008/lightning.txt Log: trying to complete slides for today Modified: pypy/extradoc/talk/maemo-summit-2008/lightning.txt ============================================================================== --- pypy/extradoc/talk/maemo-summit-2008/lightning.txt (original) +++ pypy/extradoc/talk/maemo-summit-2008/lightning.txt Fri Sep 19 08:17:36 2008 @@ -1,37 +1,94 @@ -Why yet another python interpreter? +================================ +PyPy Python on Maemo +================================ + +What is PyPy? =================================== -* PyPy - very compatible to CPython +* Python Interpreter written in (R)Python + +* framework for implementing dynamic languages + +* goal: new deal on flexibility and speed + +Why yet another Python interpreter? +=================================== + +* single source translated to C/LLVM/JVM/.NET + +* highly compatible to CPython -* Interpreter written in a static subset of Python +* super-fast JIT Compilation (in-progress) -* Waaaay easier to experiment with +* metaprogramming: Waaaay easier to experiment with -Interpreter startup time +* sandboxing, transparent proxies, ... + +PyPy-c on Maemo / expectations +===================================== + +* faster startup time (starts from image) +* Less per-object memory overhead (new GCs) +* more robust (built-in) +* high potential for optimizations +* run multiple Pythons efficiently +* but: slower (0.8-4) for now +* but: higher interpreter base size for now + +PyPy-c on Maemo / status ======================== -XXX +* working on benchmarking memory and startup time +* working on cross-compilation (scratchbox) +* first pypy-c starts up on device +* plan for next months: + * fully working pypy-c/maemo + * extensive benchmarking + * some optimizations, maybe new GC -Memory usage -============ +Garbage Collectors +======================== -* Pluggable garbage collectors - why not come with something - embedded friendly? +* PyPy has pluggable garbage collectors -* Less per-object overhead +* why not come up with something embedded friendly? * Impossible in CPython optimizations +* reusable for other language implementations! + +JIT +=== + +* In progress for x86 + +* Why not run on ARM as well? + +* speed/memory tradeoffs ?! + Multiple interpreters ===================== +* goal: run multiple Python processes efficiently + * Again, pluggable GCs * Share some static data (pyc files, modules, etc.) -JIT -=== +Contact / Q&A +========================== -* In progress for x86 +holger krekel, Maciej Fijalkowski +at http://merlinux.eu -* Why not run on ARM as well? +PyPy: http://codespeak.net/pypy + +Blog: http://morepypy.blogspot.com + +.. raw:: latex + + \begin{figure} + \includegraphics[width=64px,height=64px]{merlinux-logo.jpg} + \qquad + \includegraphics[width=80px]{../../img/py-web.png} + \end{figure} From nshepperd at codespeak.net Fri Sep 19 08:46:50 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Fri, 19 Sep 2008 08:46:50 +0200 (CEST) Subject: [pypy-svn] r58234 - pypy/branch/new-platformcheck/pypy/rpython/tool Message-ID: <20080919064650.B587E16A088@codespeak.net> Author: nshepperd Date: Fri Sep 19 08:46:47 2008 New Revision: 58234 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Log: Read symbols from a compiled executable: eliminates the need to find out where libraries are being linked from. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Fri Sep 19 08:46:47 2008 @@ -269,38 +269,33 @@ def get_symbol_data(eci, name, NM, OBJCOPY): eci = ExternalCompilationInfo(**eci._copy_attributes()) - eci.link_extra = list(eci.link_extra) + ['-static', '-Wl,--trace', '-Wl,--whole-archive'] + eci.link_extra = ['-static'] + list(eci.link_extra) eci = eci.convert_sources_to_files(being_main=True) root = mkdtemp() - srcpath = os.path.join(root, 'main.c') - src = open(srcpath, 'w') ; src.close() - try: - path = build_executable([srcpath], eci) + '.errors' - data = open(path, 'r').read() - except CompilationError, e: - data = e.message + srcpath = uniquefilepath() + binpath = os.path.join(root, 'main.bin') + exepath = os.path.join(root, 'main') + src = open(str(srcpath), 'w') + src.write('int main() {}') + src.close() + data = build_executable_cache_read([srcpath], eci) + exe = open(exepath, 'wb') + exe.write(data) + exe.close() - linked = set() - for line in data.splitlines(): - line = line.strip() - if os.path.exists(line): - linked.add(line) - else: - sub = line.split('(') - if len(sub) >= 2: - sub = sub[1].split(')')[0].strip() - if os.path.exists(sub): - linked.add(sub) - - for obj in linked: - nm = Popen(NM + ' -f sysv ' + obj, shell=True, stdout=PIPE, stderr=PIPE) - nm = list(nm.stdout.readlines()) - is_archive = False + nm = Popen(NM + ' -f sysv ' + exepath, shell=True, stdout=PIPE, stderr=PIPE) + nm = list(nm.stdout.readlines()) + for line in nm: + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[0] == name and line[2] != 'U': + # only if the symbol is a real symbol .....^ + offset = atoi(line[1], 16) + maxsize = atoi(line[4], 16) + section = line[6] + break + else: + name = '_' + name for line in nm: - if '[' in line and ']' in line: - is_archive = True - member = line.partition('[')[2].partition(']')[0] - continue line = [part.strip() for part in line.split('|')] if len(line) == 7 and line[0] == name and line[2] != 'U': # only if the symbol is a real symbol .....^ @@ -309,40 +304,20 @@ section = line[6] break else: - continue - base_offset = offset - if is_archive: - this_member = None - for line in nm: - if '[' in line and ']' in line: - this_member = line.partition('[')[2].partition(']')[0] - continue - if this_member == member: - line = [part.strip() for part in line.split('|')] - if len(line) == 7 and line[6] == section and line[2] != 'U': - base_offset = min(base_offset, atoi(line[1], 16)) - else: - for line in nm: - line = [part.strip() for part in line.split('|')] - if len(line) == 7 and line[6] == section and line[2] != 'U': - base_offset = min(base_offset, atoi(line[1], 16)) - offset = offset - base_offset - break - else: - return None - sec = NamedTemporaryFile() - if is_archive: - obj2 = NamedTemporaryFile() - ar = Popen('ar p %s %s' % (obj, member), shell=True, stdout=PIPE) - data = ar.stdout.read() - obj2.write(data) - obj2.flush() - obj = obj2.name - cmd = '%s -O binary -j \'%s\' %s %s' % (OBJCOPY, section, obj, sec.name) + return None + base_offset = offset + for line in nm: + line = [part.strip() for part in line.split('|')] + if len(line) == 7 and line[6] == section and line[2] != 'U': + base_offset = min(base_offset, atoi(line[1], 16)) + offset -= base_offset + + cmd = '%s -O binary -j \'%s\' %s %s' % (OBJCOPY, section, exepath, binpath) if os.system(cmd): raise CompilationError('objcopy error') - sec.seek(offset) - return sec.read(maxsize) + bin = open(binpath, 'rb') + bin.seek(offset) + return bin.read(maxsize) class CConfigExternEntry(object): """Abstract base class.""" From fijal at codespeak.net Fri Sep 19 09:28:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 19 Sep 2008 09:28:03 +0200 (CEST) Subject: [pypy-svn] r58235 - pypy/extradoc/talk/maemo-summit-2008 Message-ID: <20080919072803.F11BC169ED6@codespeak.net> Author: fijal Date: Fri Sep 19 09:28:01 2008 New Revision: 58235 Added: pypy/extradoc/talk/maemo-summit-2008/l2.txt (contents, props changed) Log: The way I see it Added: pypy/extradoc/talk/maemo-summit-2008/l2.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/maemo-summit-2008/l2.txt Fri Sep 19 09:28:01 2008 @@ -0,0 +1,61 @@ +============= +PyPy on Maemo +============= + +What is PyPy? +============= + +* Yet another python interpreter + +* Written in Python instead of C + +Why yet another python interpreter? +=================================== + +* To have a Python Just in time compiler + +* To experiment with Python implementation more + +* To have a decent GC + +* Whatever you wish... + +Why PyPy on maemo? +================== + +* Pluggable GCs - potential for memory saving + +* Possible faster startup, due to freezing + modules etc. + +* Pluggable object layout in memory - possibility + for saving space. + +* Optimizations impossible on CPython + +* All this things "oh it's too tedious" with CPython + +Status of pypy on maemo +======================= + +* runs + +* some experiments with embedded-friendly gc's + +Contact / Q&A +========================== + +holger krekel, Maciej Fijalkowski +at http://merlinux.eu + +PyPy: http://codespeak.net/pypy + +Blog: http://morepypy.blogspot.com + +.. raw:: latex + + \begin{figure} + \includegraphics[width=64px,height=64px]{merlinux-logo.jpg} + \qquad + \includegraphics[width=80px]{../img/py-web.png} + \end{figure} From nshepperd at codespeak.net Fri Sep 19 11:19:42 2008 From: nshepperd at codespeak.net (nshepperd at codespeak.net) Date: Fri, 19 Sep 2008 11:19:42 +0200 (CEST) Subject: [pypy-svn] r58236 - in pypy/branch/new-platformcheck/pypy/rpython/tool: . test Message-ID: <20080919091942.A57DD169F71@codespeak.net> Author: nshepperd Date: Fri Sep 19 11:19:40 2008 New Revision: 58236 Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Log: Fixed a potential bug with rffi_platform.ConstantString. Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/rffi_platform.py Fri Sep 19 11:19:40 2008 @@ -224,14 +224,15 @@ # ____________________________________________________________ def c_safe_string(string): - return string.replace('\\', '\\\\').replace('"', '\\"') + for (one, two) in [('\\', '\\\\'), ('"', '\\"'), + ('\0', '\\0')]: + string = string.replace(one, two) + return string class CConfigEntry(object): "Abstract base class." def dump(self, name, expr): - beginpad = "__START_PLATCHECK_%s\0%s\0" % ( - c_safe_string(self.key), - c_safe_string(name)) + beginpad = "__START_PLATCHECK_%s\0%s\0" % (self.key, name) return ''' struct __attribute__((packed)) { char begin_pad[%(sizeof_beginpad)i]; @@ -242,7 +243,7 @@ .contents = %(expr)s, .end_pad = "__END_PLATCHECK__" }; - ''' % {'expr' : expr, 'beginpad' : beginpad.replace('\0', '\\0'), + ''' % {'expr' : expr, 'beginpad' : c_safe_string(beginpad), 'sizeof_beginpad' : len(beginpad), 'id' : filter(str.isalnum, self.key+'PLATCHECK'+name)} def dump_by_size(self, name, expr): @@ -283,7 +284,7 @@ exe.write(data) exe.close() - nm = Popen(NM + ' -f sysv ' + exepath, shell=True, stdout=PIPE, stderr=PIPE) + nm = Popen(NM + ' -f sysv ' + exepath, shell=True, stdout=PIPE) nm = list(nm.stdout.readlines()) for line in nm: line = [part.strip() for part in line.split('|')] @@ -521,7 +522,7 @@ yield self.dump('value', self.name) def build_result(self, info, config_result): - return info['value'] + return info['value'].partition('\0')[0] class DefinedConstantInteger(ConstantInteger): """An entry in a CConfig class that stands for an externally Modified: pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/new-platformcheck/pypy/rpython/tool/test/test_rffi_platform.py Fri Sep 19 11:19:40 2008 @@ -235,6 +235,17 @@ print a assert a % struct.calcsize("P") == 0 +def test_constant_string(): + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + pre_include_bits=['#define STR "string"'] + ) + STR = rffi_platform.ConstantString('STR') + STR2 = rffi_platform.ConstantString('"inline string"') + data = rffi_platform.configure(CConfig) + assert data['STR'] == 'string' + assert data['STR2'] == 'inline string' + def test_extern_string(): from tempfile import mkdtemp dir = mkdtemp() From auc at codespeak.net Fri Sep 19 18:14:23 2008 From: auc at codespeak.net (Delmar) Date: Fri, 19 Sep 2008 17:14:23 +0100 Subject: [pypy-svn] Refi your ARM now Message-ID: <48D3C1CF.6060008@codespeak.net> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Fri Sep 19 19:50:07 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Sep 2008 19:50:07 +0200 (CEST) Subject: [pypy-svn] r58240 - in pypy/branch/tuple-nonresizable-395/pypy: module/marshal objspace/std Message-ID: <20080919175007.A3BA816A061@codespeak.net> Author: arigo Date: Fri Sep 19 19:50:06 2008 New Revision: 58240 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: Kill kill code duplication in interp_marshal.py. This makes marshalling lists a bit slower, but I don't really care as marshal is really meant for tuples and other immutable types anyway. This also removes the only place that used the W_SeqObject interface, so we'll see if it makes sense to keep it or not... Modified: pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/marshal/interp_marshal.py Fri Sep 19 19:50:06 2008 @@ -234,18 +234,10 @@ self._overflow() self.nesting -= 1 - # this function is inlined below - def put_list_w(self, list_w, lng): - self.nesting += 1 - self.put_int(lng) - idx = 0 - while idx < lng: - self.put_w_obj(list_w[idx]) - idx += 1 - self.nesting -= 1 - - def put_list_w(self, lst_w, lng): + def put_tuple_w(self, typecode, lst_w): self.nesting += 1 + self.start(typecode) + lng = len(lst_w) self.put_int(lng) idx = 0 space = self.space @@ -258,21 +250,6 @@ self._overflow() self.nesting -= 1 - def put_w_seq(self, w_seq): - self.nesting += 1 - lng = w_seq.getlength() - self.put_int(lng) - idx = 0 - space = self.space - if self.nesting < MAX_MARSHAL_DEPTH: - while idx < lng: - w_obj = w_seq.getitem(idx) - self.space.marshal_w(w_obj, self) - idx += 1 - else: - self._overflow() - self.nesting -= 1 - def _overflow(self): self.raise_exc('object too deeply nested to marshal') @@ -458,18 +435,6 @@ lng = self.get_lng() return self.get(lng) - # this function is inlined below - def get_list_w(self): - self.nesting += 1 - lng = self.get_lng() - res_w = [None] * lng - idx = 0 - while idx < lng: - res_w[idx] = self.get_w_obj(False) - idx += 1 - self.nesting -= 1 - return res_w - def get_w_obj(self, allow_null): self.nesting += 1 space = self.space @@ -486,34 +451,31 @@ return w_ret # inlined version to save a nesting level - def _new_get_list_w(): - def get_list_w(self): - self.nesting += 1 - lng = self.get_lng() - res_w = [None] * lng - idx = 0 - space = self.space - w_ret = space.w_None # something not None - if self.nesting < MAX_MARSHAL_DEPTH: - while idx < lng: - tc = self.get1() - w_ret = self._dispatch[ord(tc)](space, self, tc) - if w_ret is None: - break - res_w[idx] = w_ret - idx += 1 - else: - self._overflow() - if w_ret is None: - raise OperationError(space.w_TypeError, space.wrap( - 'NULL object in marshal data')) - self.nesting -= 1 - return res_w - return get_list_w - - get_list_w = _new_get_list_w() - # another version not to degenerate resulting list to resizable - get_tuple_w = _new_get_list_w() + def get_tuple_w(self): + self.nesting += 1 + lng = self.get_lng() + res_w = [None] * lng + idx = 0 + space = self.space + w_ret = space.w_None # something not None + if self.nesting < MAX_MARSHAL_DEPTH: + while idx < lng: + tc = self.get1() + w_ret = self._dispatch[ord(tc)](space, self, tc) + if w_ret is None: + break + res_w[idx] = w_ret + idx += 1 + else: + self._overflow() + if w_ret is None: + raise OperationError(space.w_TypeError, space.wrap( + 'NULL object in marshal data')) + self.nesting -= 1 + return res_w + + def get_list_w(self): + return self.get_tuple_w()[:] def _overflow(self): self.raise_exc('object too deeply nested to unmarshal') Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Fri Sep 19 19:50:06 2008 @@ -29,7 +29,7 @@ def getlength(self): return len(self.wrappeditems) - def getitem(self, i): + def getitemfast(self, i): return self.wrappeditems[i] Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Fri Sep 19 19:50:06 2008 @@ -87,9 +87,7 @@ put_int(int) puts an integer put_pascal(s) puts a short string put_w_obj(w_obj) puts a wrapped object -put_list_w(lst_w, lng) puts a list with lng of wrapped objects -put_w_seq(w_seq) puts a w_seqobject subclass - +put_tuple_w(TYPE, tuple_w) puts tuple_w, an unwrapped list of wrapped objects """ handled_by_any = [] @@ -315,8 +313,8 @@ register(TYPE_STRINGREF, unmarshal_stringref) def marshal_w__Tuple(space, w_tuple, m): - m.start(TYPE_TUPLE) - m.put_w_seq(w_tuple) + items = w_tuple.wrappeditems + m.put_tuple_w(TYPE_TUPLE, items) def unmarshal_Tuple(space, u, tc): items_w = u.get_tuple_w() @@ -324,8 +322,8 @@ register(TYPE_TUPLE, unmarshal_Tuple) def marshal_w__List(space, w_list, m): - m.start(TYPE_LIST) - m.put_w_seq(w_list) + items = w_list.wrappeditems[:] + m.put_tuple_w(TYPE_LIST, items) def unmarshal_List(space, u, tc): items_w = u.get_list_w() @@ -378,8 +376,7 @@ m.put_int(x.co_stacksize) m.put_int(x.co_flags) m.atom_str(TYPE_STRING, x.co_code) - m.start(TYPE_TUPLE) - m.put_list_w(x.co_consts_w[:], len(x.co_consts_w)) + m.put_tuple_w(TYPE_TUPLE, x.co_consts_w) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w]) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars) @@ -460,46 +457,38 @@ register(TYPE_UNICODE, unmarshal_Unicode) app = gateway.applevel(r''' - def set_to_list(theset): - return [item for item in theset] - - def list_to_set(datalist, frozen=False): + def tuple_to_set(datalist, frozen=False): if frozen: return frozenset(datalist) return set(datalist) ''') -set_to_list = app.interphook('set_to_list') -list_to_set = app.interphook('list_to_set') +tuple_to_set = app.interphook('tuple_to_set') # not directly supported: def marshal_w_set(space, w_set, m): - w_lis = set_to_list(space, w_set) # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.unpackiterable(w_lis) - m.start(TYPE_SET) - m.put_list_w(lis_w, len(lis_w)) + lis_w = space.viewiterable(w_set) + m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - w_lis = set_to_list(space, w_frozenset) - lis_w = space.unpackiterable(w_lis) - m.start(TYPE_FROZENSET) - m.put_list_w(lis_w, len(lis_w)) + lis_w = space.viewiterable(w_frozenset) + m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) def unmarshal_set_frozenset(space, u, tc): - items_w = u.get_list_w() + items_w = u.get_tuple_w() if tc == TYPE_SET: w_frozen = space.w_False else: w_frozen = space.w_True - w_lis = space.newlist(items_w) - return list_to_set(space, w_lis, w_frozen) + w_tup = space.newtuple(items_w) + return tuple_to_set(space, w_tup, w_frozen) register(TYPE_SET + TYPE_FROZENSET, unmarshal_set_frozenset) # dispatching for all not directly dispatched types Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py Fri Sep 19 19:50:06 2008 @@ -11,5 +11,5 @@ def getlength(self): raise NotImplementedError - def getitem(self, i): + def getitemfast(self, i): raise NotImplementedError Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Fri Sep 19 19:50:06 2008 @@ -29,7 +29,7 @@ def getlength(self): return len(self.wrappeditems) - def getitem(self, i): + def getitemfast(self, i): return self.wrappeditems[i] registerimplementation(W_TupleObject) From arigo at codespeak.net Fri Sep 19 19:52:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Sep 2008 19:52:38 +0200 (CEST) Subject: [pypy-svn] r58241 - pypy/branch/tuple-nonresizable-395/pypy/module/_sre Message-ID: <20080919175238.10C9F16A061@codespeak.net> Author: arigo Date: Fri Sep 19 19:52:37 2008 New Revision: 58241 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Log: Revert this change. If it really makes a difference then there is a bug in the list-comprehension-detection code (which should see both versions as identical as far as I can tell). Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Fri Sep 19 19:52:37 2008 @@ -95,10 +95,11 @@ """Creates a tuple of index pairs representing matched groups, a format that's convenient for SRE_Match.""" space = self.space - return space.newtuple([ - space.newtuple([space.wrap(value1), - space.wrap(value2)]) - for value1, value2 in self.create_regs(group_count)]) + lst = [] + for value1, value2 in self.create_regs(group_count): + lst.append(space.newtuple([space.wrap(value1), + space.wrap(value2)])) + return space.newtuple(lst) w_create_regs.unwrap_spec = ['self', int] def fget_start(space, self): From arigo at codespeak.net Sat Sep 20 11:48:07 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 11:48:07 +0200 (CEST) Subject: [pypy-svn] r58248 - pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__ Message-ID: <20080920094807.932AD16A15A@codespeak.net> Author: arigo Date: Sat Sep 20 11:48:06 2008 New Revision: 58248 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py Log: Make bases_w a non-resizable list. Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py Sat Sep 20 11:48:06 2008 @@ -35,7 +35,7 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -78,7 +78,7 @@ raise OperationError( space.w_TypeError, space.wrap("__bases__ must be a tuple object")) - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, @@ -116,7 +116,7 @@ elif name == "__name__": return space.wrap(self.name) elif name == "__bases__": - return space.newtuple(self.bases_w[:]) + return space.newtuple(self.bases_w) w_value = self.lookup(space, w_attr) if w_value is None: raise OperationError( From arigo at codespeak.net Sat Sep 20 11:58:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 11:58:54 +0200 (CEST) Subject: [pypy-svn] r58249 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080920095854.AF3DA16A148@codespeak.net> Author: arigo Date: Sat Sep 20 11:58:53 2008 New Revision: 58249 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typeobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Log: Make W_TypeObject.mro_w a non-resizable list. Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typeobject.py Sat Sep 20 11:58:53 2008 @@ -545,10 +545,10 @@ if not space.is_w(w_where, space.w_type): w_mro_meth = space.get(w_mro_func, w_self) w_mro = space.call_function(w_mro_meth) - w_self.mro_w = space.unpackiterable(w_mro) + w_self.mro_w = space.viewiterable(w_mro) # do some checking here return # done - w_self.mro_w = w_self.compute_default_mro() + w_self.mro_w = w_self.compute_default_mro()[:] # ____________________________________________________________ Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Sat Sep 20 11:58:53 2008 @@ -80,7 +80,7 @@ def descr_get__mro__(space, w_type): w_type = _check(space, w_type) - return space.newtuple(w_type.mro_w[:]) + return space.newtuple(w_type.mro_w) def descr_mro(space, w_type): """Return a type's method resolution order.""" From arigo at codespeak.net Sat Sep 20 12:07:56 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:07:56 +0200 (CEST) Subject: [pypy-svn] r58250 - pypy/branch/tuple-nonresizable-395/pypy/module/_sre Message-ID: <20080920100756.05271169E83@codespeak.net> Author: arigo Date: Sat Sep 20 12:07:56 2008 New Revision: 58250 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Log: Seems that --listcompr doesn't quite work as I expect it to. Anyway I fear we should not _depend_ on listcompr for annotating PyPy. For now let's rewrite this in a way that doesn't depend on listcompr; another solution would be to only call make_sure_not_resized() if listcompr is enabled. Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Sat Sep 20 12:07:56 2008 @@ -95,10 +95,12 @@ """Creates a tuple of index pairs representing matched groups, a format that's convenient for SRE_Match.""" space = self.space - lst = [] - for value1, value2 in self.create_regs(group_count): - lst.append(space.newtuple([space.wrap(value1), - space.wrap(value2)])) + regs = self.create_regs(group_count) + lst = [None] * len(regs) + for i in range(len(regs)): + value1, value2 = regs[i] + lst[i] = space.newtuple([space.wrap(value1), + space.wrap(value2)]) return space.newtuple(lst) w_create_regs.unwrap_spec = ['self', int] From arigo at codespeak.net Sat Sep 20 12:15:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:15:46 +0200 (CEST) Subject: [pypy-svn] r58251 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080920101546.D59FC16A0AA@codespeak.net> Author: arigo Date: Sat Sep 20 12:15:44 2008 New Revision: 58251 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Log: Make W_TypeObject.bases_w a fixed-size list. Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/typetype.py Sat Sep 20 12:15:44 2008 @@ -11,7 +11,7 @@ w_typetype = _precheck_for_new(space, w_typetype) - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) w_winner = w_typetype for base in bases_w: @@ -89,7 +89,7 @@ def descr_get__bases__(space, w_type): w_type = _check(space, w_type) - return space.newtuple(w_type.bases_w[:]) + return space.newtuple(w_type.bases_w) def mro_subclasses(space, w_type, temp): from pypy.objspace.std.typeobject import W_TypeObject, compute_mro @@ -115,7 +115,7 @@ " to %s.__bases__, not %s"% (w_type.name, space.type(w_value).getname(space, '?')))) - newbases_w = space.unpackiterable(w_value) + newbases_w = space.viewiterable(w_value) if len(newbases_w) == 0: raise OperationError(space.w_TypeError, space.wrap("can only assign non-empty tuple" From arigo at codespeak.net Sat Sep 20 12:23:18 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:23:18 +0200 (CEST) Subject: [pypy-svn] r58252 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080920102318.E83A716A0BB@codespeak.net> Author: arigo Date: Sat Sep 20 12:23:18 2008 New Revision: 58252 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py Log: Factor out common code, assuming that len(iterobject) is not a completely performance-critical operation. Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/iterobject.py Sat Sep 20 12:23:18 2008 @@ -16,6 +16,16 @@ w_self.w_seq = w_seq w_self.index = index + def getlength(self, space): + if self.w_seq is None: + return space.wrap(0) + index = self.index + w_length = space.len(self.w_seq) + w_len = space.sub(w_length, space.wrap(index)) + if space.is_true(space.lt(w_len,space.wrap(0))): + w_len = space.wrap(0) + return w_len + class W_SeqIterObject(W_AbstractSeqIterObject): """Sequence iterator implementation for general sequences.""" @@ -66,14 +76,7 @@ return w_item def len__SeqIter(space, w_seqiter): - if w_seqiter.w_seq is None: - return space.wrap(0) - index = w_seqiter.index - w_length = space.len(w_seqiter.w_seq) - w_len = space.sub(w_length, space.wrap(index)) - if space.is_true(space.lt(w_len,space.wrap(0))): - w_len = space.wrap(0) - return w_len + return w_seqiter.getlength(space) def iter__FastTupleIter(space, w_seqiter): @@ -93,13 +96,7 @@ return w_item def len__FastTupleIter(space, w_seqiter): - if w_seqiter.tupleitems is None: - return space.wrap(0) - totallength = len(w_seqiter.tupleitems) - remaining = totallength - w_seqiter.index - if remaining < 0: - remaining = 0 - return space.wrap(remaining) + return w_seqiter.getlength(space) def iter__FastListIter(space, w_seqiter): @@ -119,13 +116,7 @@ return w_item def len__FastListIter(space, w_seqiter): - if w_seqiter.listitems is None: - return space.wrap(0) - totallength = len(w_seqiter.listitems) - remaining = totallength - w_seqiter.index - if remaining < 0: - remaining = 0 - return space.wrap(remaining) + return w_seqiter.getlength(space) def iter__ReverseSeqIter(space, w_seqiter): From arigo at codespeak.net Sat Sep 20 12:25:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:25:03 +0200 (CEST) Subject: [pypy-svn] r58253 - pypy/branch/tuple-nonresizable-395/pypy/objspace/std Message-ID: <20080920102503.A0074169F1E@codespeak.net> Author: arigo Date: Sat Sep 20 12:25:03 2008 New Revision: 58253 Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Log: Unused import. Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Sat Sep 20 12:25:03 2008 @@ -5,7 +5,6 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.interpreter import gateway from pypy.rlib.debug import make_sure_not_resized -from pypy.annotation import model as annmodel class W_TupleObject(W_SeqObject): from pypy.objspace.std.tupletype import tuple_typedef as typedef From arigo at codespeak.net Sat Sep 20 12:30:06 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:30:06 +0200 (CEST) Subject: [pypy-svn] r58254 - pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler Message-ID: <20080920103006.9795D16A128@codespeak.net> Author: arigo Date: Sat Sep 20 12:30:05 2008 New Revision: 58254 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py Log: Don't rely on listcompr here either. That's also a minor clean-up imho. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/opt.py Sat Sep 20 12:30:05 2008 @@ -231,12 +231,13 @@ return self._visitAbstractTest(node, False) def visitTuple(self, node): - for subnode in node.nodes: + nodes = node.nodes + consts_w = [None] * len(nodes) + for i in range(len(nodes)): + subnode = nodes[i] if not isinstance(subnode, ast.Const): return node # not all constants - # this isinstance is only to make annotator happier - consts_w = [subnode.value for subnode in node.nodes - if isinstance(subnode, ast.Const)] + consts_w[i] = subnode.value return ast.Const(self.space.newtuple(consts_w)) def visitFor(self, node): From arigo at codespeak.net Sat Sep 20 12:37:05 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:37:05 +0200 (CEST) Subject: [pypy-svn] r58255 - pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser Message-ID: <20080920103705.088E016A134@codespeak.net> Author: arigo Date: Sat Sep 20 12:37:04 2008 New Revision: 58255 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Log: Kill pointless code. This optimization is done in a more general way later by opt.py. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Sat Sep 20 12:37:04 2008 @@ -421,6 +421,7 @@ items = [] for i in range(0, l, 2): # this is atoms not 1 items.append(atoms[i]) + builder.push(ast.Tuple(items, lineno)) else: # genfor: 'i for i in j' # GenExpr(GenExprInner(Name('i'), [GenExprFor(AssName('i', 'OP_ASSIGN'), Name('j'), [])])))])) @@ -428,15 +429,6 @@ genexpr_for = parse_genexpr_for(atoms[1:]) genexpr_for[0].is_outmost = True builder.push(ast.GenExpr(ast.GenExprInner(expr, genexpr_for, lineno), lineno)) - return - for item in items: - if not isinstance(item, ast.Const): - builder.push(ast.Tuple(items, lineno)) - return - # isinstance as a hint for annotator - values = [item.value for item in items if isinstance(item, ast.Const)] - builder.push(ast.Const(builder.space.newtuple(values), lineno)) - return def build_lambdef(builder, nb): """lambdef: 'lambda' [varargslist] ':' test""" From arigo at codespeak.net Sat Sep 20 12:54:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:54:02 +0200 (CEST) Subject: [pypy-svn] r58256 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920105402.9051216A0BB@codespeak.net> Author: arigo Date: Sat Sep 20 12:54:01 2008 New Revision: 58256 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: Update. Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Sat Sep 20 12:54:01 2008 @@ -1,3 +1,4 @@ * avoid duplication in FastSeqIter, if possible +* check if seqinterface.py is still needed or not * add objspace.unpackiterable shortcuts From arigo at codespeak.net Sat Sep 20 12:54:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 12:54:46 +0200 (CEST) Subject: [pypy-svn] r58257 - pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser Message-ID: <20080920105446.E072416A12F@codespeak.net> Author: arigo Date: Sat Sep 20 12:54:46 2008 New Revision: 58257 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Log: Same as r58255. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyparser/astbuilder.py Sat Sep 20 12:54:46 2008 @@ -726,17 +726,8 @@ if len(atoms) <= 2: builder.push(atoms[0]) else: - isConst = True items = [atoms[index] for index in range(0, len(atoms), 2)] - for item in items: - if not isinstance(item, ast.Const): - isConst = False - break - if not isConst: - builder.push(ast.Tuple([i for i in items if isinstance(i, ast.Node)], atoms[0].lineno)) - else: - builder.push(ast.Const(builder.space.newtuple( - [i for i in items if isinstance(i, ast.Const)]), atoms[0].lineno)) + builder.push(ast.Tuple(items, atoms[0].lineno)) def build_while_stmt(builder, nb): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" From arigo at codespeak.net Sat Sep 20 13:02:12 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 13:02:12 +0200 (CEST) Subject: [pypy-svn] r58258 - pypy/branch/tuple-nonresizable-395/pypy/interpreter Message-ID: <20080920110212.0EBFF16A139@codespeak.net> Author: arigo Date: Sat Sep 20 13:02:09 2008 New Revision: 58258 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Log: Don't rely on listcompr here either. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Sat Sep 20 13:02:09 2008 @@ -15,8 +15,12 @@ # helper -def unpack_str_tuple(space,w_str_tuple): - return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] +def unpack_str_tuple(space, w_str_tuple): + items_w = space.viewiterable(w_str_tuple) + result = [None] * len(items_w) + for i in range(len(items_w)): + result[i] = space.str_w(items_w[i]) + return result # code object contants, for co_flags below CO_OPTIMIZED = 0x0001 From arigo at codespeak.net Sat Sep 20 13:13:30 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 13:13:30 +0200 (CEST) Subject: [pypy-svn] r58259 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920111330.786EA16A15D@codespeak.net> Author: arigo Date: Sat Sep 20 13:13:30 2008 New Revision: 58259 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: Finished reviewing. Updating TODO. Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Sat Sep 20 13:13:30 2008 @@ -1,4 +1,11 @@ -* avoid duplication in FastSeqIter, if possible * check if seqinterface.py is still needed or not + +* remove space.unpacktuple(), replace it with space.viewtuple() + +* viewtuple() should also *check* the type of its argument instead + of segfaulting :-/ (that's also an issue in pypy/dist) + +* use viewtuple() instead of viewiterable() in pycode.py:descr_code__new__() + * add objspace.unpackiterable shortcuts From arigo at codespeak.net Sat Sep 20 13:21:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 13:21:38 +0200 (CEST) Subject: [pypy-svn] r58260 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920112138.4098D16A160@codespeak.net> Author: arigo Date: Sat Sep 20 13:21:36 2008 New Revision: 58260 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: Another pending issue. Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Sat Sep 20 13:21:36 2008 @@ -9,3 +9,5 @@ * use viewtuple() instead of viewiterable() in pycode.py:descr_code__new__() * add objspace.unpackiterable shortcuts + +* "translate.py -b cli" doesn't annotate any more From arigo at codespeak.net Sat Sep 20 13:23:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 13:23:54 +0200 (CEST) Subject: [pypy-svn] r58262 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920112354.89FD116A15D@codespeak.net> Author: arigo Date: Sat Sep 20 13:23:53 2008 New Revision: 58262 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: Again another issue (related to the previous one, likely) Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Sat Sep 20 13:23:53 2008 @@ -10,4 +10,6 @@ * add objspace.unpackiterable shortcuts +* "translate.py --no-listcompr" doesn't annotate any more + * "translate.py -b cli" doesn't annotate any more From arigo at codespeak.net Sat Sep 20 13:32:26 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 13:32:26 +0200 (CEST) Subject: [pypy-svn] r58263 - pypy/branch/tuple-nonresizable-395/pypy/module/posix Message-ID: <20080920113226.3E155169F44@codespeak.net> Author: arigo Date: Sat Sep 20 13:32:25 2008 New Revision: 58263 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Log: Don't rely on --listcompr here either. Modified: pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Sat Sep 20 13:32:25 2008 @@ -555,7 +555,11 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] + l_w = [space.wrap(r[0]), + space.wrap(r[1]), + space.wrap(r[2]), + space.wrap(r[3]), + space.wrap(r[4])] return space.newtuple(l_w) uname.unwrap_spec = [ObjSpace] From fijal at codespeak.net Sat Sep 20 14:19:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 14:19:45 +0200 (CEST) Subject: [pypy-svn] r58265 - in pypy/branch/cross-compilation/pypy: rlib translator/tool/test Message-ID: <20080920121945.304FB16A123@codespeak.net> Author: fijal Date: Sat Sep 20 14:19:43 2008 New Revision: 58265 Modified: pypy/branch/cross-compilation/pypy/rlib/pyplatform.py pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py Log: Fix equality and a test Modified: pypy/branch/cross-compilation/pypy/rlib/pyplatform.py ============================================================================== --- pypy/branch/cross-compilation/pypy/rlib/pyplatform.py (original) +++ pypy/branch/cross-compilation/pypy/rlib/pyplatform.py Sat Sep 20 14:19:43 2008 @@ -21,7 +21,8 @@ return not self == other def __eq__(self, other): - return self.__class__.__name__ == other.__class__.__name__ + return (self.__class__ is other.__class__ and + self.__dict__ == other.__dict__) class Maemo(Platform): def get_compiler(self): Modified: pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/cross-compilation/pypy/translator/tool/test/test_cbuild.py Sat Sep 20 14:19:43 2008 @@ -188,6 +188,18 @@ py.test.fail("Did not raise") assert eci.platform.execute() == 3 + def test_platform_equality(self): + from pypy.rlib.pyplatform import Platform + class X(Platform): + pass + class Y(Platform): + def __init__(self, x): + self.x = x + + assert X() == X() + assert Y(3) == Y(3) + assert Y(2) != Y(3) + def test_standalone_maemo(self): from pypy.rlib.pyplatform import Maemo # XXX skip if there is no scratchbox From fijal at codespeak.net Sat Sep 20 15:20:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 15:20:03 +0200 (CEST) Subject: [pypy-svn] r58268 - in pypy/branch/gc-experiments/pypy/rpython/lltypesystem: . test Message-ID: <20080920132003.46D1A169F2C@codespeak.net> Author: fijal Date: Sat Sep 20 15:20:00 2008 New Revision: 58268 Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-experiments/pypy/rpython/lltypesystem/test/test_llarena.py Log: Add a hack to llarena for having arena "views" Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llarena.py Sat Sep 20 15:20:00 2008 @@ -264,6 +264,9 @@ # work with fakearenaaddresses on which arbitrary arithmetic is # possible even on top of the llinterpreter. +# arena_new_view(ptr) is a no-op when translated, returns fresh view +# on previous arena when run on top of llinterp + def arena_malloc(nbytes, zero): """Allocate and return a new arena, optionally zero-initialized.""" return Arena(nbytes, zero).getaddr(0) @@ -299,6 +302,11 @@ following an object. For arenas containing heterogenous objects.""" return RoundedUpForAllocation(size) +def arena_new_view(ptr): + """Return a fresh memory view on an arena + """ + return Arena(ptr.arena.nbytes, False).getaddr(0) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -399,3 +407,9 @@ llimpl=llimpl_round_up_for_allocation, llfakeimpl=round_up_for_allocation, sandboxsafe=True) + +def llimpl_arena_new_view(addr): + return addr +register_external(arena_new_view, [llmemory.Address], llmemory.Address, + 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, + llfakeimpl=arena_new_view, sandboxsafe=True) Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/lltypesystem/test/test_llarena.py Sat Sep 20 15:20:00 2008 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem.llarena import arena_malloc, arena_reset from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free from pypy.rpython.lltypesystem.llarena import round_up_for_allocation -from pypy.rpython.lltypesystem.llarena import ArenaError +from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -150,6 +150,13 @@ arena_free(a) return 42 +def test_arena_new_view(): + a = arena_malloc(50, False) + arena_reserve(a, precomputed_size) + # we can now allocate the same space in new view + b = arena_new_view(a) + arena_reserve(b, precomputed_size) + def test_partial_arena_reset(): a = arena_malloc(50, False) def reserve(i): From fijal at codespeak.net Sat Sep 20 15:37:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 15:37:40 +0200 (CEST) Subject: [pypy-svn] r58269 - pypy/branch/gc-experiments/pypy/rpython/memory/test Message-ID: <20080920133740.3F89D169F75@codespeak.net> Author: fijal Date: Sat Sep 20 15:37:39 2008 New Revision: 58269 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py Log: Enable gc tests for MarkCompactGC Modified: pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py Sat Sep 20 15:37:39 2008 @@ -455,6 +455,9 @@ class TestGenerationalGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.generation import GenerationGC as GCClass +class TestMarkCompactGC(TestSemiSpaceGC): + from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass GC_CANNOT_MALLOC_NONMOVABLE = False From fijal at codespeak.net Sat Sep 20 15:39:02 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 15:39:02 +0200 (CEST) Subject: [pypy-svn] r58270 - pypy/branch/gc-experiments/pypy/rpython/memory/gc Message-ID: <20080920133902.5346D169F75@codespeak.net> Author: fijal Date: Sat Sep 20 15:39:01 2008 New Revision: 58270 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py Log: Hack until test_direct works. Some code (especially regarding finalizers) makes no sense whatsoever, in-progress Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Sat Sep 20 15:39:01 2008 @@ -1,5 +1,9 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.debug import ll_assert +from pypy.rpython.memory.gcheader import GCHeaderBuilder +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import get_address_stack, get_address_deque +from pypy.rpython.memory.support import AddressDict class GCBase(object): _alloc_flavor_ = "raw" @@ -194,6 +198,22 @@ moving_gc = True first_unused_gcflag = first_gcflag << 3 + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE): + GCBase.__init__(self) + self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + self.AddressStack = get_address_stack(chunk_size) + self.AddressDeque = get_address_deque(chunk_size) + self.AddressDict = AddressDict + self.finalizer_lock_count = 0 + self.id_free_list = self.AddressStack() + self.next_free_id = 1 + + def setup(self): + self.objects_with_finalizers = self.AddressDeque() + self.run_finalizers = self.AddressDeque() + self.objects_with_weakrefs = self.AddressStack() + self.objects_with_id = self.AddressDict() + def can_move(self, addr): return True @@ -231,6 +251,231 @@ size = llarena.round_up_for_allocation(size) return size + + def deal_with_objects_with_finalizers(self, scan): + # walk over list of objects with finalizers + # if it is not copied, add it to the list of to-be-called finalizers + # and copy it, to me make the finalizer runnable + # We try to run the finalizers in a "reasonable" order, like + # CPython does. The details of this algorithm are in + # pypy/doc/discussion/finalizer-order.txt. + new_with_finalizer = self.AddressDeque() + marked = self.AddressDeque() + pending = self.AddressStack() + self.tmpstack = self.AddressStack() + while self.objects_with_finalizers.non_empty(): + x = self.objects_with_finalizers.popleft() + ll_assert(self._finalization_state(x) != 1, + "bad finalization state 1") + if self.surviving(x): + new_with_finalizer.append(self.get_forwarding_address(x)) + continue + marked.append(x) + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 0: + self._bump_finalization_state_from_0_to_1(y) + self.trace(y, self._append_if_nonnull, pending) + elif state == 2: + self._recursively_bump_finalization_state_from_2_to_3(y) + scan = self._recursively_bump_finalization_state_from_1_to_2( + x, scan) + + while marked.non_empty(): + x = marked.popleft() + state = self._finalization_state(x) + ll_assert(state >= 2, "unexpected finalization state < 2") + newx = self.get_forwarding_address(x) + if state == 2: + self.run_finalizers.append(newx) + # we must also fix the state from 2 to 3 here, otherwise + # we leave the GCFLAG_FINALIZATION_ORDERING bit behind + # which will confuse the next collection + self._recursively_bump_finalization_state_from_2_to_3(x) + else: + new_with_finalizer.append(newx) + + self.tmpstack.delete() + pending.delete() + marked.delete() + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizer + return scan + + + def _append_if_nonnull(pointer, stack): + if pointer.address[0] != NULL: + stack.append(pointer.address[0]) + _append_if_nonnull = staticmethod(_append_if_nonnull) + + def _finalization_state(self, obj): + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 2 + else: + return 3 + else: + hdr = self.header(obj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 1 + else: + return 0 + + def _bump_finalization_state_from_0_to_1(self, obj): + ll_assert(self._finalization_state(obj) == 0, + "unexpected finalization state != 0") + hdr = self.header(obj) + hdr.tid |= GCFLAG_FINALIZATION_ORDERING + + def _recursively_bump_finalization_state_from_2_to_3(self, obj): + ll_assert(self._finalization_state(obj) == 2, + "unexpected finalization state != 2") + newobj = self.get_forwarding_address(obj) + pending = self.tmpstack + ll_assert(not pending.non_empty(), "tmpstack not empty") + pending.append(newobj) + while pending.non_empty(): + y = pending.pop() + hdr = self.header(y) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + self.trace(y, self._append_if_nonnull, pending) + + def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): + # recursively convert objects from state 1 to state 2. + # Note that copy() copies all bits, including the + # GCFLAG_FINALIZATION_ORDERING. The mapping between + # state numbers and the presence of this bit was designed + # for the following to work :-) + self.copy(obj) + return self.scan_copied(scan) + + def invalidate_weakrefs(self): + # walk over list of objects that contain weakrefs + # if the object it references survives then update the weakref + # otherwise invalidate the weakref + new_with_weakref = self.AddressStack() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.surviving(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.get_type_id(obj)) + pointing_to = (obj + offset).address[0] + # XXX I think that pointing_to cannot be NULL here + if pointing_to: + if self.surviving(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + new_with_weakref.append(obj) + else: + (obj + offset).address[0] = NULL + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref + + def update_run_finalizers(self): + # we are in an inner collection, caused by a finalizer + # the run_finalizers objects need to be copied + new_run_finalizer = self.AddressDeque() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.popleft() + new_run_finalizer.append(self.copy(obj)) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizer + + def execute_finalizers(self): + self.finalizer_lock_count += 1 + try: + while self.run_finalizers.non_empty(): + #print "finalizer" + if self.finalizer_lock_count > 1: + # the outer invocation of execute_finalizers() will do it + break + obj = self.run_finalizers.popleft() + finalizer = self.getfinalizer(self.get_type_id(obj)) + finalizer(obj) + finally: + self.finalizer_lock_count -= 1 + + def id(self, ptr): + obj = llmemory.cast_ptr_to_adr(ptr) + if self.header(obj).tid & GCFLAG_EXTERNAL: + result = self._compute_id_for_external(obj) + else: + result = self._compute_id(obj) + return llmemory.cast_adr_to_int(result) + + def _next_id(self): + # return an id not currently in use (as an address instead of an int) + if self.id_free_list.non_empty(): + result = self.id_free_list.pop() # reuse a dead id + else: + # make up a fresh id number + result = llmemory.cast_int_to_adr(self.next_free_id) + self.next_free_id += 2 # only odd numbers, to make lltype + # and llmemory happy and to avoid + # clashes with real addresses + return result + + def _compute_id(self, obj): + # look if the object is listed in objects_with_id + result = self.objects_with_id.get(obj) + if not result: + result = self._next_id() + self.objects_with_id.setitem(obj, result) + return result + + def _compute_id_for_external(self, obj): + # For prebuilt objects, we can simply return their address. + # This method is overriden by the HybridGC. + return obj + + def update_objects_with_id(self): + old = self.objects_with_id + new_objects_with_id = self.AddressDict(old.length()) + old.foreach(self._update_object_id_FAST, new_objects_with_id) + old.delete() + self.objects_with_id = new_objects_with_id + + def _update_object_id(self, obj, id, new_objects_with_id): + # safe version (used by subclasses) + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.setitem(newobj, id) + else: + self.id_free_list.append(id) + + def _update_object_id_FAST(self, obj, id, new_objects_with_id): + # unsafe version, assumes that the new_objects_with_id is large enough + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.insertclean(newobj, id) + else: + self.id_free_list.append(id) + + def debug_check_object(self, obj): + """Check the invariants about 'obj' that should be true + between collections.""" + tid = self.header(obj).tid + if tid & GCFLAG_EXTERNAL: + ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded") + ll_assert(not (self.tospace <= obj < self.free), + "external flag but object inside the semispaces") + else: + ll_assert(not (tid & GCFLAG_FORWARDED), "bug: !external+forwarded") + ll_assert(self.tospace <= obj < self.free, + "!external flag but object outside the semispaces") + ll_assert(not (tid & GCFLAG_FINALIZATION_ORDERING), + "unexpected GCFLAG_FINALIZATION_ORDERING") + + def debug_check_can_copy(self, obj): + ll_assert(not (self.tospace <= obj < self.free), + "copy() on already-copied object") + def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Sat Sep 20 15:39:01 2008 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rpython.memory.gc.base import MovingGCBase -from pypy.rpython.memory.gcheader import GCHeaderBuilder +from pypy.rpython.memory.gc.base import MovingGCBase, GCFLAG_FORWARDED,\ + TYPEID_MASK from pypy.rlib.debug import ll_assert from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -15,23 +15,22 @@ memoryError = MemoryError() class MarkCompactGC(MovingGCBase): - HDR = lltype.Struct('header', ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('tid', lltype.Signed), + ('forward_ptr', llmemory.Address)) def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=16*(1024**2)): # space_size should be maximal available virtual memory. # this way we'll never need to copy anything nor implement # paging on our own self.space_size = space_size - self.gcheaderbuilder = GCHeaderBuilder(self.HDR) - self.AddressStack = get_address_stack(chunk_size) - self.AddressDeque = get_address_deque(chunk_size) - self.AddressDict = AddressDict + MovingGCBase.__init__(self, chunk_size) self.counter = 0 def setup(self): self.space = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.space), "couldn't allocate arena") self.spaceptr = self.space + MovingGCBase.setup(self) def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) @@ -47,6 +46,10 @@ llarena.arena_reserve(result, totalsize) self.init_gc_object(result, typeid) self.spaceptr += totalsize + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) + if contains_weakptr: + self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def malloc_varsize_clear(self, typeid, length, size, itemsize, @@ -66,7 +69,8 @@ self.init_gc_object(result, typeid) (result + size_gc_header + offset_to_length).signed[0] = length self.spaceptr = result + llarena.round_up_for_allocation(totalsize) - # XXX has_finalizer etc. + if has_finalizer: + self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) def eventually_collect(self): @@ -80,14 +84,44 @@ def collect(self): self.debug_check_consistency() self.mark() - self.compact() + if self.run_finalizers.non_empty(): + self.update_run_finalizers() + if self.objects_with_finalizers.non_empty(): + self.deal_with_objects_with_finalizers(self.space) + if self.objects_with_weakrefs.non_empty(): + self.invalidate_weakrefs() + self.debug_check_consistency() + toaddr = llarena.arena_new_view(self.space) + self.create_forward_pointers(toaddr) + self.debug_check_consistency() self.update_forward_refs() + self.compact(toaddr) + self.space = toaddr self.debug_check_consistency() - def compact(self): - self.forwardrefs = self.AddressDict() + def create_forward_pointers(self, toaddr): + fromaddr = self.space + size_gc_header = self.gcheaderbuilder.size_gc_header + while fromaddr < self.spaceptr: + hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) + obj = fromaddr + size_gc_header + objsize = self.get_size(obj) + totalsize = size_gc_header + objsize + if hdr.tid & GCFLAG_MARKBIT: + # this objects survives, clear MARKBIT + if fromaddr.offset != toaddr.offset: + # this object is forwarded, set forward bit and address + hdr.tid |= GCFLAG_FORWARDED + llarena.arena_reserve(toaddr, totalsize) + hdr.forward_ptr = toaddr + size_gc_header + toaddr += size_gc_header + objsize + fromaddr += size_gc_header + objsize + + def get_type_id(self, addr): + return self.header(addr).tid & TYPEID_MASK + + def compact(self, toaddr): fromaddr = self.space - toaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header while fromaddr < self.spaceptr: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) @@ -98,31 +132,37 @@ # this object dies, clear arena llarena.arena_reset(fromaddr, totalsize, True) else: - # this objects survives, clear MARKBIT - hdr.tid &= ~GCFLAG_MARKBIT - if fromaddr != toaddr: + if hdr.tid & GCFLAG_FORWARDED: # this object needs to be copied somewhere - # first approach - copy object if possible, otherwise # copy it somewhere else and keep track of that - self.forwardrefs.setitem(fromaddr, toaddr) if toaddr + totalsize > fromaddr: # this is the worst possible scenario: object does # not fit inside space to copy xxx else: # object fits there: copy - llop.debug_print(lltype.Void, fromaddr, "copied to", toaddr, - "tid", self.header(obj).tid, - "size", totalsize) - llarena.arena_reserve(toaddr, totalsize) + hdr.tid &= ~(GCFLAG_MARKBIT|GCFLAG_FORWARDED) + #llop.debug_print(lltype.Void, fromaddr, "copied to", toaddr, + # "tid", self.header(obj).tid, + # "size", totalsize) llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize) llarena.arena_reset(fromaddr, totalsize, True) + else: + hdr.tid &= ~(GCFLAG_MARKBIT|GCFLAG_FORWARDED) + # XXX this is here only to make llarena happier, makes no + # sense whatsoever, need to disable it when translated + llarena.arena_reserve(toaddr, totalsize) + llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize) toaddr += size_gc_header + objsize fromaddr += size_gc_header + objsize self.spaceptr = toaddr def update_forward_refs(self): + self.root_walker.walk_roots( + MarkCompactGC._trace_copy, # stack roots + MarkCompactGC._trace_copy, # static in prebuilt non-gc structures + MarkCompactGC._trace_copy) # static in prebuilt gc objects ptr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header while ptr < self.spaceptr: @@ -131,12 +171,15 @@ totalsize = size_gc_header + objsize self.trace(obj, self._trace_copy, None) ptr += totalsize - self.forwardrefs = None - def _trace_copy(self, pointer, ignored): + def _trace_copy(self, pointer, ignored=None): addr = pointer.address[0] + size_gc_header = self.gcheaderbuilder.size_gc_header if addr != NULL: - pointer.address[0] = self.forwardrefs.get(addr, addr) + hdr = llmemory.cast_adr_to_ptr(addr - size_gc_header, + lltype.Ptr(self.HDR)) + if hdr.tid & GCFLAG_FORWARDED: + pointer.address[0] = hdr.forward_ptr def mark(self): self.root_walker.walk_roots( @@ -149,3 +192,7 @@ if obj != NULL: self.header(obj).tid |= GCFLAG_MARKBIT self.trace(obj, self._mark_object, None) + + def debug_check_object(self, obj): + # XXX write it down + pass Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py Sat Sep 20 15:39:01 2008 @@ -4,7 +4,6 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert @@ -39,17 +38,10 @@ def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): - MovingGCBase.__init__(self) + MovingGCBase.__init__(self, chunk_size) self.space_size = space_size self.max_space_size = max_space_size - self.gcheaderbuilder = GCHeaderBuilder(self.HDR) - self.AddressStack = get_address_stack(chunk_size) - self.AddressDeque = get_address_deque(chunk_size) - self.AddressDict = AddressDict - self.finalizer_lock_count = 0 self.red_zone = 0 - self.id_free_list = self.AddressStack() - self.next_free_id = 1 def setup(self): if DEBUG_PRINT: @@ -61,10 +53,7 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - self.objects_with_finalizers = self.AddressDeque() - self.run_finalizers = self.AddressDeque() - self.objects_with_weakrefs = self.AddressStack() - self.objects_with_id = self.AddressDict() + MovingGCBase.setup(self) # This class only defines the malloc_{fixed,var}size_clear() methods # because the spaces are filled with zeroes in advance. @@ -382,228 +371,5 @@ stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj - def deal_with_objects_with_finalizers(self, scan): - # walk over list of objects with finalizers - # if it is not copied, add it to the list of to-be-called finalizers - # and copy it, to me make the finalizer runnable - # We try to run the finalizers in a "reasonable" order, like - # CPython does. The details of this algorithm are in - # pypy/doc/discussion/finalizer-order.txt. - new_with_finalizer = self.AddressDeque() - marked = self.AddressDeque() - pending = self.AddressStack() - self.tmpstack = self.AddressStack() - while self.objects_with_finalizers.non_empty(): - x = self.objects_with_finalizers.popleft() - ll_assert(self._finalization_state(x) != 1, - "bad finalization state 1") - if self.surviving(x): - new_with_finalizer.append(self.get_forwarding_address(x)) - continue - marked.append(x) - pending.append(x) - while pending.non_empty(): - y = pending.pop() - state = self._finalization_state(y) - if state == 0: - self._bump_finalization_state_from_0_to_1(y) - self.trace(y, self._append_if_nonnull, pending) - elif state == 2: - self._recursively_bump_finalization_state_from_2_to_3(y) - scan = self._recursively_bump_finalization_state_from_1_to_2( - x, scan) - - while marked.non_empty(): - x = marked.popleft() - state = self._finalization_state(x) - ll_assert(state >= 2, "unexpected finalization state < 2") - newx = self.get_forwarding_address(x) - if state == 2: - self.run_finalizers.append(newx) - # we must also fix the state from 2 to 3 here, otherwise - # we leave the GCFLAG_FINALIZATION_ORDERING bit behind - # which will confuse the next collection - self._recursively_bump_finalization_state_from_2_to_3(x) - else: - new_with_finalizer.append(newx) - - self.tmpstack.delete() - pending.delete() - marked.delete() - self.objects_with_finalizers.delete() - self.objects_with_finalizers = new_with_finalizer - return scan - - def _append_if_nonnull(pointer, stack): - if pointer.address[0] != NULL: - stack.append(pointer.address[0]) - _append_if_nonnull = staticmethod(_append_if_nonnull) - - def _finalization_state(self, obj): - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - hdr = self.header(newobj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: - return 2 - else: - return 3 - else: - hdr = self.header(obj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: - return 1 - else: - return 0 - - def _bump_finalization_state_from_0_to_1(self, obj): - ll_assert(self._finalization_state(obj) == 0, - "unexpected finalization state != 0") - hdr = self.header(obj) - hdr.tid |= GCFLAG_FINALIZATION_ORDERING - - def _recursively_bump_finalization_state_from_2_to_3(self, obj): - ll_assert(self._finalization_state(obj) == 2, - "unexpected finalization state != 2") - newobj = self.get_forwarding_address(obj) - pending = self.tmpstack - ll_assert(not pending.non_empty(), "tmpstack not empty") - pending.append(newobj) - while pending.non_empty(): - y = pending.pop() - hdr = self.header(y) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 - self.trace(y, self._append_if_nonnull, pending) - - def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): - # recursively convert objects from state 1 to state 2. - # Note that copy() copies all bits, including the - # GCFLAG_FINALIZATION_ORDERING. The mapping between - # state numbers and the presence of this bit was designed - # for the following to work :-) - self.copy(obj) - return self.scan_copied(scan) - - def invalidate_weakrefs(self): - # walk over list of objects that contain weakrefs - # if the object it references survives then update the weakref - # otherwise invalidate the weakref - new_with_weakref = self.AddressStack() - while self.objects_with_weakrefs.non_empty(): - obj = self.objects_with_weakrefs.pop() - if not self.surviving(obj): - continue # weakref itself dies - obj = self.get_forwarding_address(obj) - offset = self.weakpointer_offset(self.get_type_id(obj)) - pointing_to = (obj + offset).address[0] - # XXX I think that pointing_to cannot be NULL here - if pointing_to: - if self.surviving(pointing_to): - (obj + offset).address[0] = self.get_forwarding_address( - pointing_to) - new_with_weakref.append(obj) - else: - (obj + offset).address[0] = NULL - self.objects_with_weakrefs.delete() - self.objects_with_weakrefs = new_with_weakref - - def update_run_finalizers(self): - # we are in an inner collection, caused by a finalizer - # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressDeque() - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.popleft() - new_run_finalizer.append(self.copy(obj)) - self.run_finalizers.delete() - self.run_finalizers = new_run_finalizer - - def execute_finalizers(self): - self.finalizer_lock_count += 1 - try: - while self.run_finalizers.non_empty(): - #print "finalizer" - if self.finalizer_lock_count > 1: - # the outer invocation of execute_finalizers() will do it - break - obj = self.run_finalizers.popleft() - finalizer = self.getfinalizer(self.get_type_id(obj)) - finalizer(obj) - finally: - self.finalizer_lock_count -= 1 - - def id(self, ptr): - obj = llmemory.cast_ptr_to_adr(ptr) - if self.header(obj).tid & GCFLAG_EXTERNAL: - result = self._compute_id_for_external(obj) - else: - result = self._compute_id(obj) - return llmemory.cast_adr_to_int(result) - - def _next_id(self): - # return an id not currently in use (as an address instead of an int) - if self.id_free_list.non_empty(): - result = self.id_free_list.pop() # reuse a dead id - else: - # make up a fresh id number - result = llmemory.cast_int_to_adr(self.next_free_id) - self.next_free_id += 2 # only odd numbers, to make lltype - # and llmemory happy and to avoid - # clashes with real addresses - return result - - def _compute_id(self, obj): - # look if the object is listed in objects_with_id - result = self.objects_with_id.get(obj) - if not result: - result = self._next_id() - self.objects_with_id.setitem(obj, result) - return result - - def _compute_id_for_external(self, obj): - # For prebuilt objects, we can simply return their address. - # This method is overriden by the HybridGC. - return obj - - def update_objects_with_id(self): - old = self.objects_with_id - new_objects_with_id = self.AddressDict(old.length()) - old.foreach(self._update_object_id_FAST, new_objects_with_id) - old.delete() - self.objects_with_id = new_objects_with_id - - def _update_object_id(self, obj, id, new_objects_with_id): - # safe version (used by subclasses) - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - new_objects_with_id.setitem(newobj, id) - else: - self.id_free_list.append(id) - - def _update_object_id_FAST(self, obj, id, new_objects_with_id): - # unsafe version, assumes that the new_objects_with_id is large enough - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - new_objects_with_id.insertclean(newobj, id) - else: - self.id_free_list.append(id) - - def debug_check_object(self, obj): - """Check the invariants about 'obj' that should be true - between collections.""" - tid = self.header(obj).tid - if tid & GCFLAG_EXTERNAL: - ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded") - ll_assert(not (self.tospace <= obj < self.free), - "external flag but object inside the semispaces") - else: - ll_assert(not (tid & GCFLAG_FORWARDED), "bug: !external+forwarded") - ll_assert(self.tospace <= obj < self.free, - "!external flag but object outside the semispaces") - ll_assert(not (tid & GCFLAG_FINALIZATION_ORDERING), - "unexpected GCFLAG_FINALIZATION_ORDERING") - - def debug_check_can_copy(self, obj): - ll_assert(not (self.tospace <= obj < self.free), - "copy() on already-copied object") - STATISTICS_NUMBERS = 0 From fijal at codespeak.net Sat Sep 20 16:50:01 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 16:50:01 +0200 (CEST) Subject: [pypy-svn] r58271 - in pypy/branch/tuple-nonresizable-395/pypy: annotation/test rlib Message-ID: <20080920145001.D85FD16A170@codespeak.net> Author: fijal Date: Sat Sep 20 16:50:00 2008 New Revision: 58271 Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py Log: Make sure that make_sure_not_resized has no effect (besides warning) when list comprehension is off. The reason behind is is that it's too tedious to fix all the places. And eventually will force us to have list comprehension optimization on ootype, because nice pictures will be screwed by warnings. Modified: pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/annotation/test/test_annrpython.py Sat Sep 20 16:50:00 2008 @@ -3081,6 +3081,7 @@ return func() a = self.RPythonAnnotator() + a.translator.config.translation.list_comprehension_operations = True py.test.raises(TooLateForChange, a.build_types, fn, [int]) Modified: pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/rlib/debug.py Sat Sep 20 16:50:00 2008 @@ -92,7 +92,13 @@ def compute_result_annotation(self, s_arg): from pypy.annotation.model import SomeList assert isinstance(s_arg, SomeList) - s_arg.listdef.never_resize() + # the logic behind it is that we try not to propagate + # make_sure_not_resized, when list comprehension is not on + if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: + s_arg.listdef.never_resize() + else: + from pypy.annotation.annrpython import log + log.WARNING('make_sure_not_resized called, but has no effect since list_comprehension is off') return s_arg def specialize_call(self, hop): From fijal at codespeak.net Sat Sep 20 17:12:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 17:12:12 +0200 (CEST) Subject: [pypy-svn] r58272 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920151212.2FBD116A137@codespeak.net> Author: fijal Date: Sat Sep 20 17:12:11 2008 New Revision: 58272 Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: This two works now Modified: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE (original) +++ pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Sat Sep 20 17:12:11 2008 @@ -9,7 +9,3 @@ * use viewtuple() instead of viewiterable() in pycode.py:descr_code__new__() * add objspace.unpackiterable shortcuts - -* "translate.py --no-listcompr" doesn't annotate any more - -* "translate.py -b cli" doesn't annotate any more From arigo at codespeak.net Sat Sep 20 17:13:19 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 17:13:19 +0200 (CEST) Subject: [pypy-svn] r58273 - in pypy/branch/tuple-nonresizable-395/pypy: interpreter module/_sre module/posix Message-ID: <20080920151319.BB95716A149@codespeak.net> Author: arigo Date: Sat Sep 20 17:13:19 2008 New Revision: 58273 Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Log: Revert r58241, r58250, r58258, r58263: after some discussion, using listcompr for making fixsized lists is ok. Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pycode.py Sat Sep 20 17:13:19 2008 @@ -15,12 +15,8 @@ # helper -def unpack_str_tuple(space, w_str_tuple): - items_w = space.viewiterable(w_str_tuple) - result = [None] * len(items_w) - for i in range(len(items_w)): - result[i] = space.str_w(items_w[i]) - return result +def unpack_str_tuple(space,w_str_tuple): + return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] # code object contants, for co_flags below CO_OPTIMIZED = 0x0001 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_sre/interp_sre.py Sat Sep 20 17:13:19 2008 @@ -95,13 +95,10 @@ """Creates a tuple of index pairs representing matched groups, a format that's convenient for SRE_Match.""" space = self.space - regs = self.create_regs(group_count) - lst = [None] * len(regs) - for i in range(len(regs)): - value1, value2 = regs[i] - lst[i] = space.newtuple([space.wrap(value1), - space.wrap(value2)]) - return space.newtuple(lst) + return space.newtuple([ + space.newtuple([space.wrap(value1), + space.wrap(value2)]) + for value1, value2 in self.create_regs(group_count)]) w_create_regs.unwrap_spec = ['self', int] def fget_start(space, self): Modified: pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/posix/interp_posix.py Sat Sep 20 17:13:19 2008 @@ -555,11 +555,7 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(r[0]), - space.wrap(r[1]), - space.wrap(r[2]), - space.wrap(r[3]), - space.wrap(r[4])] + l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] return space.newtuple(l_w) uname.unwrap_spec = [ObjSpace] From arigo at codespeak.net Sat Sep 20 17:50:09 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 17:50:09 +0200 (CEST) Subject: [pypy-svn] r58274 - pypy/branch/tuple-nonresizable-395/pypy/rpython Message-ID: <20080920155009.325EA16A12A@codespeak.net> Author: arigo Date: Sat Sep 20 17:50:07 2008 New Revision: 58274 Modified: pypy/branch/tuple-nonresizable-395/pypy/rpython/callparse.py Log: Return a list from this method, not a tuple. Modified: pypy/branch/tuple-nonresizable-395/pypy/rpython/callparse.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/rpython/callparse.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/rpython/callparse.py Sat Sep 20 17:50:07 2008 @@ -176,7 +176,7 @@ if (expected_length is not None and expected_length != len(items)): raise ValueError - return items + return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" def is_w(self, one, other): From fijal at codespeak.net Sat Sep 20 17:52:38 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 17:52:38 +0200 (CEST) Subject: [pypy-svn] r58275 - in pypy/branch/tuple-nonresizable-395/pypy: interpreter interpreter/astcompiler interpreter/test module/__builtin__ module/__builtin__/test module/_codecs module/_rawffi module/_stackless/test objspace/flow objspace/std Message-ID: <20080920155238.3122A16A12A@codespeak.net> Author: fijal Date: Sat Sep 20 17:52:36 2008 New Revision: 58275 Removed: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/seqinterface.py Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/pyassem.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/gateway.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/interactive.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/nestedscope.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py pypy/branch/tuple-nonresizable-395/pypy/interpreter/test/test_objspace.py pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/abstractinst.py pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/compiling.py pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/test/test_abstractinst.py pypy/branch/tuple-nonresizable-395/pypy/module/_codecs/interp_codecs.py pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/structure.py pypy/branch/tuple-nonresizable-395/pypy/module/_stackless/test/test_frame_chain_reconstruction.py pypy/branch/tuple-nonresizable-395/pypy/objspace/flow/objspace.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeunicodeobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/stringobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/strsliceobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py pypy/branch/tuple-nonresizable-395/pypy/objspace/std/unicodeobject.py Log: * remove seqinterface.py as it's not needed any more * remove space.unpacktuple, use space.viewiterable * add objspace.unpackiterable shortcuts * remove segfault from unpacktuple (by removing it) Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/pyassem.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/pyassem.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/astcompiler/pyassem.py Sat Sep 20 17:52:36 2008 @@ -390,7 +390,7 @@ l_w = [None] * len(keys_w) for w_key in keys_w: index = space.int_w(space.getitem(self.w_consts, w_key)) - w_v = space.unpacktuple(w_key)[0] + w_v = space.viewiterable(w_key)[0] l_w[index] = w_v return l_w Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/baseobjspace.py Sat Sep 20 17:52:36 2008 @@ -665,17 +665,6 @@ return make_sure_not_resized(self.unpackiterable(w_iterable, expected_length)[:]) - def unpacktuple(self, w_tuple, expected_length=-1): - """Same as unpackiterable(), but only for tuples. - Only use for bootstrapping or performance reasons.""" - tuple_length = self.int_w(self.len(w_tuple)) - if expected_length != -1 and tuple_length != expected_length: - raise UnpackValueError("got a tuple of length %d instead of %d" % ( - tuple_length, expected_length)) - items = [ - self.getitem(w_tuple, self.wrap(i)) for i in range(tuple_length)] - return items - def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/gateway.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/gateway.py Sat Sep 20 17:52:36 2008 @@ -207,7 +207,7 @@ % (self.scopenext(), self.scopenext())) def visit_args_w(self, el): - self.run_args.append("space.unpacktuple(%s)" % self.scopenext()) + self.run_args.append("space.viewiterable(%s)" % self.scopenext()) def visit_w_args(self, el): self.run_args.append(self.scopenext()) @@ -406,7 +406,7 @@ # baseobjspace.W_Root is for wrapped arguments to keep wrapped # baseobjspace.Wrappable subclasses imply interp_w and a typecheck # argument.Arguments is for a final rest arguments Arguments object - # 'args_w' for unpacktuple applied to rest arguments + # 'args_w' for viewiterable applied to rest arguments # 'w_args' for rest arguments passed as wrapped tuple # str,int,float: unwrap argument as such type # (function, cls) use function to check/unwrap argument of type cls Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/interactive.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/interactive.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/interactive.py Sat Sep 20 17:52:36 2008 @@ -73,7 +73,7 @@ words = self.get_words(w_clz) try: w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.unpacktuple(w_bases) + bases_w = s.viewiterable(w_bases) except error.OperationError: return words Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/nestedscope.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/nestedscope.py Sat Sep 20 17:52:36 2008 @@ -202,7 +202,7 @@ if codeobj.magic >= 0xa0df281: # CPython 2.5 AST branch merge w_freevarstuple = f.popvalue() freevars = [f.space.interp_w(Cell, cell) - for cell in f.space.unpacktuple(w_freevarstuple)] + for cell in f.space.viewiterable(w_freevarstuple)] else: nfreevars = len(codeobj.co_freevars) freevars = [f.space.interp_w(Cell, f.popvalue()) Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/pyopcode.py Sat Sep 20 17:52:36 2008 @@ -551,7 +551,7 @@ w_compile_flags, f.space.wrap(f.get_builtin()), f.space.gettypeobject(PyCode.typedef)) - w_prog, w_globals, w_locals = f.space.unpacktuple(w_resulttuple, 3) + w_prog, w_globals, w_locals = f.space.viewiterable(w_resulttuple, 3) plain = f.w_locals is not None and f.space.is_w(w_locals, f.w_locals) if plain: Modified: pypy/branch/tuple-nonresizable-395/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/interpreter/test/test_objspace.py Sat Sep 20 17:52:36 2008 @@ -54,14 +54,14 @@ raises(ValueError, self.space.unpackiterable, w_l, 3) raises(ValueError, self.space.unpackiterable, w_l, 5) - def test_unpacktuple(self): + def test_viewiterable(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.unpacktuple(w_l) == l - assert self.space.unpacktuple(w_l, 4) == l - raises(ValueError, self.space.unpacktuple, w_l, 3) - raises(ValueError, self.space.unpacktuple, w_l, 5) + assert self.space.viewiterable(w_l) == l + assert self.space.viewiterable(w_l, 4) == l + raises(ValueError, self.space.viewiterable, w_l, 3) + raises(ValueError, self.space.viewiterable, w_l, 5) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, @@ -203,7 +203,7 @@ w_res = space.call_obj_args(w_f, w_9, Arguments(space, [w_1])) - w_x, w_y = space.unpacktuple(w_res, 2) + w_x, w_y = space.viewiterable(w_res, 2) assert w_x is w_9 assert w_y is w_1 Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/abstractinst.py Sat Sep 20 17:52:36 2008 @@ -79,7 +79,7 @@ # -- case (anything, tuple) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.unpacktuple(w_klass_or_tuple): + for w_klass in space.viewiterable(w_klass_or_tuple): if abstract_isinstance_w(space, w_obj, w_klass): return True return False @@ -104,7 +104,7 @@ return True w_bases = _get_bases(space, w_derived) if w_bases is not None: - for w_base in space.unpacktuple(w_bases): + for w_base in space.viewiterable(w_bases): if _issubclass_recurse(space, w_base, w_top): return True return False @@ -134,7 +134,7 @@ # -- case (class-like-object, tuple-of-classes) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.unpacktuple(w_klass_or_tuple): + for w_klass in space.viewiterable(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass): return True return False Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/compiling.py Sat Sep 20 17:52:36 2008 @@ -67,9 +67,9 @@ "", "eval") except OperationError, e: if e.match(space, space.w_SyntaxError): - e_value_w = space.unpacktuple(e.w_value) + e_value_w = space.viewiterable(e.w_value) if len(e_value_w) == 2: - e_loc_w = space.unpacktuple(e_value_w[1]) + e_loc_w = space.viewiterable(e_value_w[1]) e.w_value = space.newtuple([e_value_w[0], space.newtuple([space.w_None]+ e_loc_w[1:])]) Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/interp_classobj.py Sat Sep 20 17:52:36 2008 @@ -280,7 +280,7 @@ if not e.match(space, space.w_TypeError): raise return [None, None] - return space.unpacktuple(w_tup, 2) + return space.viewiterable(w_tup, 2) def descr_instance_new(space, w_type, w_class, w_dict=None): # w_type is not used at all Modified: pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/test/test_abstractinst.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/test/test_abstractinst.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/__builtin__/test/test_abstractinst.py Sat Sep 20 17:52:36 2008 @@ -5,7 +5,7 @@ def test_abstract_isclass(self): space = self.space - w_B1, w_B2, w_B3, w_X, w_Y = space.unpacktuple(space.appexec([], """(): + w_B1, w_B2, w_B3, w_X, w_Y = space.viewiterable(space.appexec([], """(): class X(object): pass class Y: pass B1, B2, B3 = X(), X(), X() @@ -22,7 +22,7 @@ def test_abstract_getclass(self): space = self.space - w_x, w_y, w_A, w_MyInst = space.unpacktuple(space.appexec([], """(): + w_x, w_y, w_A, w_MyInst = space.viewiterable(space.appexec([], """(): class MyInst(object): def __init__(self, myclass): self.myclass = myclass Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_codecs/interp_codecs.py Sat Sep 20 17:52:36 2008 @@ -35,7 +35,7 @@ space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) - w_replace, w_newpos = space.unpacktuple(w_res, 2) + w_replace, w_newpos = space.viewiterable(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/interp_rawffi.py Sat Sep 20 17:52:36 2008 @@ -98,7 +98,7 @@ resshape = cache.get_array_type(letter2tp(space, letter)) else: letter = 'V' - w_shapetype, w_length = space.unpacktuple(w_shape, expected_length=2) + w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) from pypy.module._rawffi.structure import W_Structure resshape = space.interp_w(W_Structure, w_shapetype) ffi_type = resshape.get_ffi_type() @@ -109,7 +109,7 @@ letter = space.str_w(w_shape) return letter2tp(space, letter) else: - w_shapetype, w_length = space.unpacktuple(w_shape, expected_length=2) + w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) resshape = space.interp_w(W_DataShape, w_shapetype) length = space.int_w(w_length) size, alignment = resshape._size_alignment() Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/structure.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_rawffi/structure.py Sat Sep 20 17:52:36 2008 @@ -119,7 +119,7 @@ def descr_new_structure(space, w_type, w_shapeinfo): if space.is_true(space.isinstance(w_shapeinfo, space.w_tuple)): - w_size, w_alignment = space.unpacktuple(w_shapeinfo, expected_length=2) + w_size, w_alignment = space.viewiterable(w_shapeinfo, expected_length=2) S = W_Structure(space, None, space.int_w(w_size), space.int_w(w_alignment)) else: Modified: pypy/branch/tuple-nonresizable-395/pypy/module/_stackless/test/test_frame_chain_reconstruction.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/module/_stackless/test/test_frame_chain_reconstruction.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/module/_stackless/test/test_frame_chain_reconstruction.py Sat Sep 20 17:52:36 2008 @@ -92,7 +92,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -151,7 +151,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -223,7 +223,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/flow/objspace.py Sat Sep 20 17:52:36 2008 @@ -202,7 +202,7 @@ # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) # checking a tuple of classes - for w_klass in self.unpacktuple(w_check_class): + for w_klass in self.viewiterable(w_check_class): if ObjSpace.exception_match(self, w_exc_type, w_klass): return True return False @@ -264,12 +264,7 @@ checkgraph(graph) return graph - def unpacktuple(self, w_tuple, expected_length=None): -## # special case to accept either Constant tuples -## # or real tuples of Variables/Constants -## if isinstance(w_tuple, tuple): -## result = w_tuple -## else: + def viewiterable(self, w_tuple, expected_length=None): unwrapped = self.unwrap(w_tuple) result = tuple([Constant(x) for x in unwrapped]) if expected_length is not None and len(result) != expected_length: @@ -286,26 +281,6 @@ if isinstance(w_iterable, Variable) and expected_length is None: raise UnwrapException, ("cannot unpack a Variable iterable" "without knowing its length") -## # XXX TEMPORARY HACK XXX TEMPORARY HACK XXX TEMPORARY HACK -## print ("*** cannot unpack a Variable iterable " -## "without knowing its length,") -## print " assuming a list or tuple with up to 7 items" -## items = [] -## w_len = self.len(w_iterable) -## i = 0 -## while True: -## w_i = self.wrap(i) -## w_cond = self.eq(w_len, w_i) -## if self.is_true(w_cond): -## break # done -## if i == 7: -## # too many values -## raise OperationError(self.w_AssertionError, self.w_None) -## w_item = self.do_operation('getitem', w_iterable, w_i) -## items.append(w_item) -## i += 1 -## return items -## # XXX TEMPORARY HACK XXX TEMPORARY HACK XXX TEMPORARY HACK elif expected_length is not None: w_len = self.len(w_iterable) w_correct = self.eq(w_len, self.wrap(expected_length)) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/listobject.py Sat Sep 20 17:52:36 2008 @@ -2,14 +2,12 @@ from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.listtype import get_list_index from pypy.objspace.std.sliceobject import W_SliceObject -from pypy.objspace.std.seqinterface import W_SeqObject from pypy.objspace.std import slicetype from pypy.interpreter import gateway, baseobjspace from pypy.rlib.listsort import TimSort - -class W_ListObject(W_SeqObject): +class W_ListObject(W_Object): from pypy.objspace.std.listtype import list_typedef as typedef def __init__(w_self, wrappeditems): @@ -26,13 +24,6 @@ def append(w_list, w_item): w_list.wrappeditems.append(w_item) - def getlength(self): - return len(self.wrappeditems) - - def getitemfast(self, i): - return self.wrappeditems[i] - - registerimplementation(W_ListObject) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/marshal_impl.py Sat Sep 20 17:52:36 2008 @@ -343,7 +343,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) for w_tuple in w_dict.implementation.items(): - w_key, w_value = space.unpacktuple(w_tuple, 2) + w_key, w_value = space.viewiterable(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/objspace.py Sat Sep 20 17:52:36 2008 @@ -379,7 +379,7 @@ space = self # too early for unpackiterable as well :-( name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.unpacktuple(space.getitem(w_args, space.wrap(1))) + bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) dic = space.unwrap(space.getitem(w_args, space.wrap(2))) dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) bases = list(bases) @@ -629,12 +629,32 @@ return instance allocate_instance._annspecialcase_ = "specialize:arg(1)" - def unpacktuple(self, w_tuple, expected_length=-1): - assert isinstance(w_tuple, self.TupleObjectCls) - t = w_tuple.getitems() - if expected_length != -1 and expected_length != len(t): - raise ValueError, "got a tuple of length %d instead of %d" % ( - len(t), expected_length) + # two following functions are almost identical, but in fact they + # have different return type. First one is a resizable list, second + # one is not + + def unpackiterable(self, w_obj, expected_length=-1): + if isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems[:] + elif isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.unpackiterable(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) + return t + + def viewiterable(self, w_obj, expected_length=-1): + """ Fast paths + """ + if isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems + elif isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.viewiterable(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) return t def sliceindices(self, w_slice, w_length): Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeobject.py Sat Sep 20 17:52:36 2008 @@ -533,7 +533,7 @@ def str_endswith__Rope_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -552,7 +552,7 @@ def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/ropeunicodeobject.py Sat Sep 20 17:52:36 2008 @@ -485,7 +485,7 @@ def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = ropeunicode_w(space, w_prefix) if rope.startswith(unistr, prefix, start, end): return space.w_True @@ -494,7 +494,7 @@ def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = ropeunicode_w(space, w_suffix) if rope.endswith(unistr, suffix, start, end): return space.w_True Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/stringobject.py Sat Sep 20 17:52:36 2008 @@ -593,7 +593,7 @@ def str_endswith__String_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -611,7 +611,7 @@ def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/strsliceobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/strsliceobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/strsliceobject.py Sat Sep 20 17:52:36 2008 @@ -141,7 +141,7 @@ def str_endswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -155,7 +155,7 @@ def str_startswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/tupleobject.py Sat Sep 20 17:52:36 2008 @@ -1,12 +1,11 @@ from pypy.objspace.std.objspace import * -from pypy.objspace.std.seqinterface import W_SeqObject from pypy.objspace.std.inttype import wrapint from pypy.rlib.rarithmetic import intmask from pypy.objspace.std.sliceobject import W_SliceObject from pypy.interpreter import gateway from pypy.rlib.debug import make_sure_not_resized -class W_TupleObject(W_SeqObject): +class W_TupleObject(W_Object): from pypy.objspace.std.tupletype import tuple_typedef as typedef def __init__(w_self, wrappeditems): @@ -22,15 +21,6 @@ items = [space.unwrap(w_item) for w_item in w_tuple.wrappeditems] # XXX generic mixed types unwrap return tuple(items) - def getitems(self): - return self.wrappeditems - - def getlength(self): - return len(self.wrappeditems) - - def getitemfast(self, i): - return self.wrappeditems[i] - registerimplementation(W_TupleObject) Modified: pypy/branch/tuple-nonresizable-395/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/tuple-nonresizable-395/pypy/objspace/std/unicodeobject.py Sat Sep 20 17:52:36 2008 @@ -474,7 +474,7 @@ def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = space.unicode_w(w_prefix) if _check_startswith_substring(unistr, prefix, start, end): return space.w_True @@ -483,7 +483,7 @@ def unicode_endswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = space.unicode_w(w_suffix) if _check_endswith_substring(unistr, suffix, start, end): return space.w_True From fijal at codespeak.net Sat Sep 20 17:53:18 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 20 Sep 2008 17:53:18 +0200 (CEST) Subject: [pypy-svn] r58276 - pypy/branch/tuple-nonresizable-395/pypy Message-ID: <20080920155318.921ED16A12A@codespeak.net> Author: fijal Date: Sat Sep 20 17:53:17 2008 New Revision: 58276 Removed: pypy/branch/tuple-nonresizable-395/pypy/TODO.BEFOREMERGE Log: seems to be ready to merge From arigo at codespeak.net Sat Sep 20 18:16:11 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 18:16:11 +0200 (CEST) Subject: [pypy-svn] r58277 - pypy/branch/tuple-nonresizable-395/lib-python Message-ID: <20080920161611.50A7E169EFE@codespeak.net> Author: arigo Date: Sat Sep 20 18:16:09 2008 New Revision: 58277 Modified: pypy/branch/tuple-nonresizable-395/lib-python/conftest.py Log: Calls to unpacktuple() forgotten. Modified: pypy/branch/tuple-nonresizable-395/lib-python/conftest.py ============================================================================== --- pypy/branch/tuple-nonresizable-395/lib-python/conftest.py (original) +++ pypy/branch/tuple-nonresizable-395/lib-python/conftest.py Sat Sep 20 18:16:09 2008 @@ -166,12 +166,12 @@ set_argv = app.interphook('set_argv') def start_intercept(space): - w_suites, w_doctestmodules = space.unpacktuple(intercept_test_support(space)) + w_suites, w_doctestmodules = space.viewiterable(intercept_test_support(space)) return w_suites, w_doctestmodules def collect_intercept(space, w_suites, w_doctestmodules): w_result = callex(space, collect_intercepted, space, w_suites, w_doctestmodules) - w_namemethods, w_doctestlist = space.unpacktuple(w_result) + w_namemethods, w_doctestlist = space.viewiterable(w_result) return w_namemethods, w_doctestlist class SimpleRunItem(py.test.collect.Item): @@ -240,14 +240,14 @@ # setup {name -> wrapped testcase method} for w_item in space.unpackiterable(w_namemethods): - w_name, w_method = space.unpacktuple(w_item) + w_name, w_method = space.viewiterable(w_item) name = space.str_w(w_name) testitem = AppTestCaseMethod(name, parent=self, w_method=w_method) self.name2item[name] = testitem # setup {name -> wrapped doctest module} for w_item in space.unpackiterable(w_doctestlist): - w_name, w_module = space.unpacktuple(w_item) + w_name, w_module = space.viewiterable(w_item) name = space.str_w(w_name) testitem = AppDocTestModule(name, parent=self, w_module=w_module) self.name2item[name] = testitem From arigo at codespeak.net Sat Sep 20 18:54:20 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 18:54:20 +0200 (CEST) Subject: [pypy-svn] r58278 - in pypy/dist/pypy/interpreter: . test Message-ID: <20080920165420.C909C16A12A@codespeak.net> Author: arigo Date: Sat Sep 20 18:54:19 2008 New Revision: 58278 Modified: pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/interpreter/test/test_raise.py Log: (pedronis around, arigo) Tests and fix for when exceptions should show up in sys.exc_info() or not. Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sat Sep 20 18:54:19 2008 @@ -95,6 +95,9 @@ operr = self.last_exception next_instr = self.handle_operation_error(ec, operr, attach_tb=False) + except RaiseWithExplicitTraceback, e: + next_instr = self.handle_operation_error(ec, e.operr, + attach_tb=False) except KeyboardInterrupt: next_instr = self.handle_asynchronous_error(ec, self.space.w_KeyboardInterrupt) @@ -122,7 +125,6 @@ return self.handle_operation_error(ec, operr) def handle_operation_error(self, ec, operr, attach_tb=True): - self.last_exception = operr if attach_tb: pytraceback.record_application_traceback( self.space, operr, self, self.last_instr) @@ -533,9 +535,8 @@ raise OperationError(space.w_TypeError, space.wrap("raise: arg 3 must be a traceback or None")) operror.application_traceback = tb - # re-raise, no new traceback obj will be attached - f.last_exception = operror - raise Reraise + # special 3-arguments raise, no new traceback obj will be attached + raise RaiseWithExplicitTraceback(operror) def LOAD_LOCALS(f, *ignored): f.pushvalue(f.w_locals) @@ -979,17 +980,20 @@ ### ____________________________________________________________ ### -class Reraise(Exception): - """Signal an application-level OperationError that should not grow - a new traceback entry nor trigger the trace hook.""" - class ExitFrame(Exception): pass class Return(ExitFrame): - """Obscure.""" + """Raised when exiting a frame via a 'return' statement.""" class Yield(ExitFrame): - """Obscure.""" + """Raised when exiting a frame via a 'yield' statement.""" + +class Reraise(Exception): + """Raised at interp-level by a bare 'raise' statement.""" +class RaiseWithExplicitTraceback(Exception): + """Raised at interp-level by a 3-arguments 'raise' statement.""" + def __init__(self, operr): + self.operr = operr class BytecodeCorruption(Exception): """Detected bytecode corruption. Never caught; it's an error.""" @@ -1157,6 +1161,7 @@ frame.pushvalue(frame.space.wrap(unroller)) frame.pushvalue(operationerr.w_value) frame.pushvalue(operationerr.w_type) + frame.last_exception = operationerr return self.handlerposition # jump to the handler Modified: pypy/dist/pypy/interpreter/test/test_raise.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_raise.py (original) +++ pypy/dist/pypy/interpreter/test/test_raise.py Sat Sep 20 18:54:19 2008 @@ -90,6 +90,75 @@ assert exc_val ==exc_val2 assert exc_tb ==exc_tb2 + def test_reraise(self): + # some collection of funny code + import sys + raises(ValueError, """ + import sys + try: + raise ValueError + except: + try: + raise IndexError + finally: + assert sys.exc_info()[0] is ValueError + raise + """) + raises(ValueError, """ + def foo(): + import sys + assert sys.exc_info()[0] is ValueError + raise + try: + raise ValueError + except: + try: + raise IndexError + finally: + foo() + """) + raises(IndexError, """ + def spam(): + import sys + try: + raise KeyError + except KeyError: + pass + assert sys._getframe().f_exc_type is ValueError + try: + raise ValueError + except: + try: + raise IndexError + finally: + spam() + """) + + try: + raise ValueError + except: + try: + raise KeyError + except: + ok = sys.exc_info()[0] is KeyError + assert ok + + raises(IndexError, """ + import sys + try: + raise ValueError + except: + some_traceback = sys.exc_info()[2] + try: + raise KeyError + except: + try: + raise IndexError, IndexError(), some_traceback + finally: + assert sys.exc_info()[0] is KeyError + assert sys.exc_info()[2] is not some_traceback + """) + def test_tuple_type(self): def f(): raise ((StopIteration, 123), 456, 789) From hpk at codespeak.net Sat Sep 20 18:57:38 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 20 Sep 2008 18:57:38 +0200 (CEST) Subject: [pypy-svn] r58279 - in pypy/build/benchmem: . benchmark testing Message-ID: <20080920165738.B7FB516A167@codespeak.net> Author: hpk Date: Sat Sep 20 18:57:37 2008 New Revision: 58279 Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py pypy/build/benchmem/benchtool.py pypy/build/benchmem/smaps.py pypy/build/benchmem/testing/test_benchtool.py Log: always append benchmark results to a file, use denser format Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py ============================================================================== --- pypy/build/benchmem/benchmark/create_recursive_tuples.py (original) +++ pypy/build/benchmem/benchmark/create_recursive_tuples.py Sat Sep 20 18:57:37 2008 @@ -5,4 +5,11 @@ checkpoint() for j in range(iter2): x = (x,) + x = (x,) + x = (x,) + x = (x,) + x = (x,) + x = (x,) + x = (x,) + x = (x,) Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Sat Sep 20 18:57:37 2008 @@ -1,5 +1,6 @@ +#!/usr/bin/env python """ - benchtool.py [--benchlog=logfile] [--executable=path1] [benchname1.py] [benchname2.py] + benchtool.py [options] [benchname1.py] [benchname2.py] record memory usage for given benchmarks (or all if none specified). @@ -13,8 +14,11 @@ class BenchRunner(object): + SEPBENCH = "=" * 80 + def __init__(self, executable, benchlog): - self.benchlog = py.path.local(benchlog) + self.benchlogpath = py.path.local(benchlog) + self.logstream = self.benchlogpath.open("a") self.executable = executable self.tmpdir = py.path.local.make_numbered_dir(prefix="bench") @@ -31,31 +35,33 @@ arglist = ",".join(map(str, args)) source = py.code.Source(path.read(), """ + + def write(c): + sys.stdout.write(c) + sys.stdout.flush() + def checkpoint(): - sys.stdout.write("c") + write("c") sys.stdin.read(1) if __name__ == "__main__": import os, sys pid = os.getpid() - sys.stdout.write(str(pid)) - sys.stdout.write("\\n") + write(str(pid) + "\\n") checkpoint() %s(checkpoint, %s) checkpoint() - sys.stdout.write("F") - sys.stdout.close() + write("F") + sys.stdin.close() """ %(name, arglist)) p = self.tmpdir.join(path.basename) p.write(source) return p - def writeexecinfo(self, benchname): - f = self.benchlog.open("a") - print >>f, "=" * 80 - print >>f, "#executable=%r" %(str(self.executable ),) - print >>f, "#benchname=%r" %(benchname,) - f.close() - # xxx add more + def writeexecinfo(self, benchname, args): + print >>self.logstream, self.SEPBENCH + print >>self.logstream, "#executable=%r" %(str(self.executable ),) + print >>self.logstream, "#benchname=%r" %(benchname,) + print >>self.logstream, "#benchargs=%r" %(args,) def run_checkpointed_bench(self, filename, args): benchpyfile = self._preparebench(filename, args) @@ -65,39 +71,94 @@ stdout, stdin = os.popen2(cmd) pid = int(stdin.readline()) - self.writeexecinfo(benchpyfile.basename) - rec = smaps.SmapsRecorder(pid, self.benchlog) - stdout.write(".") - stdout.flush() + self.writeexecinfo(benchpyfile.basename, args) + rec = smaps.SmapsRecorder(pid, self.logstream) + self.interact_with_child_checkpoints(rec, stdout, stdin) + + def interact_with_child_checkpoints(self, rec, stdout, stdin): + def write(c): + stdout.write(c) + stdout.flush() while not stdin.closed: c = stdin.read(1) if c == "F": + stdin.close() + stdout.close() break rec.snapshot() stdout.write(".") stdout.flush() #sys.stdout.write(".") #sys.stdout.flush() - rec.finish() - self.log("finished", cmd) -class BenchResult: - pass - -class BenchlogReader(object): - def __init__(self, logpath): - self.logpath = logpath - - def getname2results(self): - name2results = {} - for result in self.readresults(): - l = name2results.setdefault(result.benchname, []) - l.append(result) - return name2results +# +# ================ reading a benchmark log file ======================= +# - def readresults(self): - xxx +class Benchresults(object): + def __init__(self): + self.name2results = {} + + def parse_logfile(self, logpath): + f = logpath.open() + for result in BenchmarkResult.parse(f): + #print "parsed", result + l = self.name2results.setdefault(result.benchname, []) + l.append(result) + f.close() +class BenchmarkResult: + def __init__(self, snapshots, executable, benchname, benchargs): + assert snapshots + self.snapshots = snapshots + self.executable = executable + self.benchname = benchname + self.benchargs = benchargs + + @classmethod + def parse(cls, f): + while not f.closed: + line = f.readline() + if not line.strip(): + break + if not line.startswith("#executable"): + print "ignoring", line + continue + executable = eval(line.split("=", 1)[1]) + benchname = eval(f.readline().split("=", 1)[1]) + benchargs = eval(f.readline().split("=", 1)[1]) + + snapshots = [] + line = f.readline() + while 1: + mappings = [] + while line != smaps.SmapsRecorder.SEPSNAPSHOT: + mappings.append(smaps.Mapping(line)) + line = f.readline() + #print "reading", line.strip() + snapshots.append(Snapshot(mappings)) + line = f.readline() + if not line or line == BenchRunner.SEPBENCH: + break + yield BenchmarkResult(snapshots, executable=executable, + benchname=benchname, benchargs=benchargs) + + def _verify_integrity(self): + for snap in self.snapshots: + for mapping in snap.mappings: + clean = mapping.shared_clean + mapping.private_clean + dirty = mapping.private_dirty + mapping.shared_dirty + assert mapping.rss == dirty + clean + +class Snapshot(object): + def __init__(self, mappings): + assert mappings + self.mappings = mappings + for name in smaps.Mapping._attrnames: + setattr(self, name, sum([getattr(x, name) for x in mappings])) + + def memusage(self): + return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) # # ============================================================================== @@ -146,3 +207,4 @@ runner = BenchRunner(executable, benchlog) for name in names: runner.run_checkpointed_bench(name, (100, 100)) + print "bench results append to -->>>", benchlog Modified: pypy/build/benchmem/smaps.py ============================================================================== --- pypy/build/benchmem/smaps.py (original) +++ pypy/build/benchmem/smaps.py Sat Sep 20 18:57:37 2008 @@ -2,99 +2,55 @@ import py class SmapsRecorder: - SEPLINE = "-"*80 + "\n" + SEPSNAPSHOT = "-"*80 + "\n" - def __init__(self, pid, logpath): - self.logpath = py.path.local(logpath) - self._file = self.logpath.open("a") + def __init__(self, pid, stream): + self.stream = stream self.pid = pid self.smapspath = py.path.local("/proc/%d/smaps" %(pid,)) assert self.smapspath.check() def snapshot(self): - s = self.smapspath.read() - self._file.write(s) - self._file.write(self.SEPLINE) - self._file.flush() - - def finish(self): - self._file.close() - -class SmapsReader: - def __init__(self, snapshots): - self.snapshots = snapshots - - @classmethod - def parse(cls, path): - """ parse a file previously recorded through SmapsRecorder. """ - s = path.read() - parts = filter(None, map(str.strip, s.split(SmapsRecorder.SEPLINE))) - snapshots = [SmapsSnapshot.parse(s) for s in parts] - return cls(snapshots) - - def _verify_integrity(self): - for snap in self.snapshots: - for mapping in snap.mappings: - clean = mapping.shared_clean + mapping.private_clean - dirty = mapping.private_dirty + mapping.shared_dirty - assert mapping.rss == dirty + clean - - -class SmapsMapping: - def __init__(self, headvars, bodyvars): - self.__dict__.update(headvars) - self.__dict__.update(bodyvars) + f = self.smapspath.open() + headline = f.readline() + while not f.closed: + attrs = [] + while not f.closed: + sizeline = f.readline() + if not sizeline: # EOF + break + parts = sizeline.split() + if len(parts) != 3: + headline = sizeline + break + value = int(parts[1]) + attrs.append(value) + attrs = attrs[:6] # not interested in "Referenced" + sizes = ",".join(map(str, attrs)) + assert sizes + self.stream.write("%-24s %s" %(sizes, headline)) + if not sizeline: + break + f.close() + self.stream.write(self.SEPSNAPSHOT) + self.stream.flush() + + +# +# objects gained from reading of the logfile +# +class Mapping: + _attrnames = ("size rss shared_clean shared_dirty " + "private_clean private_dirty".split()) + # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' + _headattrnames = "virtualaddress mode page dev inode filename".split() + + def __init__(self, line): + parts = line.split() + attrs = parts.pop(0) + for name, value in zip(self._attrnames, map(int, attrs.split(","))): + setattr(self, name, value) + for name, value in zip(self._headattrnames, parts): + setattr(self, name, value) self.inode = int(self.inode) -class SmapsSnapshot(object): - # example '402c1000-402c2000 rw-p 00011000 1f:04 1110 /lib/libnsl-2.5.so' - headrex = re.compile(r""" - (?P\w+-\w+)\s+ - (?P[rwxps-]+)\s+ - (?P\w+)\s+ - (?P\w\w:\w\w)\s+ - (?P\w+)\s+ - (?P\S*)""", - re.VERBOSE) - # example 'Size: 4648 kB' - linerex = re.compile(r"(\S+):\s+(\d+)\skB") - - def __init__(self, mappings): - self.mappings = mappings - self.UNIT = "kB" - for name in ("size rss shared_clean shared_dirty " - "private_clean private_dirty".split()): - setattr(self, name, sum([getattr(x, name) for x in mappings])) - - def memusage(self): - return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) - - @classmethod - def parse(cls, string): - lines = string.split("\n") - mappings = [] - while lines: - line = lines.pop(0) - if not line.strip(): - continue - m = cls.headrex.match(line) - if m is None: - print "ignoring", line - continue - headvars = m.groupdict() - bodyvars = {} - while lines: - line = lines.pop(0) - m = cls.linerex.match(line) - if m is None: - lines.insert(0, line) - mappings.append(SmapsMapping(headvars, bodyvars)) - break - name, num = m.groups() - name = name.lower() - num = int(num) - assert name not in bodyvars - bodyvars[name] = num - else: - mappings.append(SmapsMapping(headvars, bodyvars)) - return cls(mappings) Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Sat Sep 20 18:57:37 2008 @@ -12,155 +12,85 @@ tmpdir = py.test.ensuretemp("smapsrecorder") logpath = tmpdir.join("logfile") pid = os.getpid() - rec = smaps.SmapsRecorder(pid=pid, logpath=logpath) + f = logpath.open("w") + rec = smaps.SmapsRecorder(pid=pid, stream=f) s = logpath.read() - assert s.count(smaps.SmapsRecorder.SEPLINE) == 0 + assert s.count(smaps.SmapsRecorder.SEPSNAPSHOT) == 0 rec.snapshot() rec.snapshot() del rec s = logpath.read() - assert s.count(smaps.SmapsRecorder.SEPLINE) == 2 + assert s.count(smaps.SmapsRecorder.SEPSNAPSHOT) == 2 def test_benchrunner_functional(): tmpdir = py.test.ensuretemp("benchrunner") log=tmpdir.join("log") - runner = benchtool.BenchRunner(executable="python2.5", benchlog=log) def checker(path, *args): if log.check(): log.remove() + runner = benchtool.BenchRunner(executable="python2.5", benchlog=log) runner.run_checkpointed_bench(path, args) assert log.check() - reader = smaps.SmapsReader.parse(log) + benchresult = benchtool.Benchresults() + benchresult.parse_logfile(log) #assert reader.executable #assert reader.executable - assert len(reader.snapshots) == 10 + 2 - reader._verify_integrity() + assert len(benchresult.name2results) == 1 + results = benchresult.name2results.values()[0] + assert len(results) == 1 + assert len(results[0].snapshots) == 10 + 2 for path in benchtool.benchmarkdir.listdir("*.py"): if path.basename[0] != "_": yield checker, path, 10, 10 -class TestSmapsSnapshot: - def setup_class(cls): - cls.s = py.std.textwrap.dedent("""\ - 08048000-0813f000 r-xp 00000000 fd:00 75457 sometext - Size: 988 kB - Rss: 796 kB - Shared_Clean: 0 kB - Shared_Dirty: 0 kB - Private_Clean: 796 kB - Private_Dirty: 0 kB - 402c2000-402c4000 rw-p 402c2000 00:00 0 - Size: 16 kB - Rss: 8 kB - Shared_Clean: 0 kB - Shared_Dirty: 0 kB - Private_Clean: 0 kB - Private_Dirty: 8 kB""") - - def test_parse_mappings(self): - smapsreader = smaps.SmapsSnapshot.parse(self.s) - mappings = smapsreader.mappings - assert len(mappings) == 2 - mapping = mappings[0] - assert mapping.virtualaddress == "08048000-0813f000" - assert mapping.mode == "r-xp" - assert mapping.dev == "fd:00" - assert mapping.inode == 75457 - assert mapping.filename == "sometext" - assert mapping.size == 988 - assert mapping.rss == 796 - assert mapping.shared_clean == 0 - assert mapping.shared_dirty == 0 - assert mapping.private_clean == 796 - assert mapping.private_dirty == 0 - #assert mapping.referenced is None - - mapping = mappings[1] - assert mapping.virtualaddress == "402c2000-402c4000" - assert mapping.mode == "rw-p" - assert mapping.dev == "00:00" - assert mapping.inode == 0 - assert not mapping.filename - assert mapping.size == 16 - assert mapping.rss == 8 - assert mapping.shared_clean == 0 - assert mapping.shared_dirty == 0 - assert mapping.private_clean == 0 - assert mapping.private_dirty == 8 - #assert mapping.referenced == 12 - - def test_summing(self): - snap = smaps.SmapsSnapshot.parse(self.s) - for name in ('size rss shared_clean shared_dirty ' - 'private_clean private_dirty').split(): - sumval = getattr(snap, name) - val1 = getattr(snap.mappings[0], name) - val2 = getattr(snap.mappings[1], name) - assert sumval == val1 + val2 - -example_data2 = ''' -08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 -Size: 988 kB -Rss: 796 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 796 kB -Private_Dirty: 0 kB -0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 -Size: 148 kB -Rss: 120 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 12 kB -Private_Dirty: 108 kB -08164000-0825c000 rw-p 08164000 00:00 0 [heap] -Size: 992 kB -Rss: 924 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 924 kB -b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 -Size: 240 kB -Rss: 60 kB -Shared_Clean: 60 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 0 kB -''' - -example_data = ''' -08048000-0813f000 r-xp 00000000 fd:00 75457 /usr/bin/python2.5 -Size: 988 kB -Rss: 796 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 796 kB -Private_Dirty: 0 kB -Referenced: 796 kB -0813f000-08164000 rw-p 000f6000 fd:00 75457 /usr/bin/python2.5 -Size: 148 kB -Rss: 120 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 12 kB -Private_Dirty: 108 kB -Referenced: 120 kB -08164000-0825c000 rw-p 08164000 00:00 0 [heap] -Size: 992 kB -Rss: 924 kB -Shared_Clean: 0 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 924 kB -Referenced: 924 kB -b7baf000-b7beb000 r-xp 00000000 08:01 218 /lib/libncurses.so.5.6 -Size: 240 kB -Rss: 60 kB -Shared_Clean: 60 kB -Shared_Dirty: 0 kB -Private_Clean: 0 kB -Private_Dirty: 0 kB -Referenced: 60 kB -''' +def test_log_mapping(): + s = py.std.textwrap.dedent("""\ + 402c2000-402c4000 rw-p 402c2000 00:00 0 + Size: 16 kB + Rss: 8 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 8 kB""") + basedir = py.test.ensuretemp("log_mapping") + p = basedir.join("smaps.example") + p.write(s) + + io = py.std.StringIO.StringIO() + rec = smaps.SmapsRecorder(os.getpid(), io) + rec.smapspath = p + rec.snapshot() + result = io.getvalue() + lines = result.split("\n") + l0 = " ".join(lines[0].split()) + assert l0 == "16,8,0,0,0,8 402c2000-402c4000 rw-p 402c2000 00:00 0" + assert lines[1] == smaps.SmapsRecorder.SEPSNAPSHOT.strip() + +def test_parse_mapping(): + line = ("988,796,0,0,796,0 08048000-0813f000 " + "r-xp 00000000 fd:00 75457 sometext") + mapping = smaps.Mapping(line) + assert mapping.virtualaddress == "08048000-0813f000" + assert mapping.mode == "r-xp" + assert mapping.dev == "fd:00" + assert mapping.inode == 75457 + assert mapping.filename == "sometext" + assert mapping.size == 988 + assert mapping.rss == 796 + assert mapping.shared_clean == 0 + assert mapping.shared_dirty == 0 + assert mapping.private_clean == 796 + assert mapping.private_dirty == 0 + +def test_summing(): + line = ("988,796,0,0,796,0 08048000-0813f000 " + "r-xp 00000000 fd:00 75457 sometext") + snap = benchtool.Snapshot([smaps.Mapping(line), smaps.Mapping(line)]) + for name in ('size rss shared_clean shared_dirty ' + 'private_clean private_dirty').split(): + sumval = getattr(snap, name) + val1 = getattr(snap.mappings[0], name) + val2 = getattr(snap.mappings[1], name) + assert sumval == val1 + val2 + From hpk at codespeak.net Sat Sep 20 19:24:01 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 20 Sep 2008 19:24:01 +0200 (CEST) Subject: [pypy-svn] r58280 - in pypy/build/benchmem: . testing Message-ID: <20080920172401.A099D169EE7@codespeak.net> Author: hpk Date: Sat Sep 20 19:23:58 2008 New Revision: 58280 Added: pypy/build/benchmem/benchreport.py Modified: pypy/build/benchmem/benchtool.py pypy/build/benchmem/testing/test_benchtool.py Log: tweaking things a bit so one can run e.g. benchtool.py --executable=python2.5 benchtool.py --executable=pypy-c which logs to a single log file (bench.log by default) and then benchreport.py (reading from bench.log) reports maximum dirty numbers. that's rather non-sensical output for now but am off now for a bit and wanted to checkin anyway. Added: pypy/build/benchmem/benchreport.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchreport.py Sat Sep 20 19:23:58 2008 @@ -0,0 +1,18 @@ +import py +import smaps, benchtool + +if __name__ == "__main__": + benchlog = py.path.local("bench.log") + reader = benchtool.LogReader() + reader.parse_logfile(benchlog) + + tw = py.io.TerminalWriter() + + for name, results in reader.name2results.items(): + tw.sep("=", name) + for result in results: + print "%-30s max dirty = %s + %s" % ( + result.executable, + result.max("shared_dirty"), + result.max("private_dirty") + ) Modified: pypy/build/benchmem/benchtool.py ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/benchtool.py Sat Sep 20 19:23:58 2008 @@ -95,14 +95,14 @@ # ================ reading a benchmark log file ======================= # -class Benchresults(object): +class LogReader(object): def __init__(self): self.name2results = {} def parse_logfile(self, logpath): f = logpath.open() for result in BenchmarkResult.parse(f): - #print "parsed", result + print "parsed", result l = self.name2results.setdefault(result.benchname, []) l.append(result) f.close() @@ -115,6 +115,12 @@ self.benchname = benchname self.benchargs = benchargs + def max(self, attrname): + maxvalue = 0 + for snap in self.snapshots: + maxvalue = max(maxvalue, getattr(snap, attrname)) + return maxvalue + @classmethod def parse(cls, f): while not f.closed: @@ -138,8 +144,9 @@ #print "reading", line.strip() snapshots.append(Snapshot(mappings)) line = f.readline() - if not line or line == BenchRunner.SEPBENCH: - break + #print "reading", line.strip() + if not line or line.startswith(BenchRunner.SEPBENCH): + break yield BenchmarkResult(snapshots, executable=executable, benchname=benchname, benchargs=benchargs) @@ -206,5 +213,5 @@ runner = BenchRunner(executable, benchlog) for name in names: - runner.run_checkpointed_bench(name, (100, 100)) + runner.run_checkpointed_bench(name, (100, 1000)) print "bench results append to -->>>", benchlog Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Sat Sep 20 19:23:58 2008 @@ -31,7 +31,7 @@ runner = benchtool.BenchRunner(executable="python2.5", benchlog=log) runner.run_checkpointed_bench(path, args) assert log.check() - benchresult = benchtool.Benchresults() + benchresult = benchtool.LogReader() benchresult.parse_logfile(log) #assert reader.executable #assert reader.executable From arigo at codespeak.net Sat Sep 20 23:35:42 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 23:35:42 +0200 (CEST) Subject: [pypy-svn] r58281 - in pypy/dist: lib-python pypy/annotation pypy/annotation/test pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/pyparser pypy/interpreter/test pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/_codecs pypy/module/_file pypy/module/_pickle_support pypy/module/_rawffi pypy/module/_sre pypy/module/_stackless/test pypy/module/marshal pypy/module/posix pypy/module/rctime pypy/module/struct pypy/objspace/flow pypy/objspace/std pypy/rlib pypy/rlib/test pypy/rpython Message-ID: <20080920213542.7D5B0169EAE@codespeak.net> Author: arigo Date: Sat Sep 20 23:35:38 2008 New Revision: 58281 Modified: pypy/dist/lib-python/conftest.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/listdef.py pypy/dist/pypy/annotation/test/test_annrpython.py pypy/dist/pypy/interpreter/argument.py pypy/dist/pypy/interpreter/astcompiler/ast.py pypy/dist/pypy/interpreter/astcompiler/opt.py pypy/dist/pypy/interpreter/astcompiler/pyassem.py pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/interactive.py pypy/dist/pypy/interpreter/nestedscope.py pypy/dist/pypy/interpreter/pycode.py pypy/dist/pypy/interpreter/pyframe.py pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/interpreter/pyparser/astbuilder.py pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py pypy/dist/pypy/interpreter/test/test_objspace.py pypy/dist/pypy/module/__builtin__/abstractinst.py pypy/dist/pypy/module/__builtin__/compiling.py pypy/dist/pypy/module/__builtin__/interp_classobj.py pypy/dist/pypy/module/__builtin__/test/test_abstractinst.py pypy/dist/pypy/module/_codecs/interp_codecs.py pypy/dist/pypy/module/_file/interp_file.py pypy/dist/pypy/module/_pickle_support/maker.py pypy/dist/pypy/module/_rawffi/interp_rawffi.py pypy/dist/pypy/module/_rawffi/structure.py pypy/dist/pypy/module/_sre/interp_sre.py pypy/dist/pypy/module/_stackless/test/test_frame_chain_reconstruction.py pypy/dist/pypy/module/marshal/interp_marshal.py pypy/dist/pypy/module/posix/interp_posix.py pypy/dist/pypy/module/rctime/interp_time.py pypy/dist/pypy/module/struct/interp_struct.py pypy/dist/pypy/objspace/flow/objspace.py pypy/dist/pypy/objspace/std/iterobject.py pypy/dist/pypy/objspace/std/listobject.py pypy/dist/pypy/objspace/std/marshal_impl.py pypy/dist/pypy/objspace/std/model.py pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/ropeobject.py pypy/dist/pypy/objspace/std/ropeunicodeobject.py pypy/dist/pypy/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/strsliceobject.py pypy/dist/pypy/objspace/std/tupleobject.py pypy/dist/pypy/objspace/std/tupletype.py pypy/dist/pypy/objspace/std/typeobject.py pypy/dist/pypy/objspace/std/typetype.py pypy/dist/pypy/objspace/std/unicodeobject.py pypy/dist/pypy/rlib/debug.py pypy/dist/pypy/rlib/test/test_debug.py pypy/dist/pypy/rpython/callparse.py Log: issue395 resolved (fijal for most of the work) Merge the tuple-nonresizable-395 branch, making sure that some lists are "non-resizable", i.e. can be translated as a plain array - in particular the list W_TupleObject.wrappeditems. * make_sure_not_resized(): annotator hint to check non-resizability * if --no-listcompr, make_sure_not_resized() is disabled and only produces a warning (always the case with ootype for now) * new method space.viewiterable() returns a non-resizable immutable list * space.unpacktuple() is gone, use space.viewiterable() * space.viewiterable() and space.unpackiterable() are more efficient * space.newtuple() must receive a non-resizable list argument. Modified: pypy/dist/lib-python/conftest.py ============================================================================== --- pypy/dist/lib-python/conftest.py (original) +++ pypy/dist/lib-python/conftest.py Sat Sep 20 23:35:38 2008 @@ -166,12 +166,12 @@ set_argv = app.interphook('set_argv') def start_intercept(space): - w_suites, w_doctestmodules = space.unpacktuple(intercept_test_support(space)) + w_suites, w_doctestmodules = space.viewiterable(intercept_test_support(space)) return w_suites, w_doctestmodules def collect_intercept(space, w_suites, w_doctestmodules): w_result = callex(space, collect_intercepted, space, w_suites, w_doctestmodules) - w_namemethods, w_doctestlist = space.unpacktuple(w_result) + w_namemethods, w_doctestlist = space.viewiterable(w_result) return w_namemethods, w_doctestlist class SimpleRunItem(py.test.collect.Item): @@ -240,14 +240,14 @@ # setup {name -> wrapped testcase method} for w_item in space.unpackiterable(w_namemethods): - w_name, w_method = space.unpacktuple(w_item) + w_name, w_method = space.viewiterable(w_item) name = space.str_w(w_name) testitem = AppTestCaseMethod(name, parent=self, w_method=w_method) self.name2item[name] = testitem # setup {name -> wrapped doctest module} for w_item in space.unpackiterable(w_doctestlist): - w_name, w_module = space.unpacktuple(w_item) + w_name, w_module = space.viewiterable(w_item) name = space.str_w(w_name) testitem = AppDocTestModule(name, parent=self, w_module=w_module) self.name2item[name] = testitem Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Sat Sep 20 23:35:38 2008 @@ -737,7 +737,7 @@ if (expected_length is not None and expected_length != len(s_obj.items)): raise ValueError - return s_obj.items + return list(s_obj.items) if (s_obj.__class__ is SomeObject and getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] Modified: pypy/dist/pypy/annotation/listdef.py ============================================================================== --- pypy/dist/pypy/annotation/listdef.py (original) +++ pypy/dist/pypy/annotation/listdef.py Sat Sep 20 23:35:38 2008 @@ -26,6 +26,7 @@ self.bookkeeper = bookkeeper self.itemof = {} # set of all ListDefs using this ListItem self.read_locations = {} + self.dont_resize = False if bookkeeper is None: self.dont_change_any_more = True @@ -37,7 +38,7 @@ def resize(self): if not self.resized: - if self.dont_change_any_more: + if self.dont_change_any_more or self.dont_resize: raise TooLateForChange self.resized = True @@ -62,8 +63,13 @@ # things more general self, other = other, self + if other.dont_resize: + if self.resized: + raise TooLateForChange() + self.dont_resize = True if other.mutated: self.mutate() - if other.resized: self.resize() + if other.resized: + self.resize() if other.range_step != self.range_step: self.setrangestep(self._step_map[type(self.range_step), type(other.range_step)]) @@ -181,6 +187,11 @@ self.listitem.mutate() self.listitem.resize() + def never_resize(self): + if self.listitem.resized: + raise TooLateForChange() + self.listitem.dont_resize = True + MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Sat Sep 20 23:35:38 2008 @@ -10,7 +10,7 @@ from pypy.translator.translator import graphof as tgraphof from pypy.annotation import policy from pypy.annotation import specialize -from pypy.annotation.listdef import ListDef +from pypy.annotation.listdef import ListDef, TooLateForChange from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong @@ -3040,6 +3040,66 @@ a.build_types(f, [str]) + def test_listitem_no_mutating(self): + from pypy.rlib.debug import check_annotation + called = [] + + def checker(ann, bk): + called.append(True) + assert not ann.listdef.listitem.mutated + ann.listdef.never_resize() + + def f(): + l = [1,2,3] + check_annotation(l, checker) + return l + + def g(): + l = f() + l.append(4) + + a = self.RPythonAnnotator() + py.test.raises(TooLateForChange, a.build_types, g, []) + assert called + + def test_listitem_no_mutating2(self): + from pypy.rlib.debug import make_sure_not_resized + + def f(): + return make_sure_not_resized([1,2,3]) + + def g(): + l = [1,2,3] + l.append(4) + return l + + def fn(i): + if i: + func = f + else: + func = g + return func() + + a = self.RPythonAnnotator() + a.translator.config.translation.list_comprehension_operations = True + py.test.raises(TooLateForChange, a.build_types, fn, [int]) + + + def test_listitem_never_resize(self): + from pypy.rlib.debug import check_annotation + + def checker(ann, bk): + ann.listdef.never_resize() + + def f(): + l = [1,2,3] + l.append(4) + check_annotation(l, checker) + + a = self.RPythonAnnotator() + py.test.raises(TooLateForChange, a.build_types, f, []) + + def g(n): return [0,1,2,n] Modified: pypy/dist/pypy/interpreter/argument.py ============================================================================== --- pypy/dist/pypy/interpreter/argument.py (original) +++ pypy/dist/pypy/interpreter/argument.py Sat Sep 20 23:35:38 2008 @@ -136,9 +136,13 @@ unfiltered_kwds_w[argname] = w_arg assert not space.is_true(data_w_stararg) else: - args_w = data_args_w[:] - for w_stararg in space.unpackiterable(data_w_stararg): - args_w.append(w_stararg) + stararg_w = space.unpackiterable(data_w_stararg) + datalen = len(data_args_w) + args_w = [None] * (datalen + len(stararg_w)) + for i in range(0, datalen): + args_w[i] = data_args_w[i] + for i in range(0, len(stararg_w)): + args_w[i + datalen] = stararg_w[i] assert len(args_w) == need_cnt kwds_w = {} @@ -267,9 +271,9 @@ if has_vararg: if upfront > co_argcount: assert extravarargs is not None - stararg_w = extravarargs + stararg_w = extravarargs + [None] * self.nargs for i in range(self.nargs): - stararg_w.append(self.frame.peekvalue(self.nargs - 1 - i)) + stararg_w[i + len(extravarargs)] = self.frame.peekvalue(self.nargs - 1 - i) else: args_left = co_argcount - upfront stararg_w = [None] * (avail - co_argcount) @@ -315,7 +319,10 @@ def __init__(self, space, args_w, kwds_w=None, w_stararg=None, w_starstararg=None): self.space = space + assert isinstance(args_w, list) self.arguments_w = args_w + from pypy.rlib.debug import make_sure_not_resized + make_sure_not_resized(self.arguments_w) self.kwds_w = kwds_w self.w_stararg = w_stararg self.w_starstararg = w_starstararg @@ -360,7 +367,8 @@ "unpack the *arg and **kwd into w_arguments and kwds_w" # --- unpack the * argument now --- if self.w_stararg is not None: - self.arguments_w += self.space.unpackiterable(self.w_stararg) + self.arguments_w = (self.arguments_w + + self.space.unpackiterable(self.w_stararg)) self.w_stararg = None # --- unpack the ** argument now --- if self.kwds_w is None: @@ -405,8 +413,9 @@ if len(self.arguments_w) > argcount: raise ValueError, "too many arguments (%d expected)" % argcount if self.w_stararg is not None: - self.arguments_w += self.space.unpackiterable(self.w_stararg, - argcount - len(self.arguments_w)) + self.arguments_w = (self.arguments_w + + self.space.viewiterable(self.w_stararg, + argcount - len(self.arguments_w))) self.w_stararg = None elif len(self.arguments_w) < argcount: raise ValueError, "not enough arguments (%d expected)" % argcount @@ -529,7 +538,7 @@ assert extravarargs is not None starargs_w = extravarargs if num_args: - starargs_w.extend(args_w) + starargs_w = starargs_w + args_w elif num_args > args_left: starargs_w = args_w[args_left:] else: Modified: pypy/dist/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/dist/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/dist/pypy/interpreter/astcompiler/ast.py Sat Sep 20 23:35:38 2008 @@ -5923,6 +5923,7 @@ child.accept(self) def _mutate_list(self, lst): + # XXX O(n^2) i = 0 while i < len(lst): item = lst[i].mutate(self) Modified: pypy/dist/pypy/interpreter/astcompiler/opt.py ============================================================================== --- pypy/dist/pypy/interpreter/astcompiler/opt.py (original) +++ pypy/dist/pypy/interpreter/astcompiler/opt.py Sat Sep 20 23:35:38 2008 @@ -231,11 +231,13 @@ return self._visitAbstractTest(node, False) def visitTuple(self, node): - consts_w = [] - for subnode in node.nodes: + nodes = node.nodes + consts_w = [None] * len(nodes) + for i in range(len(nodes)): + subnode = nodes[i] if not isinstance(subnode, ast.Const): return node # not all constants - consts_w.append(subnode.value) + consts_w[i] = subnode.value return ast.Const(self.space.newtuple(consts_w)) def visitFor(self, node): Modified: pypy/dist/pypy/interpreter/astcompiler/pyassem.py ============================================================================== --- pypy/dist/pypy/interpreter/astcompiler/pyassem.py (original) +++ pypy/dist/pypy/interpreter/astcompiler/pyassem.py Sat Sep 20 23:35:38 2008 @@ -390,7 +390,7 @@ l_w = [None] * len(keys_w) for w_key in keys_w: index = space.int_w(space.getitem(self.w_consts, w_key)) - w_v = space.unpacktuple(w_key)[0] + w_v = space.viewiterable(w_key)[0] l_w[index] = w_v return l_w Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Sat Sep 20 23:35:38 2008 @@ -8,6 +8,7 @@ from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -657,16 +658,12 @@ (i, plural)) return items - def unpacktuple(self, w_tuple, expected_length=-1): - """Same as unpackiterable(), but only for tuples. - Only use for bootstrapping or performance reasons.""" - tuple_length = self.int_w(self.len(w_tuple)) - if expected_length != -1 and tuple_length != expected_length: - raise UnpackValueError("got a tuple of length %d instead of %d" % ( - tuple_length, expected_length)) - items = [ - self.getitem(w_tuple, self.wrap(i)) for i in range(tuple_length)] - return items + def viewiterable(self, w_iterable, expected_length=-1): + """ More or less the same as unpackiterable, but does not return + a copy. Please don't modify the result + """ + return make_sure_not_resized(self.unpackiterable(w_iterable, + expected_length)[:]) def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Sat Sep 20 23:35:38 2008 @@ -213,7 +213,7 @@ w(self.code), self.w_func_globals, w_closure, - nt(self.defs_w), + nt(self.defs_w[:]), self.w_func_dict, self.w_module, ] @@ -244,7 +244,7 @@ values_w = self.defs_w if not values_w: return space.w_None - return space.newtuple(values_w) + return space.newtuple(values_w[:]) def fset_func_defaults(space, self, w_defaults): if space.is_w(w_defaults, space.w_None): Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Sat Sep 20 23:35:38 2008 @@ -207,7 +207,7 @@ % (self.scopenext(), self.scopenext())) def visit_args_w(self, el): - self.run_args.append("space.unpacktuple(%s)" % self.scopenext()) + self.run_args.append("space.viewiterable(%s)" % self.scopenext()) def visit_w_args(self, el): self.run_args.append(self.scopenext()) @@ -406,7 +406,7 @@ # baseobjspace.W_Root is for wrapped arguments to keep wrapped # baseobjspace.Wrappable subclasses imply interp_w and a typecheck # argument.Arguments is for a final rest arguments Arguments object - # 'args_w' for unpacktuple applied to rest arguments + # 'args_w' for viewiterable applied to rest arguments # 'w_args' for rest arguments passed as wrapped tuple # str,int,float: unwrap argument as such type # (function, cls) use function to check/unwrap argument of type cls Modified: pypy/dist/pypy/interpreter/interactive.py ============================================================================== --- pypy/dist/pypy/interpreter/interactive.py (original) +++ pypy/dist/pypy/interpreter/interactive.py Sat Sep 20 23:35:38 2008 @@ -73,7 +73,7 @@ words = self.get_words(w_clz) try: w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.unpacktuple(w_bases) + bases_w = s.viewiterable(w_bases) except error.OperationError: return words Modified: pypy/dist/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/dist/pypy/interpreter/nestedscope.py (original) +++ pypy/dist/pypy/interpreter/nestedscope.py Sat Sep 20 23:35:38 2008 @@ -202,7 +202,7 @@ if codeobj.magic >= 0xa0df281: # CPython 2.5 AST branch merge w_freevarstuple = f.popvalue() freevars = [f.space.interp_w(Cell, cell) - for cell in f.space.unpacktuple(w_freevarstuple)] + for cell in f.space.viewiterable(w_freevarstuple)] else: nfreevars = len(codeobj.co_freevars) freevars = [f.space.interp_w(Cell, f.popvalue()) Modified: pypy/dist/pypy/interpreter/pycode.py ============================================================================== --- pypy/dist/pypy/interpreter/pycode.py (original) +++ pypy/dist/pypy/interpreter/pycode.py Sat Sep 20 23:35:38 2008 @@ -11,15 +11,12 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rlib.rarithmetic import intmask +from pypy.rlib.debug import make_sure_not_resized # helper def unpack_str_tuple(space,w_str_tuple): - els = [] - for w_el in space.unpackiterable(w_str_tuple): - els.append(space.str_w(w_el)) - return els - + return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] # code object contants, for co_flags below CO_OPTIMIZED = 0x0001 @@ -66,7 +63,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = consts + self.co_consts_w = make_sure_not_resized(consts) self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -122,12 +119,13 @@ This method is called by our compile builtin function. """ assert isinstance(code, types.CodeType) - newconsts_w = [] + newconsts_w = [None] * len(code.co_consts) + num = 0 for const in code.co_consts: if isinstance(const, types.CodeType): # from stable compiler const = PyCode._from_code(space, const, hidden_applevel=hidden_applevel) - - newconsts_w.append(space.wrap(const)) + newconsts_w[num] = space.wrap(const) + num += 1 # stick the underlying CPython magic value, if the code object # comes from there return PyCode(space, code.co_argcount, @@ -213,12 +211,14 @@ def _to_code(self): """For debugging only.""" - consts = [] + consts = [None] * len(self.co_consts_w) + num = 0 for w in self.co_consts_w: if isinstance(w, PyCode): - consts.append(w._to_code()) + consts[num] = w._to_code() else: - consts.append(self.space.unwrap(w)) + consts[num] = self.space.unwrap(w) + num += 1 return new.code( self.co_argcount, self.co_nlocals, self.co_stacksize, @@ -321,8 +321,11 @@ space.wrap("code: argcount must not be negative")) if nlocals < 0: raise OperationError(space.w_ValueError, - space.wrap("code: nlocals must not be negative")) - consts_w = space.unpacktuple(w_constants) + space.wrap("code: nlocals must not be negative")) + if not space.is_true(space.isinstance(w_constants, space.w_tuple)): + raise OperationError(space.w_TypeError, + space.wrap("Expected tuple for constants")) + consts_w = space.viewiterable(w_constants) names = unpack_str_tuple(space, w_names) varnames = unpack_str_tuple(space, w_varnames) if w_freevars is not None: Modified: pypy/dist/pypy/interpreter/pyframe.py ============================================================================== --- pypy/dist/pypy/interpreter/pyframe.py (original) +++ pypy/dist/pypy/interpreter/pyframe.py Sat Sep 20 23:35:38 2008 @@ -152,15 +152,22 @@ dic_w[key] = w_value return dic_w - def popvalues(self, n): - values_w = [None] * n - while True: - n -= 1 - if n < 0: - break - hint(n, concrete=True) - values_w[n] = self.popvalue() - return values_w + # we need two popvalues that return different data types: + # one in case we want list another in case of tuple + def _new_popvalues(): + def popvalues(self, n): + values_w = [None] * n + while True: + n -= 1 + if n < 0: + break + hint(n, concrete=True) + values_w[n] = self.popvalue() + return values_w + return popvalues + popvalues = _new_popvalues() + popvalues_mutable = _new_popvalues() + del _new_popvalues def peekvalues(self, n): values_w = [None] * n Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sat Sep 20 23:35:38 2008 @@ -552,7 +552,7 @@ w_compile_flags, f.space.wrap(f.get_builtin()), f.space.gettypeobject(PyCode.typedef)) - w_prog, w_globals, w_locals = f.space.unpacktuple(w_resulttuple, 3) + w_prog, w_globals, w_locals = f.space.viewiterable(w_resulttuple, 3) plain = f.w_locals is not None and f.space.is_w(w_locals, f.w_locals) if plain: @@ -679,7 +679,7 @@ f.pushvalue(w_tuple) def BUILD_LIST(f, itemcount, *ignored): - items = f.popvalues(itemcount) + items = f.popvalues_mutable(itemcount) w_list = f.space.newlist(items) f.pushvalue(w_list) @@ -905,7 +905,7 @@ def MAKE_FUNCTION(f, numdefaults, *ignored): w_codeobj = f.popvalue() codeobj = f.space.interp_w(PyCode, w_codeobj) - defaultarguments = f.popvalues(numdefaults) + defaultarguments = f.popvalues_mutable(numdefaults) fn = function.Function(f.space, codeobj, f.w_globals, defaultarguments) f.pushvalue(f.space.wrap(fn)) Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Sat Sep 20 23:35:38 2008 @@ -416,11 +416,12 @@ if l == 1: builder.push(atoms[0]) return - items = [] token = atoms[1] if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COMMA']: + items = [] for i in range(0, l, 2): # this is atoms not 1 items.append(atoms[i]) + builder.push(ast.Tuple(items, lineno)) else: # genfor: 'i for i in j' # GenExpr(GenExprInner(Name('i'), [GenExprFor(AssName('i', 'OP_ASSIGN'), Name('j'), [])])))])) @@ -428,20 +429,6 @@ genexpr_for = parse_genexpr_for(atoms[1:]) genexpr_for[0].is_outmost = True builder.push(ast.GenExpr(ast.GenExprInner(expr, genexpr_for, lineno), lineno)) - return - isConst = True - values = [] - for item in items: - if isinstance(item, ast.Const): - values.append(item.value) - else: - isConst = False - break - if isConst: - builder.push(ast.Const(builder.space.newtuple(values), lineno)) - else: - builder.push(ast.Tuple(items, lineno)) - return def build_lambdef(builder, nb): """lambdef: 'lambda' [varargslist] ':' test""" @@ -739,21 +726,8 @@ if len(atoms) <= 2: builder.push(atoms[0]) else: - names = [] - values = [] - isConst = True - for index in range(0, len(atoms), 2): - item = atoms[index] - names.append(item) - if isinstance(item, ast.Const): - values.append(item) - else: - isConst = False - if isConst: - builder.push(ast.Const(builder.space.newtuple(values), atoms[0].lineno)) - else: - builder.push(ast.Tuple(names, atoms[0].lineno)) - + items = [atoms[index] for index in range(0, len(atoms), 2)] + builder.push(ast.Tuple(items, atoms[0].lineno)) def build_while_stmt(builder, nb): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" Modified: pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py (original) +++ pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py Sat Sep 20 23:35:38 2008 @@ -17,9 +17,10 @@ def as_w_tuple(self, space, lineno=False): num, value, lineno = self.nodes[0] - content = [space.wrap(num), space.wrap(value)] if lineno: - content.append(space.wrap(lineno)) + content = [space.wrap(num), space.wrap(value), space.wrap(lineno)] + else: + content = [space.wrap(num), space.wrap(value)] return space.newtuple(content) @@ -34,8 +35,8 @@ return tuple(l) def as_w_tuple(self, space, lineno=False): - l = [space.wrap(self.num)] - l += [node.as_w_tuple(space, lineno) for node in self.nodes] + l = ([space.wrap(self.num)] + + [node.as_w_tuple(space, lineno) for node in self.nodes]) return space.newtuple(l) Modified: pypy/dist/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_objspace.py (original) +++ pypy/dist/pypy/interpreter/test/test_objspace.py Sat Sep 20 23:35:38 2008 @@ -54,14 +54,14 @@ raises(ValueError, self.space.unpackiterable, w_l, 3) raises(ValueError, self.space.unpackiterable, w_l, 5) - def test_unpacktuple(self): + def test_viewiterable(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.unpacktuple(w_l) == l - assert self.space.unpacktuple(w_l, 4) == l - raises(ValueError, self.space.unpacktuple, w_l, 3) - raises(ValueError, self.space.unpacktuple, w_l, 5) + assert self.space.viewiterable(w_l) == l + assert self.space.viewiterable(w_l, 4) == l + raises(ValueError, self.space.viewiterable, w_l, 3) + raises(ValueError, self.space.viewiterable, w_l, 5) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, @@ -203,7 +203,7 @@ w_res = space.call_obj_args(w_f, w_9, Arguments(space, [w_1])) - w_x, w_y = space.unpacktuple(w_res, 2) + w_x, w_y = space.viewiterable(w_res, 2) assert w_x is w_9 assert w_y is w_1 Modified: pypy/dist/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/dist/pypy/module/__builtin__/abstractinst.py Sat Sep 20 23:35:38 2008 @@ -79,7 +79,7 @@ # -- case (anything, tuple) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.unpacktuple(w_klass_or_tuple): + for w_klass in space.viewiterable(w_klass_or_tuple): if abstract_isinstance_w(space, w_obj, w_klass): return True return False @@ -104,7 +104,7 @@ return True w_bases = _get_bases(space, w_derived) if w_bases is not None: - for w_base in space.unpacktuple(w_bases): + for w_base in space.viewiterable(w_bases): if _issubclass_recurse(space, w_base, w_top): return True return False @@ -134,7 +134,7 @@ # -- case (class-like-object, tuple-of-classes) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.unpacktuple(w_klass_or_tuple): + for w_klass in space.viewiterable(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass): return True return False Modified: pypy/dist/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/compiling.py (original) +++ pypy/dist/pypy/module/__builtin__/compiling.py Sat Sep 20 23:35:38 2008 @@ -67,9 +67,9 @@ "", "eval") except OperationError, e: if e.match(space, space.w_SyntaxError): - e_value_w = space.unpacktuple(e.w_value) + e_value_w = space.viewiterable(e.w_value) if len(e_value_w) == 2: - e_loc_w = space.unpacktuple(e_value_w[1]) + e_loc_w = space.viewiterable(e_value_w[1]) e.w_value = space.newtuple([e_value_w[0], space.newtuple([space.w_None]+ e_loc_w[1:])]) Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/interp_classobj.py Sat Sep 20 23:35:38 2008 @@ -35,7 +35,7 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -78,7 +78,7 @@ raise OperationError( space.w_TypeError, space.wrap("__bases__ must be a tuple object")) - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, @@ -280,7 +280,7 @@ if not e.match(space, space.w_TypeError): raise return [None, None] - return space.unpacktuple(w_tup, 2) + return space.viewiterable(w_tup, 2) def descr_instance_new(space, w_type, w_class, w_dict=None): # w_type is not used at all Modified: pypy/dist/pypy/module/__builtin__/test/test_abstractinst.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_abstractinst.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_abstractinst.py Sat Sep 20 23:35:38 2008 @@ -5,7 +5,7 @@ def test_abstract_isclass(self): space = self.space - w_B1, w_B2, w_B3, w_X, w_Y = space.unpacktuple(space.appexec([], """(): + w_B1, w_B2, w_B3, w_X, w_Y = space.viewiterable(space.appexec([], """(): class X(object): pass class Y: pass B1, B2, B3 = X(), X(), X() @@ -22,7 +22,7 @@ def test_abstract_getclass(self): space = self.space - w_x, w_y, w_A, w_MyInst = space.unpacktuple(space.appexec([], """(): + w_x, w_y, w_A, w_MyInst = space.viewiterable(space.appexec([], """(): class MyInst(object): def __init__(self, myclass): self.myclass = myclass Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/interp_codecs.py Sat Sep 20 23:35:38 2008 @@ -35,7 +35,7 @@ space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) - w_replace, w_newpos = space.unpacktuple(w_res, 2) + w_replace, w_newpos = space.viewiterable(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Sat Sep 20 23:35:38 2008 @@ -419,7 +419,7 @@ result.append(space.wrap('\n')) if newlines & 4: result.append(space.wrap('\r\n')) - return space.newtuple(result) + return space.newtuple(result[:]) def descr_file_softspace(space, file): return space.wrap(file.softspace) Modified: pypy/dist/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/dist/pypy/module/_pickle_support/maker.py (original) +++ pypy/dist/pypy/module/_pickle_support/maker.py Sat Sep 20 23:35:38 2008 @@ -88,15 +88,18 @@ create a tuple with the object and store a tuple with the positions of NULLs as first element. """ - nulls = [] - tup = [space.w_None] + tup = [None] * (len(seq_w) + 1) w = space.wrap - + num = 1 + nulls = [None for i in seq_w if i is None] + null_num = 0 for w_obj in seq_w: if w_obj is None: - nulls.append(w(len(tup)-1)) + nulls[null_num] = w(num - 1) + null_num += 1 w_obj = space.w_None - tup.append(w_obj) + tup[num] = w_obj + num += 1 tup[0] = space.newtuple(nulls) return space.newtuple(tup) Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Sat Sep 20 23:35:38 2008 @@ -98,7 +98,7 @@ resshape = cache.get_array_type(letter2tp(space, letter)) else: letter = 'V' - w_shapetype, w_length = space.unpacktuple(w_shape, expected_length=2) + w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) from pypy.module._rawffi.structure import W_Structure resshape = space.interp_w(W_Structure, w_shapetype) ffi_type = resshape.get_ffi_type() @@ -109,7 +109,7 @@ letter = space.str_w(w_shape) return letter2tp(space, letter) else: - w_shapetype, w_length = space.unpacktuple(w_shape, expected_length=2) + w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) resshape = space.interp_w(W_DataShape, w_shapetype) length = space.int_w(w_length) size, alignment = resshape._size_alignment() @@ -148,7 +148,7 @@ """ ffi_restype, resshape = unpack_resshape(space, w_restype) w = space.wrap - argtypes_w = space.unpackiterable(w_argtypes) + argtypes_w = space.viewiterable(w_argtypes) w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w(name), w_argtypes, w(resshape)]) try: Modified: pypy/dist/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/structure.py (original) +++ pypy/dist/pypy/module/_rawffi/structure.py Sat Sep 20 23:35:38 2008 @@ -119,7 +119,7 @@ def descr_new_structure(space, w_type, w_shapeinfo): if space.is_true(space.isinstance(w_shapeinfo, space.w_tuple)): - w_size, w_alignment = space.unpacktuple(w_shapeinfo, expected_length=2) + w_size, w_alignment = space.viewiterable(w_shapeinfo, expected_length=2) S = W_Structure(space, None, space.int_w(w_size), space.int_w(w_alignment)) else: Modified: pypy/dist/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/dist/pypy/module/_sre/interp_sre.py (original) +++ pypy/dist/pypy/module/_sre/interp_sre.py Sat Sep 20 23:35:38 2008 @@ -95,11 +95,10 @@ """Creates a tuple of index pairs representing matched groups, a format that's convenient for SRE_Match.""" space = self.space - lst = [] - for value1, value2 in self.create_regs(group_count): - lst.append(space.newtuple([space.wrap(value1), - space.wrap(value2)])) - return space.newtuple(lst) + return space.newtuple([ + space.newtuple([space.wrap(value1), + space.wrap(value2)]) + for value1, value2 in self.create_regs(group_count)]) w_create_regs.unwrap_spec = ['self', int] def fget_start(space, self): Modified: pypy/dist/pypy/module/_stackless/test/test_frame_chain_reconstruction.py ============================================================================== --- pypy/dist/pypy/module/_stackless/test/test_frame_chain_reconstruction.py (original) +++ pypy/dist/pypy/module/_stackless/test/test_frame_chain_reconstruction.py Sat Sep 20 23:35:38 2008 @@ -92,7 +92,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -151,7 +151,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -223,7 +223,7 @@ return co, f, g """) - w_co, w_f, w_g = space.unpacktuple(w_res) + w_co, w_f, w_g = space.viewiterable(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code Modified: pypy/dist/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/dist/pypy/module/marshal/interp_marshal.py Sat Sep 20 23:35:38 2008 @@ -234,24 +234,16 @@ self._overflow() self.nesting -= 1 - # this function is inlined below - def put_list_w(self, list_w, lng): - self.nesting += 1 - self.put_int(lng) - idx = 0 - while idx < lng: - self.put_w_obj(list_w[idx]) - idx += 1 - self.nesting -= 1 - - def put_list_w(self, list_w, lng): + def put_tuple_w(self, typecode, lst_w): self.nesting += 1 + self.start(typecode) + lng = len(lst_w) self.put_int(lng) idx = 0 space = self.space if self.nesting < MAX_MARSHAL_DEPTH: while idx < lng: - w_obj = list_w[idx] + w_obj = lst_w[idx] self.space.marshal_w(w_obj, self) idx += 1 else: @@ -443,18 +435,6 @@ lng = self.get_lng() return self.get(lng) - # this function is inlined below - def get_list_w(self): - self.nesting += 1 - lng = self.get_lng() - res_w = [None] * lng - idx = 0 - while idx < lng: - res_w[idx] = self.get_w_obj(False) - idx += 1 - self.nesting -= 1 - return res_w - def get_w_obj(self, allow_null): self.nesting += 1 space = self.space @@ -471,7 +451,7 @@ return w_ret # inlined version to save a nesting level - def get_list_w(self): + def get_tuple_w(self): self.nesting += 1 lng = self.get_lng() res_w = [None] * lng @@ -494,6 +474,9 @@ self.nesting -= 1 return res_w + def get_list_w(self): + return self.get_tuple_w()[:] + def _overflow(self): self.raise_exc('object too deeply nested to unmarshal') Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sat Sep 20 23:35:38 2008 @@ -97,7 +97,7 @@ FIELDS = PORTABLE_STAT_FIELDS else: FIELDS = STAT_FIELDS # also when not translating at all - lst = [] + lst = [None] * ll_os_stat.N_INDEXABLE_FIELDS w_keywords = space.newdict() for i, (name, TYPE) in FIELDS: value = getattr(st, name) @@ -105,7 +105,7 @@ # value = int(value) # rounded to an integer for indexed access w_value = space.wrap(value) if i < ll_os_stat.N_INDEXABLE_FIELDS: - lst.append(w_value) + lst[i] = w_value else: space.setitem(w_keywords, space.wrap(name), w_value) Modified: pypy/dist/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/dist/pypy/module/rctime/interp_time.py (original) +++ pypy/dist/pypy/module/rctime/interp_time.py Sat Sep 20 23:35:38 2008 @@ -166,17 +166,16 @@ return rffi.r_time_t(seconds) def _tm_to_tuple(space, t): - time_tuple = [] - - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900)) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1)) # want january == 1 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_mday')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_hour')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_min')) ) - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_sec')) ) - time_tuple.append(space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7)) # want monday == 0 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1)) # want january, 1 == 1 - time_tuple.append(space.wrap(rffi.getintfield(t, 'c_tm_isdst')) ) + time_tuple = [ + space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900), + space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1 + space.wrap(rffi.getintfield(t, 'c_tm_mday')), + space.wrap(rffi.getintfield(t, 'c_tm_hour')), + space.wrap(rffi.getintfield(t, 'c_tm_min')), + space.wrap(rffi.getintfield(t, 'c_tm_sec')), + space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0 + space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1 + space.wrap(rffi.getintfield(t, 'c_tm_isdst'))] w_struct_time = _get_module_object(space, 'struct_time') w_time_tuple = space.newtuple(time_tuple) Modified: pypy/dist/pypy/module/struct/interp_struct.py ============================================================================== --- pypy/dist/pypy/module/struct/interp_struct.py (original) +++ pypy/dist/pypy/module/struct/interp_struct.py Sat Sep 20 23:35:38 2008 @@ -33,5 +33,5 @@ fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space) - return space.newtuple(fmtiter.result_w) + return space.newtuple(fmtiter.result_w[:]) unpack.unwrap_spec = [ObjSpace, str, 'bufferstr'] Modified: pypy/dist/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/flow/objspace.py (original) +++ pypy/dist/pypy/objspace/flow/objspace.py Sat Sep 20 23:35:38 2008 @@ -202,7 +202,7 @@ # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) # checking a tuple of classes - for w_klass in self.unpacktuple(w_check_class): + for w_klass in self.viewiterable(w_check_class): if ObjSpace.exception_match(self, w_exc_type, w_klass): return True return False @@ -264,12 +264,7 @@ checkgraph(graph) return graph - def unpacktuple(self, w_tuple, expected_length=None): -## # special case to accept either Constant tuples -## # or real tuples of Variables/Constants -## if isinstance(w_tuple, tuple): -## result = w_tuple -## else: + def viewiterable(self, w_tuple, expected_length=None): unwrapped = self.unwrap(w_tuple) result = tuple([Constant(x) for x in unwrapped]) if expected_length is not None and len(result) != expected_length: @@ -286,26 +281,6 @@ if isinstance(w_iterable, Variable) and expected_length is None: raise UnwrapException, ("cannot unpack a Variable iterable" "without knowing its length") -## # XXX TEMPORARY HACK XXX TEMPORARY HACK XXX TEMPORARY HACK -## print ("*** cannot unpack a Variable iterable " -## "without knowing its length,") -## print " assuming a list or tuple with up to 7 items" -## items = [] -## w_len = self.len(w_iterable) -## i = 0 -## while True: -## w_i = self.wrap(i) -## w_cond = self.eq(w_len, w_i) -## if self.is_true(w_cond): -## break # done -## if i == 7: -## # too many values -## raise OperationError(self.w_AssertionError, self.w_None) -## w_item = self.do_operation('getitem', w_iterable, w_i) -## items.append(w_item) -## i += 1 -## return items -## # XXX TEMPORARY HACK XXX TEMPORARY HACK XXX TEMPORARY HACK elif expected_length is not None: w_len = self.len(w_iterable) w_correct = self.eq(w_len, self.wrap(expected_length)) Modified: pypy/dist/pypy/objspace/std/iterobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/iterobject.py (original) +++ pypy/dist/pypy/objspace/std/iterobject.py Sat Sep 20 23:35:38 2008 @@ -16,16 +16,34 @@ w_self.w_seq = w_seq w_self.index = index + def getlength(self, space): + if self.w_seq is None: + return space.wrap(0) + index = self.index + w_length = space.len(self.w_seq) + w_len = space.sub(w_length, space.wrap(index)) + if space.is_true(space.lt(w_len,space.wrap(0))): + w_len = space.wrap(0) + return w_len + class W_SeqIterObject(W_AbstractSeqIterObject): """Sequence iterator implementation for general sequences.""" -class W_FastSeqIterObject(W_AbstractSeqIterObject): - """Sequence iterator specialized for lists or tuples, accessing +class W_FastListIterObject(W_AbstractSeqIterObject): + """Sequence iterator specialized for lists, accessing directly their RPython-level list of wrapped objects. """ def __init__(w_self, w_seq, wrappeditems): W_AbstractSeqIterObject.__init__(w_self, w_seq) - w_self.wrappeditems = wrappeditems + w_self.listitems = wrappeditems + +class W_FastTupleIterObject(W_AbstractSeqIterObject): + """Sequence iterator specialized for tuples, accessing + directly their RPython-level list of wrapped objects. + """ + def __init__(w_self, w_seq, wrappeditems): + W_AbstractSeqIterObject.__init__(w_self, w_seq) + w_self.tupleitems = wrappeditems class W_ReverseSeqIterObject(W_Object): from pypy.objspace.std.itertype import reverse_iter_typedef as typedef @@ -37,7 +55,8 @@ registerimplementation(W_SeqIterObject) -registerimplementation(W_FastSeqIterObject) +registerimplementation(W_FastListIterObject) +registerimplementation(W_FastTupleIterObject) registerimplementation(W_ReverseSeqIterObject) def iter__SeqIter(space, w_seqiter): @@ -57,40 +76,47 @@ return w_item def len__SeqIter(space, w_seqiter): - if w_seqiter.w_seq is None: - return space.wrap(0) + return w_seqiter.getlength(space) + + +def iter__FastTupleIter(space, w_seqiter): + return w_seqiter + +def next__FastTupleIter(space, w_seqiter): + if w_seqiter.tupleitems is None: + raise OperationError(space.w_StopIteration, space.w_None) index = w_seqiter.index - w_length = space.len(w_seqiter.w_seq) - w_len = space.sub(w_length, space.wrap(index)) - if space.is_true(space.lt(w_len,space.wrap(0))): - w_len = space.wrap(0) - return w_len + try: + w_item = w_seqiter.tupleitems[index] + except IndexError: + w_seqiter.tupleitems = None + w_seqiter.w_seq = None + raise OperationError(space.w_StopIteration, space.w_None) + w_seqiter.index = index + 1 + return w_item +def len__FastTupleIter(space, w_seqiter): + return w_seqiter.getlength(space) -def iter__FastSeqIter(space, w_seqiter): + +def iter__FastListIter(space, w_seqiter): return w_seqiter -def next__FastSeqIter(space, w_seqiter): - if w_seqiter.wrappeditems is None: +def next__FastListIter(space, w_seqiter): + if w_seqiter.listitems is None: raise OperationError(space.w_StopIteration, space.w_None) index = w_seqiter.index try: - w_item = w_seqiter.wrappeditems[index] + w_item = w_seqiter.listitems[index] except IndexError: - w_seqiter.wrappeditems = None + w_seqiter.listitems = None w_seqiter.w_seq = None raise OperationError(space.w_StopIteration, space.w_None) w_seqiter.index = index + 1 return w_item -def len__FastSeqIter(space, w_seqiter): - if w_seqiter.wrappeditems is None: - return space.wrap(0) - totallength = len(w_seqiter.wrappeditems) - remaining = totallength - w_seqiter.index - if remaining < 0: - remaining = 0 - return space.wrap(remaining) +def len__FastListIter(space, w_seqiter): + return w_seqiter.getlength(space) def iter__ReverseSeqIter(space, w_seqiter): Modified: pypy/dist/pypy/objspace/std/listobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/listobject.py (original) +++ pypy/dist/pypy/objspace/std/listobject.py Sat Sep 20 23:35:38 2008 @@ -2,13 +2,11 @@ from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.listtype import get_list_index from pypy.objspace.std.sliceobject import W_SliceObject -from pypy.objspace.std.tupleobject import W_TupleObject from pypy.objspace.std import slicetype from pypy.interpreter import gateway, baseobjspace from pypy.rlib.listsort import TimSort - class W_ListObject(W_Object): from pypy.objspace.std.listtype import list_typedef as typedef @@ -26,7 +24,6 @@ def append(w_list, w_item): w_list.wrappeditems.append(w_item) - registerimplementation(W_ListObject) @@ -95,7 +92,7 @@ def iter__List(space, w_list): from pypy.objspace.std import iterobject - return iterobject.W_FastSeqIterObject(w_list, w_list.wrappeditems) + return iterobject.W_FastListIterObject(w_list, w_list.wrappeditems) def add__List_List(space, w_list1, w_list2): return W_ListObject(w_list1.wrappeditems + w_list2.wrappeditems) @@ -253,10 +250,6 @@ l = w_list2.wrappeditems return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) -def setitem__List_Slice_Tuple(space, w_list, w_slice, w_tuple): - t = w_tuple.wrappeditems - return _setitem_slice_helper(space, w_list, w_slice, t, len(t)) - def setitem__List_Slice_ANY(space, w_list, w_slice, w_iterable): l = space.unpackiterable(w_iterable) return _setitem_slice_helper(space, w_list, w_slice, l, len(l)) Modified: pypy/dist/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/dist/pypy/objspace/std/marshal_impl.py (original) +++ pypy/dist/pypy/objspace/std/marshal_impl.py Sat Sep 20 23:35:38 2008 @@ -87,8 +87,7 @@ put_int(int) puts an integer put_pascal(s) puts a short string put_w_obj(w_obj) puts a wrapped object -put_list_w(list_w, lng) puts a list of lng wrapped objects - +put_tuple_w(TYPE, tuple_w) puts tuple_w, an unwrapped list of wrapped objects """ handled_by_any = [] @@ -314,19 +313,17 @@ register(TYPE_STRINGREF, unmarshal_stringref) def marshal_w__Tuple(space, w_tuple, m): - m.start(TYPE_TUPLE) items = w_tuple.wrappeditems - m.put_list_w(items, len(items)) + m.put_tuple_w(TYPE_TUPLE, items) def unmarshal_Tuple(space, u, tc): - items_w = u.get_list_w() + items_w = u.get_tuple_w() return space.newtuple(items_w) register(TYPE_TUPLE, unmarshal_Tuple) def marshal_w__List(space, w_list, m): - m.start(TYPE_LIST) - items = w_list.wrappeditems - m.put_list_w(items, len(items)) + items = w_list.wrappeditems[:] + m.put_tuple_w(TYPE_LIST, items) def unmarshal_List(space, u, tc): items_w = u.get_list_w() @@ -346,7 +343,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) for w_tuple in w_dict.implementation.items(): - w_key, w_value = space.unpacktuple(w_tuple, 2) + w_key, w_value = space.viewiterable(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) @@ -379,8 +376,7 @@ m.put_int(x.co_stacksize) m.put_int(x.co_flags) m.atom_str(TYPE_STRING, x.co_code) - m.start(TYPE_TUPLE) - m.put_list_w(x.co_consts_w, len(x.co_consts_w)) + m.put_tuple_w(TYPE_TUPLE, x.co_consts_w) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w]) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames) m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars) @@ -423,7 +419,8 @@ flags = u.get_int() code = unmarshal_str(u) u.start(TYPE_TUPLE) - consts_w = u.get_list_w() + consts_w = u.get_tuple_w() + # copy in order not to merge it with anything else names = unmarshal_strlist(u, TYPE_TUPLE) varnames = unmarshal_strlist(u, TYPE_TUPLE) freevars = unmarshal_strlist(u, TYPE_TUPLE) @@ -460,46 +457,38 @@ register(TYPE_UNICODE, unmarshal_Unicode) app = gateway.applevel(r''' - def set_to_list(theset): - return [item for item in theset] - - def list_to_set(datalist, frozen=False): + def tuple_to_set(datalist, frozen=False): if frozen: return frozenset(datalist) return set(datalist) ''') -set_to_list = app.interphook('set_to_list') -list_to_set = app.interphook('list_to_set') +tuple_to_set = app.interphook('tuple_to_set') # not directly supported: def marshal_w_set(space, w_set, m): - w_lis = set_to_list(space, w_set) # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.unpackiterable(w_lis) - m.start(TYPE_SET) - m.put_list_w(lis_w, len(lis_w)) + lis_w = space.viewiterable(w_set) + m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - w_lis = set_to_list(space, w_frozenset) - lis_w = space.unpackiterable(w_lis) - m.start(TYPE_FROZENSET) - m.put_list_w(lis_w, len(lis_w)) + lis_w = space.viewiterable(w_frozenset) + m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) def unmarshal_set_frozenset(space, u, tc): - items_w = u.get_list_w() + items_w = u.get_tuple_w() if tc == TYPE_SET: w_frozen = space.w_False else: w_frozen = space.w_True - w_lis = space.newlist(items_w) - return list_to_set(space, w_lis, w_frozen) + w_tup = space.newtuple(items_w) + return tuple_to_set(space, w_tup, w_frozen) register(TYPE_SET + TYPE_FROZENSET, unmarshal_set_frozenset) # dispatching for all not directly dispatched types Modified: pypy/dist/pypy/objspace/std/model.py ============================================================================== --- pypy/dist/pypy/objspace/std/model.py (original) +++ pypy/dist/pypy/objspace/std/model.py Sat Sep 20 23:35:38 2008 @@ -103,7 +103,8 @@ longobject.W_LongObject: [], noneobject.W_NoneObject: [], iterobject.W_SeqIterObject: [], - iterobject.W_FastSeqIterObject: [], + iterobject.W_FastListIterObject: [], + iterobject.W_FastTupleIterObject: [], iterobject.W_ReverseSeqIterObject: [], unicodeobject.W_UnicodeObject: [], dictproxyobject.W_DictProxyObject: [], Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Sat Sep 20 23:35:38 2008 @@ -8,6 +8,7 @@ from pypy.interpreter.pyopcode import unrolling_compare_dispatch_table, \ BytecodeCorruption from pypy.rlib.objectmodel import instantiate +from pypy.rlib.debug import make_sure_not_resized from pypy.interpreter.gateway import PyPyCacheDir from pypy.tool.cache import Cache from pypy.tool.sourcetools import func_with_new_name @@ -378,7 +379,7 @@ space = self # too early for unpackiterable as well :-( name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.unpacktuple(space.getitem(w_args, space.wrap(1))) + bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) dic = space.unwrap(space.getitem(w_args, space.wrap(2))) dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) bases = list(bases) @@ -563,6 +564,7 @@ def newtuple(self, list_w): from pypy.objspace.std.tupletype import wraptuple assert isinstance(list_w, list) + make_sure_not_resized(list_w) return wraptuple(self, list_w) def newlist(self, list_w): @@ -627,12 +629,32 @@ return instance allocate_instance._annspecialcase_ = "specialize:arg(1)" - def unpacktuple(self, w_tuple, expected_length=-1): - assert isinstance(w_tuple, self.TupleObjectCls) - t = w_tuple.getitems() - if expected_length != -1 and expected_length != len(t): - raise ValueError, "got a tuple of length %d instead of %d" % ( - len(t), expected_length) + # two following functions are almost identical, but in fact they + # have different return type. First one is a resizable list, second + # one is not + + def unpackiterable(self, w_obj, expected_length=-1): + if isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems[:] + elif isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.unpackiterable(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) + return t + + def viewiterable(self, w_obj, expected_length=-1): + """ Fast paths + """ + if isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems + elif isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.viewiterable(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) return t def sliceindices(self, w_slice, w_length): Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Sat Sep 20 23:35:38 2008 @@ -533,7 +533,7 @@ def str_endswith__Rope_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -552,7 +552,7 @@ def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/dist/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeunicodeobject.py Sat Sep 20 23:35:38 2008 @@ -485,7 +485,7 @@ def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = ropeunicode_w(space, w_prefix) if rope.startswith(unistr, prefix, start, end): return space.w_True @@ -494,7 +494,7 @@ def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = ropeunicode_w(space, w_suffix) if rope.endswith(unistr, suffix, start, end): return space.w_True Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Sat Sep 20 23:35:38 2008 @@ -593,7 +593,7 @@ def str_endswith__String_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -611,7 +611,7 @@ def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/dist/pypy/objspace/std/strsliceobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/strsliceobject.py (original) +++ pypy/dist/pypy/objspace/std/strsliceobject.py Sat Sep 20 23:35:38 2008 @@ -141,7 +141,7 @@ def str_endswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -155,7 +155,7 @@ def str_startswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/dist/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/tupleobject.py (original) +++ pypy/dist/pypy/objspace/std/tupleobject.py Sat Sep 20 23:35:38 2008 @@ -3,11 +3,13 @@ from pypy.rlib.rarithmetic import intmask from pypy.objspace.std.sliceobject import W_SliceObject from pypy.interpreter import gateway +from pypy.rlib.debug import make_sure_not_resized class W_TupleObject(W_Object): from pypy.objspace.std.tupletype import tuple_typedef as typedef def __init__(w_self, wrappeditems): + make_sure_not_resized(wrappeditems) w_self.wrappeditems = wrappeditems # a list of wrapped values def __repr__(w_self): @@ -19,9 +21,6 @@ items = [space.unwrap(w_item) for w_item in w_tuple.wrappeditems] # XXX generic mixed types unwrap return tuple(items) - def getitems(self): - return self.wrappeditems - registerimplementation(W_TupleObject) @@ -62,7 +61,7 @@ def iter__Tuple(space, w_tuple): from pypy.objspace.std import iterobject - return iterobject.W_FastSeqIterObject(w_tuple, w_tuple.wrappeditems) + return iterobject.W_FastTupleIterObject(w_tuple, w_tuple.wrappeditems) def add__Tuple_Tuple(space, w_tuple1, w_tuple2): items1 = w_tuple1.wrappeditems Modified: pypy/dist/pypy/objspace/std/tupletype.py ============================================================================== --- pypy/dist/pypy/objspace/std/tupletype.py (original) +++ pypy/dist/pypy/objspace/std/tupletype.py Sat Sep 20 23:35:38 2008 @@ -13,7 +13,7 @@ space.is_w(space.type(w_sequence), space.w_tuple)): return w_sequence else: - tuple_w = space.unpackiterable(w_sequence) + tuple_w = space.viewiterable(w_sequence) w_obj = space.allocate_instance(space.TupleObjectCls, w_tupletype) space.TupleObjectCls.__init__(w_obj, tuple_w) return w_obj Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Sat Sep 20 23:35:38 2008 @@ -545,10 +545,10 @@ if not space.is_w(w_where, space.w_type): w_mro_meth = space.get(w_mro_func, w_self) w_mro = space.call_function(w_mro_meth) - w_self.mro_w = space.unpackiterable(w_mro) + w_self.mro_w = space.viewiterable(w_mro) # do some checking here return # done - w_self.mro_w = w_self.compute_default_mro() + w_self.mro_w = w_self.compute_default_mro()[:] # ____________________________________________________________ Modified: pypy/dist/pypy/objspace/std/typetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/typetype.py (original) +++ pypy/dist/pypy/objspace/std/typetype.py Sat Sep 20 23:35:38 2008 @@ -11,7 +11,7 @@ w_typetype = _precheck_for_new(space, w_typetype) - bases_w = space.unpackiterable(w_bases) + bases_w = space.viewiterable(w_bases) w_winner = w_typetype for base in bases_w: @@ -115,7 +115,7 @@ " to %s.__bases__, not %s"% (w_type.name, space.type(w_value).getname(space, '?')))) - newbases_w = space.unpackiterable(w_value) + newbases_w = space.viewiterable(w_value) if len(newbases_w) == 0: raise OperationError(space.w_TypeError, space.wrap("can only assign non-empty tuple" Modified: pypy/dist/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/unicodeobject.py (original) +++ pypy/dist/pypy/objspace/std/unicodeobject.py Sat Sep 20 23:35:38 2008 @@ -474,7 +474,7 @@ def unicode_startswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.unpacktuple(w_prefixes): + for w_prefix in space.viewiterable(w_prefixes): prefix = space.unicode_w(w_prefix) if _check_startswith_substring(unistr, prefix, start, end): return space.w_True @@ -483,7 +483,7 @@ def unicode_endswith__Unicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.unpacktuple(w_suffixes): + for w_suffix in space.viewiterable(w_suffixes): suffix = space.unicode_w(w_suffix) if _check_endswith_substring(unistr, suffix, start, end): return space.w_True Modified: pypy/dist/pypy/rlib/debug.py ============================================================================== --- pypy/dist/pypy/rlib/debug.py (original) +++ pypy/dist/pypy/rlib/debug.py Sat Sep 20 23:35:38 2008 @@ -64,7 +64,7 @@ callable which checks if annotation is as expected, arguments passed are (current annotation, bookkeeper) """ - pass + return arg class Entry(ExtRegistryEntry): _about_ = check_annotation @@ -80,3 +80,27 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) +def make_sure_not_resized(arg): + """ Function checking whether annotation of SomeList is never resized, + useful for debugging. Does nothing when run directly + """ + return arg + +class Entry(ExtRegistryEntry): + _about_ = make_sure_not_resized + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeList + assert isinstance(s_arg, SomeList) + # the logic behind it is that we try not to propagate + # make_sure_not_resized, when list comprehension is not on + if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: + s_arg.listdef.never_resize() + else: + from pypy.annotation.annrpython import log + log.WARNING('make_sure_not_resized called, but has no effect since list_comprehension is off') + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) Modified: pypy/dist/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_debug.py (original) +++ pypy/dist/pypy/rlib/test/test_debug.py Sat Sep 20 23:35:38 2008 @@ -1,6 +1,6 @@ import py -from pypy.rlib.debug import check_annotation +from pypy.rlib.debug import check_annotation, make_sure_not_resized from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -27,3 +27,12 @@ py.test.raises(Error, "interpret(g, [3])") +def test_make_sure_not_resized(): + from pypy.annotation.listdef import TooLateForChange + def f(): + result = [1,2,3] + make_sure_not_resized(result) + result.append(4) + return len(result) + + py.test.raises(TooLateForChange, interpret, f, []) Modified: pypy/dist/pypy/rpython/callparse.py ============================================================================== --- pypy/dist/pypy/rpython/callparse.py (original) +++ pypy/dist/pypy/rpython/callparse.py Sat Sep 20 23:35:38 2008 @@ -176,7 +176,7 @@ if (expected_length is not None and expected_length != len(items)): raise ValueError - return items + return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" def is_w(self, one, other): From arigo at codespeak.net Sat Sep 20 23:35:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Sep 2008 23:35:53 +0200 (CEST) Subject: [pypy-svn] r58282 - pypy/branch/tuple-nonresizable-395 Message-ID: <20080920213553.74646169EB3@codespeak.net> Author: arigo Date: Sat Sep 20 23:35:52 2008 New Revision: 58282 Removed: pypy/branch/tuple-nonresizable-395/ Log: Remove merged branch. From fijal at codespeak.net Sun Sep 21 13:27:45 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 21 Sep 2008 13:27:45 +0200 (CEST) Subject: [pypy-svn] r58294 - pypy/dist/pypy/objspace/std Message-ID: <20080921112745.E6A65169EA8@codespeak.net> Author: fijal Date: Sun Sep 21 13:27:43 2008 New Revision: 58294 Modified: pypy/dist/pypy/objspace/std/objspace.py Log: Raise correct error here. Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Sun Sep 21 13:27:43 2008 @@ -1,5 +1,5 @@ from pypy.objspace.std.register_all import register_all -from pypy.interpreter.baseobjspace import ObjSpace, Wrappable +from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, UnpackValueError from pypy.interpreter.error import OperationError, debug_print from pypy.interpreter.typedef import get_unique_interplevel_subclass from pypy.interpreter import argument @@ -641,7 +641,7 @@ else: return ObjSpace.unpackiterable(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t def viewiterable(self, w_obj, expected_length=-1): @@ -654,7 +654,7 @@ else: return ObjSpace.viewiterable(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise ValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t def sliceindices(self, w_slice, w_length): From fijal at codespeak.net Sun Sep 21 13:32:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 21 Sep 2008 13:32:20 +0200 (CEST) Subject: [pypy-svn] r58295 - in pypy/dist/pypy: rlib/test rpython/test Message-ID: <20080921113220.0F061169F18@codespeak.net> Author: fijal Date: Sun Sep 21 13:32:19 2008 New Revision: 58295 Modified: pypy/dist/pypy/rlib/test/test_debug.py pypy/dist/pypy/rpython/test/test_llinterp.py Log: Fix the test by passing around additional config options. Allow this. Modified: pypy/dist/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_debug.py (original) +++ pypy/dist/pypy/rlib/test/test_debug.py Sun Sep 21 13:32:19 2008 @@ -35,4 +35,5 @@ result.append(4) return len(result) - py.test.raises(TooLateForChange, interpret, f, []) + py.test.raises(TooLateForChange, interpret, f, [], + list_comprehension_operations=True) Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Sun Sep 21 13:32:19 2008 @@ -111,11 +111,11 @@ def interpret(func, values, view='auto', viewbefore='auto', policy=None, someobjects=False, type_system="lltype", backendopt=False, - config=None, malloc_check=True): + config=None, malloc_check=True, **kwargs): interp, graph = get_interpreter(func, values, view, viewbefore, policy, someobjects, type_system=type_system, backendopt=backendopt, config=config, - malloc_check=malloc_check) + malloc_check=malloc_check, **kwargs) result = interp.eval_graph(graph, values) if malloc_check and interp.mallocs: raise MallocMismatch(interp.mallocs) From fijal at codespeak.net Sun Sep 21 14:02:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 21 Sep 2008 14:02:58 +0200 (CEST) Subject: [pypy-svn] r58296 - in pypy/dist/pypy: config rlib tool translator/c translator/c/test translator/tool translator/tool/test Message-ID: <20080921120258.92821169EF7@codespeak.net> Author: fijal Date: Sun Sep 21 14:02:57 2008 New Revision: 58296 Added: pypy/dist/pypy/rlib/pyplatform.py - copied unchanged from r58295, pypy/branch/cross-compilation/pypy/rlib/pyplatform.py Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/tool/gcc_cache.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/test/test_standalone.py pypy/dist/pypy/translator/tool/cbuild.py pypy/dist/pypy/translator/tool/test/test_cbuild.py Log: Merge cross-compilation branch (without benchmark work) This contains: * A bit separated cbuild logic for rffi_platform and friends * --platform=xxx translation option * support for maemo target platform Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Sun Sep 21 14:02:57 2008 @@ -346,10 +346,19 @@ ] def set_platform(config, platform): - if platform == 'maemo': - from pypy.translator.tool.cbuild import ExternalCompilationInfo - # XXX evil hackery - func_defs = list(ExternalCompilationInfo.__init__.func_defaults) - func_defs[-1] = 'maemo' - ExternalCompilationInfo.__init__.im_func.func_defaults = tuple(func_defs) - + from pypy.rlib.pyplatform import Platform, Maemo, OverloadCompilerPlatform + from pypy.rlib import pyplatform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + if isinstance(platform, str): + if platform == 'maemo': + platform = Maemo() + elif platform == 'host': + return + else: + raise NotImplementedError('Platform = %s' % (platform,)) + assert isinstance(platform, Platform) + pyplatform.platform = platform + if config.translation.cc: + pyplatform.platform = OverloadCompilerPlatform(platform, + config.translation.cc) + Modified: pypy/dist/pypy/tool/gcc_cache.py ============================================================================== --- pypy/dist/pypy/tool/gcc_cache.py (original) +++ pypy/dist/pypy/tool/gcc_cache.py Sun Sep 21 14:02:57 2008 @@ -20,8 +20,7 @@ try: return path.read() except py.error.Error: - result = py.process.cmdexec(eci.get_emulator_for_platform() + - build_executable(c_files, eci)) + result = eci.platform.execute(build_executable(c_files, eci)) path.write(result) return result Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sun Sep 21 14:02:57 2008 @@ -312,7 +312,7 @@ bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) - def getccompiler(self): + def getccompiler(self): cc = self.config.translation.cc # Copy extrafiles to target directory, if needed extrafiles = [] @@ -399,7 +399,9 @@ if self.config.translation.cc: cc = self.config.translation.cc else: - cc = 'gcc' + cc = self.eci.platform.get_compiler() + if cc is None: + cc = 'gcc' make_no_prof = '' if self.has_profopt(): profopt = self.config.translation.profopt Modified: pypy/dist/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/c/test/test_standalone.py Sun Sep 21 14:02:57 2008 @@ -1,5 +1,5 @@ import py -import sys, os +import sys, os, re from pypy.rlib.rarithmetic import r_longlong from pypy.translator.translator import TranslationContext @@ -226,3 +226,30 @@ assert " ll_strtod.h" in makefile assert " ll_strtod.o" in makefile +def test_cross_compilation(): + from pypy.rlib.pyplatform import Platform + from pypy.config.translationoption import set_platform + + class X(Platform): + def get_compiler(self): + return 'x' + + def entry_point(argv): + return 0 + + t = TranslationContext() + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + + set_platform(t.config, X()) + try: + eci = ExternalCompilationInfo(platform=X()) + + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source() + + makefile = udir.join(cbuilder.modulename, 'Makefile').read() + + m = re.search('^CC\s*=\s*x$', makefile) + finally: + set_platform(t.config, Platform()) Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Sun Sep 21 14:02:57 2008 @@ -40,7 +40,7 @@ compile_extra = [], link_extra = [], frameworks = [], - platform = 'host'): + platform = None): """ pre_include_bits: list of pieces of text that should be put at the top of the generated .c files, before any #include. They shouldn't @@ -82,13 +82,15 @@ link to a framework bundle. Not suitable for unix-like .dylib installations. - platform: an unique identifier of compile platform, useful for - caching. + platform: an object that can identify the platform """ for name in self._ATTRIBUTES: value = locals()[name] assert isinstance(value, (list, tuple)) setattr(self, name, tuple(value)) + if platform is None: + from pypy.rlib import pyplatform + platform = pyplatform.platform self.platform = platform def from_compiler_flags(cls, flags): @@ -174,6 +176,7 @@ for attr in self._ATTRIBUTES: val = getattr(self, attr) info.append("%s=%s" % (attr, repr(val))) + info.append("platform=%s" % self.platform.__class__.__name__) return "" % ", ".join(info) def merge(self, *others): @@ -263,24 +266,6 @@ d['separate_module_sources'] = () return ExternalCompilationInfo(**d) - def get_emulator_for_platform(self): - if self.platform == 'host': - return '' - elif self.platform == 'maemo': - # XXX how to do it in better way??? - return '/scratchbox/login ' - else: - raise NotImplementedError("Platform = %s" % (self.platform,)) - - def get_compiler_for_platform(self): - if self.platform == 'host': - return None - elif self.platform == 'maemo': - # XXX this should be settable somehow, not sure exactly how - return '/scratchbox/compilers/cs2005q3.2-glibc-arm/bin/sbox-arm-linux-gcc' - else: - raise NotImplementedError("Platform = %s" % (self.platform,)) - if sys.platform == 'win32': so_ext = '.dll' else: @@ -531,7 +516,7 @@ if compiler_exe is not None: self.compiler_exe = compiler_exe else: - self.compiler_exe = eci.get_compiler_for_platform() + self.compiler_exe = eci.platform.get_compiler() self.profbased = profbased if not sys.platform in ('win32', 'darwin'): # xxx if 'm' not in self.libraries: Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/dist/pypy/translator/tool/test/test_cbuild.py Sun Sep 21 14:02:57 2008 @@ -158,14 +158,50 @@ 'dxowqbncpqympqhe-config') def test_platforms(self): - eci = ExternalCompilationInfo(platform='xxx') + from pypy.rlib.pyplatform import Maemo + eci = ExternalCompilationInfo(platform=Maemo()) eci2 = ExternalCompilationInfo() assert eci != eci2 assert hash(eci) != hash(eci2) + assert repr(eci) != repr(eci2) py.test.raises(Exception, eci2.merge, eci) - assert eci.merge(eci).platform == 'xxx' + assert eci.merge(eci).platform == Maemo() + + def test_platform(self): + from pypy.rlib.pyplatform import Platform + class Expected(Exception): + pass + + class X(Platform): + def get_compiler(self): + raise Expected + + def execute(self): + return 3 + + eci = ExternalCompilationInfo(platform=X()) + try: + build_executable([self.modfile], eci) + except Expected: + pass + else: + py.test.fail("Did not raise") + assert eci.platform.execute() == 3 + + def test_platform_equality(self): + from pypy.rlib.pyplatform import Platform + class X(Platform): + pass + class Y(Platform): + def __init__(self, x): + self.x = x + + assert X() == X() + assert Y(3) == Y(3) + assert Y(2) != Y(3) def test_standalone_maemo(self): + from pypy.rlib.pyplatform import Maemo # XXX skip if there is no scratchbox if not py.path.local('/scratchbox/login').check(): py.test.skip("No scratchbox detected") @@ -183,9 +219,10 @@ if sys.platform == 'win32': py.test.skip("No cross-compilation on windows yet") else: - eci = ExternalCompilationInfo(platform='maemo', + eci = ExternalCompilationInfo(platform=Maemo(), libraries=['m']) output = build_executable([c_file], eci) py.test.raises(py.process.cmdexec.Error, py.process.cmdexec, output) - result = py.process.cmdexec(eci.get_emulator_for_platform() + output) + result = eci.platform.execute(output) assert result.startswith('4.0') + From arigo at codespeak.net Sun Sep 21 23:28:04 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 21 Sep 2008 23:28:04 +0200 (CEST) Subject: [pypy-svn] r58312 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080921212804.13032169EE8@codespeak.net> Author: arigo Date: Sun Sep 21 23:28:03 2008 New Revision: 58312 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: My dates. Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Sun Sep 21 23:28:03 2008 @@ -12,7 +12,7 @@ Antonio Cuni 5-15 possible ??? Holger Krekel 5-13th interested in sharing room Maciej Fijalkowski 5-15 possible ??? -Armin Rigo +Armin Rigo 3-inf private Samuele Pedroni Carl Friedrich Bolz 3rd Oct - ??? my flat Alexander Schremmer 5-15 possible @@ -30,7 +30,6 @@ Maciej Fijalkowski Jakub Gustak Stephan Diehl -Armin Rigo Samuele Pedroni Carl Friedrich Bolz ? ? Alexander Schremmer ? ? From fijal at codespeak.net Mon Sep 22 12:57:46 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 12:57:46 +0200 (CEST) Subject: [pypy-svn] r58314 - pypy/branch/2.5-features/pypy/interpreter Message-ID: <20080922105746.1B89E169E97@codespeak.net> Author: fijal Date: Mon Sep 22 12:57:44 2008 New Revision: 58314 Modified: pypy/branch/2.5-features/pypy/interpreter/pyopcode.py Log: This seems to be needed (and is present on dist). Makes StdObjSpace work again on top of cpy 2.4 Modified: pypy/branch/2.5-features/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/pyopcode.py Mon Sep 22 12:57:44 2008 @@ -740,8 +740,13 @@ w_locals = space.w_None w_modulename = space.wrap(modulename) w_globals = f.w_globals - w_obj = space.call_function(w_import, w_modulename, w_globals, - w_locals, w_fromlist, w_flag) + if w_flag is None: + w_obj = space.call_function(w_import, w_modulename, w_globals, + w_locals, w_fromlist) + else: + w_obj = space.call_function(w_import, w_modulename, w_globals, + w_locals, w_fromlist, w_flag) + f.pushvalue(w_obj) def IMPORT_STAR(f, *ignored): From fijal at codespeak.net Mon Sep 22 13:12:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 13:12:12 +0200 (CEST) Subject: [pypy-svn] r58315 - pypy/branch/2.5-features/pypy/interpreter/test Message-ID: <20080922111212.34E49169E31@codespeak.net> Author: fijal Date: Mon Sep 22 13:12:11 2008 New Revision: 58315 Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Log: Make this file syntactically valid on cpy 2.4 Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Mon Sep 22 13:12:11 2008 @@ -27,11 +27,18 @@ assert [x for x in g] == [1] def test_generator5(self): + import sys + if sys.version_info < (2, 5): + skip("yield as an expression works only on Python >= 2.5") + d = {} + exec """if 1: def f(): v = (yield ) yield v g = f() g.next() + """ in d + d = d['g'] assert g.send(42) == 42 def test_throw1(self): @@ -59,6 +66,11 @@ raises(StopIteration, g.next) def test_throw4(self): + import sys + if sys.version_info < (2, 5): + skip("yield as an expression works only on Python >= 2.5") + d = {} + exec """if 1: def f(): try: yield 1 @@ -66,6 +78,8 @@ except: yield 3 g = f() + """ in d + g = d['g'] assert g.next() == 1 assert g.next() == 2 assert g.throw(NameError("Error")) == 3 From pedronis at codespeak.net Mon Sep 22 14:47:21 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 22 Sep 2008 14:47:21 +0200 (CEST) Subject: [pypy-svn] r58317 - in pypy/branch/pypy-pytrunk/pypy: . tool/pytest Message-ID: <20080922124721.2C487169E83@codespeak.net> Author: pedronis Date: Mon Sep 22 14:47:20 2008 New Revision: 58317 Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py pypy/branch/pypy-pytrunk/pypy/tool/pytest/ (props changed) Log: (iko, pedronis) FileLogSession functionality has been included in py.lib trunk proper Modified: pypy/branch/pypy-pytrunk/pypy/conftest.py ============================================================================== --- pypy/branch/pypy-pytrunk/pypy/conftest.py (original) +++ pypy/branch/pypy-pytrunk/pypy/conftest.py Mon Sep 22 14:47:20 2008 @@ -32,10 +32,6 @@ help="run pexpect tests directly"), ) -# glue experimental filelog session -from pypy.tool.pytest.filelog.session import add_filelog_option, FileLogSession -add_filelog_option() - _SPACECACHE={} def gettestobjspace(name=None, **kwds): From pedronis at codespeak.net Mon Sep 22 15:26:31 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 22 Sep 2008 15:26:31 +0200 (CEST) Subject: [pypy-svn] r58320 - in pypy/build/testrunner: . test test/examples test/examples/normal Message-ID: <20080922132631.1D541169EC6@codespeak.net> Author: pedronis Date: Mon Sep 22 15:26:29 2008 New Revision: 58320 Added: pypy/build/testrunner/ (props changed) pypy/build/testrunner/runner.py (contents, props changed) pypy/build/testrunner/test/ (props changed) pypy/build/testrunner/test/examples/ pypy/build/testrunner/test/examples/normal/ pypy/build/testrunner/test/examples/normal/example.py (contents, props changed) pypy/build/testrunner/test/examples/normal/example_importerror.py (contents, props changed) pypy/build/testrunner/test/examples/normal/failingsetup.py (contents, props changed) pypy/build/testrunner/test/examples/normal/failingsetup_tricky.py (contents, props changed) pypy/build/testrunner/test/test_runner.py (contents, props changed) Log: (iko, pedronis) start working on a per-dir (parallel) test runner Added: pypy/build/testrunner/runner.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/runner.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1,64 @@ +import sys, os, signal +import py +from py.compat import subprocess + + +def run(args, cwd, out): + f = out.open('w') + try: + return subprocess.call(args, cwd=str(cwd), stdout=f, stderr=f) + finally: + f.close() + + +def getsignalname(n): + for name, value in signal.__dict__.items(): + if value == n and name.startswith('SIG'): + return name + return 'signal %d' % (n,) + +def execute_test(cwd, test, out, logfname, interp=None, test_driver=None): + if interp is None: + interp = [os.path.abspath(sys.executable)] + if test_driver is None: + test_driver = [os.path.abspath(os.path.join('py', 'bin', 'py.test'))] + + args = interp+test_driver + args += ['--resultlog=%s' % logfname, test] + + args = map(str, args) + + exitcode = run(args, cwd, out) + return exitcode + + + +def execute_tests(cwd, testdirs, logfile, out, test_driver=None): + sessdir = py.path.local.make_numbered_dir(prefix='usession-testrunner-', keep=4) + c = 0 + failure = False + + for test in testdirs: + basename = py.path.local(test).purebasename + logfname = sessdir.join("%d-%s-pytest-log" % (c, basename)) + one_output = sessdir.join("%d-%s-output" % (c, basename)) + c += 1 + exitcode = execute_test(cwd, test, one_output, logfname, + test_driver=test_driver) + if c > 1: + out.write(79*'_'+'\n') + output = one_output.read() + out.write(output) + if logfname.check(file=1): + logdata = logfname.read() + logfile.write(logdata) + if exitcode: + failure = True + if exitcode != 1: + pass # xxx unexpected exit cases + + return failure + + + + Added: pypy/build/testrunner/test/examples/normal/example.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/test/examples/normal/example.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1,18 @@ + +def test_one(): + assert 1 == 10/10 + +def test_two(): + assert 2 == 3 + +def test_three(): + assert "hello" == "world" + +def test_many(): + for i in range(100): + yield test_one, + +class TestStuff: + + def test_final(self): + crash Added: pypy/build/testrunner/test/examples/normal/example_importerror.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/test/examples/normal/example_importerror.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1 @@ +print 1/0 Added: pypy/build/testrunner/test/examples/normal/failingsetup.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/test/examples/normal/failingsetup.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1,6 @@ + +def setup_module(mod): + raise RuntimeError + +def test_bar(self): + assert True Added: pypy/build/testrunner/test/examples/normal/failingsetup_tricky.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/test/examples/normal/failingsetup_tricky.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1,6 @@ + +def setup_module(mod): + raise RuntimeError + +def test_goo(self): + yield (lambda: None) Added: pypy/build/testrunner/test/test_runner.py ============================================================================== --- (empty file) +++ pypy/build/testrunner/test/test_runner.py Mon Sep 22 15:26:29 2008 @@ -0,0 +1,97 @@ +import py, sys, os, signal, cStringIO + +import runner + + +class TestExecuteTest(object): + + def setup_class(cls): + cls.real_run = (runner.run,) + cls.called = [] + cls.exitcode = [0] + + def fake_run(args, cwd, out): + cls.called = (args, cwd, out) + return cls.exitcode[0] + runner.run = fake_run + + def teardown_class(cls): + runner.run = cls.real_run[0] + + def test_basic(self): + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') + + expected = [os.path.abspath(sys.executable), + os.path.abspath(os.path.join('py', 'bin', 'py.test')), + '--resultlog=LOGFILE', + 'test_one'] + + assert self.called == (expected, '/wd', 'out') + assert res == 0 + + + def test_explicit(self): + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE', + interp=['INTERP', 'IARG'], + test_driver=['driver', 'darg']) + + expected = ['INTERP', 'IARG', + 'driver', 'darg', + '--resultlog=LOGFILE', + 'test_one'] + + assert self.called == (expected, '/wd', 'out') + assert res == 0 + + + def test_error(self): + self.exitcode[:] = [1] + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') + assert res == 1 + + + self.exitcode[:] = [-signal.SIGSEGV] + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') + assert res == -signal.SIGSEGV + + +class TestRunner(object): + + + def setup_class(cls): + cls.udir = py.path.local.make_numbered_dir(prefix='usession-runner-', + keep=3) + cls.udir.join('test_normal').ensure(dir=1) + for p in py.path.local(__file__).dirpath( + 'examples', 'normal').listdir("*.py"): + p.copy(cls.udir.join('test_normal', 'test_'+p.basename)) + + def teardown_class(cls): + pass + + def test_one_dir(self): + test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + + log = cStringIO.StringIO() + out = cStringIO.StringIO() + + res = runner.execute_tests(self.udir, ['test_normal'], log, out, + test_driver=test_driver) + + assert res + + assert out.getvalue() + + log_lines = log.getvalue().splitlines() + + assert log_lines[0] == ". test_normal/test_example.py:test_one" + nfailures = 0 + noutcomes = 0 + for line in log_lines: + if line[0] != ' ': + noutcomes += 1 + if line[0] != '.': + nfailures += 1 + + assert noutcomes == 107 + assert nfailures == 6 From pedronis at codespeak.net Mon Sep 22 16:20:55 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 22 Sep 2008 16:20:55 +0200 (CEST) Subject: [pypy-svn] r58321 - in pypy/build/testrunner: . test Message-ID: <20080922142055.099541684D0@codespeak.net> Author: pedronis Date: Mon Sep 22 16:20:52 2008 New Revision: 58321 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: (iko, pedronis) added first incarnation of testdirs collecting code Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Mon Sep 22 16:20:52 2008 @@ -1,6 +1,6 @@ import sys, os, signal import py -from py.compat import subprocess +from py.compat import subprocess, optparse def run(args, cwd, out): @@ -59,6 +59,63 @@ return failure - - - + +def is_test_py_file(p): + name = p.basename + return name.startswith('test_') and name.endswith('.py') + +def collect_one_testdir(testdirs, root, reldir, tests): + testdirs.append(reldir) + return + +def collect_testdirs(testdirs, p, root=None): + if root is None: + root = p + + reldir = p.relto(root) + entries = [p1 for p1 in p.listdir() if p1.check(dotfile=0)] + + if p != root: + for p1 in entries: + if is_test_py_file(p1): + collect_one_testdir(testdirs, root, reldir, + [t for t in entries if is_test_py_file(t)]) + return + + for p1 in entries: + if p1.check(dir=1, link=0): + collect_testdirs(testdirs, p1, root=root) + + +if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option("--logfile", dest="logfile", default=None, + help="accumulated machine-readable logfile") + parser.add_option("--output", dest="output", default='-', + help="plain test output (default: stdout)") + parser.add_option("--config", dest="config", default=None, + help="configuration python file (optional)") + parser.add_option("--root", dest="root", default=".", + help="root directory for the run") + opts, args = parser.parse_args() + + if opts.logfile is None: + print "no logfile specified" + sys.exit(2) + + logfile = open(opts.logfile, "w") + if opts.output == '-': + out = sys.stdout + else: + out = open(opts.output, "w") + + root = py.path.local(opts.root) + + testdirs = [] + + collect_testdirs(testdirs, root) + + res = execute_tests(root, testdirs, logfile, out) + + if res: + sys.exit(1) Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Mon Sep 22 16:20:52 2008 @@ -69,6 +69,13 @@ def teardown_class(cls): pass + def test_collect_testdirs_simple(self): + res = [] + runner.collect_testdirs(res, self.udir) + + assert res == ['test_normal'] + + def test_one_dir(self): test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] From pedronis at codespeak.net Mon Sep 22 17:51:06 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 22 Sep 2008 17:51:06 +0200 (CEST) Subject: [pypy-svn] r58322 - in pypy/build/testrunner: . test Message-ID: <20080922155106.B9828169EC9@codespeak.net> Author: pedronis Date: Mon Sep 22 17:51:04 2008 New Revision: 58322 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: (iko, pedronis) - support parallel runs - configurability hack through python code overriding the run parameters instance Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Mon Sep 22 17:51:04 2008 @@ -1,4 +1,4 @@ -import sys, os, signal +import sys, os, signal, thread, Queue import py from py.compat import subprocess, optparse @@ -17,12 +17,7 @@ return name return 'signal %d' % (n,) -def execute_test(cwd, test, out, logfname, interp=None, test_driver=None): - if interp is None: - interp = [os.path.abspath(sys.executable)] - if test_driver is None: - test_driver = [os.path.abspath(os.path.join('py', 'bin', 'py.test'))] - +def execute_test(cwd, test, out, logfname, interp, test_driver): args = interp+test_driver args += ['--resultlog=%s' % logfname, test] @@ -32,59 +27,124 @@ return exitcode +def worker(num, n, run_param, testdirs, result_queue): + sessdir = run_param.sessdir + root = run_param.root + test_driver = run_param.test_driver + interp = run_param.interp + # xxx cfg thread start + while 1: + try: + test = testdirs.pop(0) + except IndexError: + result_queue.put(None) # done + return + basename = py.path.local(test).purebasename + logfname = sessdir.join("%d-%s-pytest-log" % (num, basename)) + one_output = sessdir.join("%d-%s-output" % (num, basename)) + num += n -def execute_tests(cwd, testdirs, logfile, out, test_driver=None): - sessdir = py.path.local.make_numbered_dir(prefix='usession-testrunner-', keep=4) - c = 0 - failure = False - - for test in testdirs: - basename = py.path.local(test).purebasename - logfname = sessdir.join("%d-%s-pytest-log" % (c, basename)) - one_output = sessdir.join("%d-%s-output" % (c, basename)) - c += 1 - exitcode = execute_test(cwd, test, one_output, logfname, - test_driver=test_driver) - if c > 1: - out.write(79*'_'+'\n') + exitcode = execute_test(root, test, one_output, logfname, + interp, test_driver) + + # xxx cfg cleanup after testdir + output = one_output.read() - out.write(output) if logfname.check(file=1): logdata = logfname.read() - logfile.write(logdata) + else: + logdata = "" + if exitcode: failure = True if exitcode != 1: pass # xxx unexpected exit cases + else: + failure = False - return failure + result_queue.put((failure, logdata, output)) + + + +def start_workers(n, run_param, testdirs): + result_queue = Queue.Queue() + for i in range(n): + thread.start_new_thread(worker, (i, n, run_param, testdirs, + result_queue)) + return result_queue + + +def execute_tests(run_param, testdirs, logfile, out): + sessdir = py.path.local.make_numbered_dir(prefix='usession-testrunner-', + keep=4) + run_param.sessdir = sessdir + + N = run_param.parallel_runs + failure = False + + result_queue =start_workers(N, run_param, testdirs) + + done = 0 + first_ever = True + while True: + res = result_queue.get() + if res is None: + done += 1 + if done == N: + break + continue + + somefailed, logdata, output = res + failure = failure or somefailed + + if first_ever: + first_ever = False + else: + out.write(79*'_'+'\n') + + out.write(output) + if logdata: + logfile.write(logdata) + return failure -def is_test_py_file(p): - name = p.basename - return name.startswith('test_') and name.endswith('.py') - -def collect_one_testdir(testdirs, root, reldir, tests): - testdirs.append(reldir) - return - -def collect_testdirs(testdirs, p, root=None): - if root is None: - root = p - reldir = p.relto(root) - entries = [p1 for p1 in p.listdir() if p1.check(dotfile=0)] +class RunParam(object): + interp = [os.path.abspath(sys.executable)] + test_driver = [os.path.abspath(os.path.join('py', 'bin', 'py.test'))] + parallel_runs = 1 + + def __init__(self, root): + self.root = root + self.self = self + + def is_test_py_file(self, p): + name = p.basename + return name.startswith('test_') and name.endswith('.py') + + def collect_one_testdir(self, testdirs, reldir, tests): + testdirs.append(reldir) + return + + def collect_testdirs(self, testdirs, p=None): + if p is None: + p = self.root + + reldir = p.relto(self.root) + entries = [p1 for p1 in p.listdir() if p1.check(dotfile=0)] + + if p != self.root: + for p1 in entries: + if self.is_test_py_file(p1): + self.collect_one_testdir(testdirs, reldir, + [t for t in entries + if self.is_test_py_file(t)]) + return - if p != root: for p1 in entries: - if is_test_py_file(p1): - collect_one_testdir(testdirs, root, reldir, - [t for t in entries if is_test_py_file(t)]) - return - - for p1 in entries: - if p1.check(dir=1, link=0): - collect_testdirs(testdirs, p1, root=root) + if p1.check(dir=1, link=0): + self.collect_testdirs(testdirs, p1) + if __name__ == '__main__': @@ -93,10 +153,15 @@ help="accumulated machine-readable logfile") parser.add_option("--output", dest="output", default='-', help="plain test output (default: stdout)") - parser.add_option("--config", dest="config", default=None, + parser.add_option("--config", dest="config", default=[], + action="append", help="configuration python file (optional)") parser.add_option("--root", dest="root", default=".", help="root directory for the run") + parser.add_option("--parallel-runs", dest="parallel_runs", default=0, + type="int", + help="number of parallel test runs") + opts, args = parser.parse_args() if opts.logfile is None: @@ -112,10 +177,22 @@ root = py.path.local(opts.root) testdirs = [] + + + run_param = RunParam(root) + # the config files are python files whose run overrides the content + # of the run_param instance namespace + # in that code function overriding method should not take self + # though a self and self.__class__ are available if needed + for config_py_file in opts.config: + execfile(config_py_file, run_param.__dict__) - collect_testdirs(testdirs, root) + run_param.collect_testdirs(testdirs) - res = execute_tests(root, testdirs, logfile, out) + if opts.parallel_runs: + run_param.parallel_runs = opts.parallel_runs + + res = execute_tests(run_param, testdirs, logfile, out) if res: sys.exit(1) Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Mon Sep 22 17:51:04 2008 @@ -18,18 +18,6 @@ def teardown_class(cls): runner.run = cls.real_run[0] - def test_basic(self): - res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') - - expected = [os.path.abspath(sys.executable), - os.path.abspath(os.path.join('py', 'bin', 'py.test')), - '--resultlog=LOGFILE', - 'test_one'] - - assert self.called == (expected, '/wd', 'out') - assert res == 0 - - def test_explicit(self): res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE', interp=['INTERP', 'IARG'], @@ -46,12 +34,16 @@ def test_error(self): self.exitcode[:] = [1] - res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE', + interp=['INTERP', 'IARG'], + test_driver=['driver', 'darg']) assert res == 1 self.exitcode[:] = [-signal.SIGSEGV] - res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE') + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE', + interp=['INTERP', 'IARG'], + test_driver=['driver', 'darg']) assert res == -signal.SIGSEGV @@ -71,7 +63,9 @@ def test_collect_testdirs_simple(self): res = [] - runner.collect_testdirs(res, self.udir) + run_param = runner.RunParam(self.udir) + + run_param.collect_testdirs(res) assert res == ['test_normal'] @@ -81,9 +75,12 @@ log = cStringIO.StringIO() out = cStringIO.StringIO() + + param = runner.RunParam(self.udir) + param.test_driver = test_driver + param.parallel_runs = 3 - res = runner.execute_tests(self.udir, ['test_normal'], log, out, - test_driver=test_driver) + res = runner.execute_tests(param, ['test_normal'], log, out) assert res @@ -102,3 +99,7 @@ assert noutcomes == 107 assert nfailures == 6 + + # xxx test with more than one dir + + # xxx test for main logic From pedronis at codespeak.net Mon Sep 22 17:52:44 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 22 Sep 2008 17:52:44 +0200 (CEST) Subject: [pypy-svn] r58323 - pypy/build/bot2 Message-ID: <20080922155244.7EFC4169EE7@codespeak.net> Author: pedronis Date: Mon Sep 22 17:52:44 2008 New Revision: 58323 Modified: pypy/build/bot2/TODO Log: status Modified: pypy/build/bot2/TODO ============================================================================== --- pypy/build/bot2/TODO (original) +++ pypy/build/bot2/TODO Mon Sep 22 17:52:44 2008 @@ -1,6 +1,6 @@ - runner for running tests per directory and optionally in parallel (this should be inspired by the current autotest.py running strategy minus the htmlconftest - details) + details) [IN-PROGRESS] - buildbot Nightly scheduler that can pick a uniform revision server side and with support for specifying possibly branches From fijal at codespeak.net Mon Sep 22 18:08:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 18:08:47 +0200 (CEST) Subject: [pypy-svn] r58324 - pypy/branch/2.5-features/pypy/tool/pytest Message-ID: <20080922160847.0A182169E82@codespeak.net> Author: fijal Date: Mon Sep 22 18:08:46 2008 New Revision: 58324 Modified: pypy/branch/2.5-features/pypy/tool/pytest/confpath.py Log: Run 2.5 tests instead of 2.4.1 Modified: pypy/branch/2.5-features/pypy/tool/pytest/confpath.py ============================================================================== --- pypy/branch/2.5-features/pypy/tool/pytest/confpath.py (original) +++ pypy/branch/2.5-features/pypy/tool/pytest/confpath.py Mon Sep 22 18:08:46 2008 @@ -7,5 +7,5 @@ testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) libpythondir = distdir.join('lib-python') -regrtestdir = libpythondir.join('2.4.1', 'test') -modregrtestdir = libpythondir.join('modified-2.4.1', 'test') +regrtestdir = libpythondir.join('2.5.1', 'test') +modregrtestdir = libpythondir.join('modified-2.5.1', 'test') From fijal at codespeak.net Mon Sep 22 20:04:10 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 20:04:10 +0200 (CEST) Subject: [pypy-svn] r58330 - pypy/branch/2.5-features/pypy/interpreter/test Message-ID: <20080922180410.AE296169F0A@codespeak.net> Author: fijal Date: Mon Sep 22 20:04:09 2008 New Revision: 58330 Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Log: This checks make no sense any more, since sys.version_info is always 2.5 Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Mon Sep 22 20:04:09 2008 @@ -27,9 +27,6 @@ assert [x for x in g] == [1] def test_generator5(self): - import sys - if sys.version_info < (2, 5): - skip("yield as an expression works only on Python >= 2.5") d = {} exec """if 1: def f(): @@ -66,9 +63,6 @@ raises(StopIteration, g.next) def test_throw4(self): - import sys - if sys.version_info < (2, 5): - skip("yield as an expression works only on Python >= 2.5") d = {} exec """if 1: def f(): @@ -183,16 +177,10 @@ raises(ValueError, me.next) def test_generator_expression(self): - import sys - if sys.version_info < (2, 4): - skip("generator expressions only work on Python >= 2.4") exec "res = sum(i*i for i in range(5))" assert res == 30 def test_generator_expression_2(self): - import sys - if sys.version_info < (2, 4): - skip("generator expressions only work on Python >= 2.4") d = {} exec """ def f(): From hpk at codespeak.net Mon Sep 22 20:35:57 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 20:35:57 +0200 (CEST) Subject: [pypy-svn] r58331 - in pypy/build/benchmem: . testing Message-ID: <20080922183557.76E99169E7A@codespeak.net> Author: hpk Date: Mon Sep 22 20:35:55 2008 New Revision: 58331 Added: pypy/build/benchmem/report.py - copied, changed from r58280, pypy/build/benchmem/benchreport.py pypy/build/benchmem/runbench.py - copied, changed from r58280, pypy/build/benchmem/benchtool.py Removed: pypy/build/benchmem/benchreport.py pypy/build/benchmem/benchtool.py Modified: pypy/build/benchmem/testing/test_benchtool.py Log: tweak things until measuring multiple interpreters and some ascii-reporting works. The way i am using it currently: python runbench.py -e "python2.5,pypy-c--optmem,pypy-c--opt3" && python report.py Copied: pypy/build/benchmem/report.py (from r58280, pypy/build/benchmem/benchreport.py) ============================================================================== --- pypy/build/benchmem/benchreport.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 20:35:55 2008 @@ -1,18 +1,63 @@ import py import smaps, benchtool +def asciitable(table): + colmax = [] + for row in table: + if not colmax: + colmax = [len(str(x)) for x in row] + else: + colmax = [max(len(str(x)),y) for x,y in zip(row, colmax)] + lines = [] + for row in table: + line = [] + for col,width in zip(row, colmax): + line.append("%%-%ds" %(width) %(col)) + lines.append(" ".join(line)) + return "\n".join(lines) + +def maxtable_overview(reader): + tw = py.io.TerminalWriter() + + for name, results in reader.name2results.items(): + tw.sep("=", name) + row0 = "executable maxpdirty maxrss".split() + rows = [row0] + for result in results: + rows.append([result.executable, + result.max("private_dirty"), + result.max("rss"), + ]) + tw.line(asciitable(rows)) + +def checkpointdetails(reader): + tw = py.io.TerminalWriter() + for name, results in reader.name2results.items(): + tw.sep("=", "private_dirty at checkpoints: %s" %(name,)) + row0 = ["num"] + [result.executable for result in results] + + numsnapshosts = len(results[0].snapshots) + rows = [row0] + for i in range(numsnapshosts): + row = [i] + for result in results: + row.append(result.snapshots[i].private_dirty) + rows.append(row) + tw.line(asciitable(rows)) + if __name__ == "__main__": benchlog = py.path.local("bench.log") reader = benchtool.LogReader() reader.parse_logfile(benchlog) - tw = py.io.TerminalWriter() + maxtable_overview(reader) + checkpointdetails(reader) - for name, results in reader.name2results.items(): - tw.sep("=", name) - for result in results: - print "%-30s max dirty = %s + %s" % ( - result.executable, - result.max("shared_dirty"), - result.max("private_dirty") - ) + #for name, results in reader.name2results.items(): + # tw.sep("=", name) + # for result in results: + # print "%-30s max dirty = %s + %s" % ( + # result.executable, + # result.max("shared_dirty"), + # result.max("private_dirty") + ### ) Copied: pypy/build/benchmem/runbench.py (from r58280, pypy/build/benchmem/benchtool.py) ============================================================================== --- pypy/build/benchmem/benchtool.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 20:35:55 2008 @@ -44,11 +44,12 @@ write("c") sys.stdin.read(1) if __name__ == "__main__": - import os, sys + import os, sys, gc pid = os.getpid() write(str(pid) + "\\n") checkpoint() %s(checkpoint, %s) + gc.collect() # XXX checkpoint() write("F") sys.stdin.close() @@ -57,7 +58,7 @@ p.write(source) return p - def writeexecinfo(self, benchname, args): + def write_benchheader(self, benchname, args): print >>self.logstream, self.SEPBENCH print >>self.logstream, "#executable=%r" %(str(self.executable ),) print >>self.logstream, "#benchname=%r" %(benchname,) @@ -71,7 +72,7 @@ stdout, stdin = os.popen2(cmd) pid = int(stdin.readline()) - self.writeexecinfo(benchpyfile.basename, args) + self.write_benchheader(benchpyfile.basename, args) rec = smaps.SmapsRecorder(pid, self.logstream) self.interact_with_child_checkpoints(rec, stdout, stdin) @@ -102,7 +103,7 @@ def parse_logfile(self, logpath): f = logpath.open() for result in BenchmarkResult.parse(f): - print "parsed", result + #print "parsed", result l = self.name2results.setdefault(result.benchname, []) l.append(result) f.close() @@ -177,15 +178,6 @@ parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", help="logfile for recording benchmark measurements") -def getexecutable(options): - executable = options.executable - if os.sep not in executable: - executable = py.path.local.sysfind("python2.5") - assert executable, "%s not found" %(executable) - executable = py.path.local(executable) - assert executable.check(), "%s does not exist" % executable - return executable - def getbenchfiles(options, args): if args: benchfiles = [py.path.local(x) for x in args] @@ -207,11 +199,17 @@ if __name__ == '__main__': (options, args) = parser.parse_args() - executable = getexecutable(options) names = getbenchfiles(options, args) benchlog = getbenchlog(options) - - runner = BenchRunner(executable, benchlog) - for name in names: - runner.run_checkpointed_bench(name, (100, 1000)) + for executable in options.executable.split(","): + if not executable: + continue + p = py.path.local(executable) + if not p.check(): + p = py.path.local.sysfind(executable) + if not p.check(): + raise SystemExit("could not find %r"% (executable)) + runner = BenchRunner(executable, benchlog) + for name in names: + runner.run_checkpointed_bench(name, (100, 1000)) print "bench results append to -->>>", benchlog Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Mon Sep 22 20:35:55 2008 @@ -1,7 +1,7 @@ import py import os, sys -import benchtool +import runbench import smaps def setup_module(mod): @@ -28,10 +28,10 @@ def checker(path, *args): if log.check(): log.remove() - runner = benchtool.BenchRunner(executable="python2.5", benchlog=log) + runner = runbench.BenchRunner(executable="python2.5", benchlog=log) runner.run_checkpointed_bench(path, args) assert log.check() - benchresult = benchtool.LogReader() + benchresult = runbench.LogReader() benchresult.parse_logfile(log) #assert reader.executable #assert reader.executable @@ -40,7 +40,7 @@ assert len(results) == 1 assert len(results[0].snapshots) == 10 + 2 - for path in benchtool.benchmarkdir.listdir("*.py"): + for path in runbench.benchmarkdir.listdir("*.py"): if path.basename[0] != "_": yield checker, path, 10, 10 @@ -67,6 +67,27 @@ assert l0 == "16,8,0,0,0,8 402c2000-402c4000 rw-p 402c2000 00:00 0" assert lines[1] == smaps.SmapsRecorder.SEPSNAPSHOT.strip() + py.test.skip("implement denser/delta format sometime") + rec.snapshot() # delta format does not produce anything for identical snapshots + result2 = io.getvalue() + assert result == result2 + + p.write(py.std.textwrap.dedent("""\ + 402c2000-402c4000 rw-p 402c2000 00:00 0 + Size: 32 kB + Rss: 32 kB + Shared_Clean: 0 kB + Shared_Dirty: 0 kB + Private_Clean: 0 kB + Private_Dirty: 32 kB""")) + + rec.snapshot() + result = io.getvalue() + lines = result.split("\n") + assert lines[1].strip() == "CHECKPOINT 2" + l2 = " ".join(lines[2].split()) + assert l2 == "32,32,0,0,0,32 402c2000-402c4000 rw-p 402c2000 00:00 0" + def test_parse_mapping(): line = ("988,796,0,0,796,0 08048000-0813f000 " "r-xp 00000000 fd:00 75457 sometext") @@ -86,11 +107,10 @@ def test_summing(): line = ("988,796,0,0,796,0 08048000-0813f000 " "r-xp 00000000 fd:00 75457 sometext") - snap = benchtool.Snapshot([smaps.Mapping(line), smaps.Mapping(line)]) + snap = runbench.Snapshot([smaps.Mapping(line), smaps.Mapping(line)]) for name in ('size rss shared_clean shared_dirty ' 'private_clean private_dirty').split(): sumval = getattr(snap, name) val1 = getattr(snap.mappings[0], name) val2 = getattr(snap.mappings[1], name) assert sumval == val1 + val2 - From hpk at codespeak.net Mon Sep 22 20:45:54 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 20:45:54 +0200 (CEST) Subject: [pypy-svn] r58333 - pypy/build/benchmem Message-ID: <20080922184554.07BBE169F09@codespeak.net> Author: hpk Date: Mon Sep 22 20:45:54 2008 New Revision: 58333 Modified: pypy/build/benchmem/report.py pypy/build/benchmem/runbench.py Log: add module docstrings for the two scripts Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 20:45:54 2008 @@ -1,3 +1,10 @@ +""" + report.py + + read bench.log file and produce some overviews. + xxx extend, introduce options + +""" import py import smaps, benchtool Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 20:45:54 2008 @@ -1,8 +1,9 @@ #!/usr/bin/env python """ - benchtool.py [options] [benchname1.py] [benchname2.py] + runbench.py [options] [benchname1.py] [benchname2.py] - record memory usage for given benchmarks (or all if none specified). + measure memory usage for given benchmarks (or all if none specified). + and append results to a log file ("bench.log" by default). """ import py From hpk at codespeak.net Mon Sep 22 20:46:53 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 20:46:53 +0200 (CEST) Subject: [pypy-svn] r58334 - pypy/build/benchmem Message-ID: <20080922184653.3E8BB169F0C@codespeak.net> Author: hpk Date: Mon Sep 22 20:46:52 2008 New Revision: 58334 Modified: pypy/build/benchmem/report.py Log: rename here as well Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 20:46:52 2008 @@ -6,7 +6,7 @@ """ import py -import smaps, benchtool +import smaps, runbench def asciitable(table): colmax = [] @@ -54,7 +54,7 @@ if __name__ == "__main__": benchlog = py.path.local("bench.log") - reader = benchtool.LogReader() + reader = runbench.LogReader() reader.parse_logfile(benchlog) maxtable_overview(reader) From hpk at codespeak.net Mon Sep 22 20:50:07 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 20:50:07 +0200 (CEST) Subject: [pypy-svn] r58335 - pypy/build/benchmem Message-ID: <20080922185007.20534169E32@codespeak.net> Author: hpk Date: Mon Sep 22 20:50:06 2008 New Revision: 58335 Modified: pypy/build/benchmem/report.py (contents, props changed) pypy/build/benchmem/smaps.py (props changed) Log: FIXEOL + svn:executable Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 20:50:06 2008 @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ report.py From hpk at codespeak.net Mon Sep 22 20:51:11 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 20:51:11 +0200 (CEST) Subject: [pypy-svn] r58336 - pypy/build/benchmem Message-ID: <20080922185111.E66F0169E51@codespeak.net> Author: hpk Date: Mon Sep 22 20:51:11 2008 New Revision: 58336 Modified: pypy/build/benchmem/runbench.py (props changed) Log: svn:executable also on this From hpk at codespeak.net Mon Sep 22 21:16:56 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 21:16:56 +0200 (CEST) Subject: [pypy-svn] r58338 - pypy/build/benchmem/benchmark Message-ID: <20080922191656.29E58169EC6@codespeak.net> Author: hpk Date: Mon Sep 22 21:16:54 2008 New Revision: 58338 Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py Log: remove unnecessary multiple invocations here. Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py ============================================================================== --- pypy/build/benchmem/benchmark/create_recursive_tuples.py (original) +++ pypy/build/benchmem/benchmark/create_recursive_tuples.py Mon Sep 22 21:16:54 2008 @@ -5,11 +5,4 @@ checkpoint() for j in range(iter2): x = (x,) - x = (x,) - x = (x,) - x = (x,) - x = (x,) - x = (x,) - x = (x,) - x = (x,) From hpk at codespeak.net Mon Sep 22 21:58:07 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 21:58:07 +0200 (CEST) Subject: [pypy-svn] r58342 - in pypy/build/benchmem: . benchmark testing Message-ID: <20080922195807.52EA8169E36@codespeak.net> Author: hpk Date: Mon Sep 22 21:58:06 2008 New Revision: 58342 Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py pypy/build/benchmem/benchmark/list_of_instances_with_ints.py pypy/build/benchmem/benchmark/simple_linked_instances.py pypy/build/benchmem/runbench.py pypy/build/benchmem/testing/test_benchtool.py Log: simplify current benchmarks which mostly measure the size of certain objects. Modified: pypy/build/benchmem/benchmark/create_recursive_tuples.py ============================================================================== --- pypy/build/benchmem/benchmark/create_recursive_tuples.py (original) +++ pypy/build/benchmem/benchmark/create_recursive_tuples.py Mon Sep 22 21:58:06 2008 @@ -1,8 +1,5 @@ - -def bench_create_recursive_tuples(checkpoint, iter1, iter2): +def bench_create_recursive_tuples(checkpoint, iter1): x = () for i in range(iter1): - checkpoint() - for j in range(iter2): - x = (x,) - + x = (x,) + checkpoint(collect=True) Modified: pypy/build/benchmem/benchmark/list_of_instances_with_ints.py ============================================================================== --- pypy/build/benchmem/benchmark/list_of_instances_with_ints.py (original) +++ pypy/build/benchmem/benchmark/list_of_instances_with_ints.py Mon Sep 22 21:58:06 2008 @@ -1,12 +1,9 @@ - class A(object): - def __init__(self, x, y): + def __init__(self, x): self.x = x - self.y = y -def bench_list_of_instances_with_ints(checkpoint, iter1, iter2): +def bench_list_of_instances_with_int(checkpoint, iter1): l = [] for i in range(iter1): - checkpoint() - for j in range(iter2): - l.append(A(i, j)) + l.append(A(i)) + checkpoint(collect=True) Modified: pypy/build/benchmem/benchmark/simple_linked_instances.py ============================================================================== --- pypy/build/benchmem/benchmark/simple_linked_instances.py (original) +++ pypy/build/benchmem/benchmark/simple_linked_instances.py Mon Sep 22 21:58:06 2008 @@ -2,9 +2,8 @@ def __init__(self, other): self.other = other -def bench_linked_list(checkpoint, iter1, iter2): +def bench_linked_instances(checkpoint, iter1): x = None for i in range(iter1): - checkpoint() - for j in range(iter2): - x = A(x) + x = A(x) + checkpoint(collect=True) Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 21:58:06 2008 @@ -36,12 +36,15 @@ arglist = ",".join(map(str, args)) source = py.code.Source(path.read(), """ + import gc def write(c): sys.stdout.write(c) sys.stdout.flush() - def checkpoint(): + def checkpoint(collect=False): + if collect: + gc.collect() write("c") sys.stdin.read(1) if __name__ == "__main__": @@ -212,5 +215,5 @@ raise SystemExit("could not find %r"% (executable)) runner = BenchRunner(executable, benchlog) for name in names: - runner.run_checkpointed_bench(name, (100, 1000)) + runner.run_checkpointed_bench(name, (100000,)) print "bench results append to -->>>", benchlog Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Mon Sep 22 21:58:06 2008 @@ -38,11 +38,11 @@ assert len(benchresult.name2results) == 1 results = benchresult.name2results.values()[0] assert len(results) == 1 - assert len(results[0].snapshots) == 10 + 2 + assert len(results[0].snapshots) == 2 + 1 for path in runbench.benchmarkdir.listdir("*.py"): if path.basename[0] != "_": - yield checker, path, 10, 10 + yield checker, path, 10 def test_log_mapping(): s = py.std.textwrap.dedent("""\ From fijal at codespeak.net Mon Sep 22 22:00:02 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 22:00:02 +0200 (CEST) Subject: [pypy-svn] r58343 - pypy/branch/2.5-features/lib-python Message-ID: <20080922200002.99334169E3E@codespeak.net> Author: fijal Date: Mon Sep 22 22:00:01 2008 New Revision: 58343 Modified: pypy/branch/2.5-features/lib-python/conftest.py Log: More yak shaving. Adapt places where py.test calls were vastly outdated. Modified: pypy/branch/2.5-features/lib-python/conftest.py ============================================================================== --- pypy/branch/2.5-features/lib-python/conftest.py (original) +++ pypy/branch/2.5-features/lib-python/conftest.py Mon Sep 22 22:00:01 2008 @@ -68,7 +68,8 @@ return seconds return float(timeout) -def callex(space, func, *args, **kwargs): +def callex(space, func, *args, **kwargs): + from py.__.test.outcome import Failed try: return func(*args, **kwargs) except OperationError, e: @@ -79,8 +80,8 @@ if appexcinfo.traceback: print "appexcinfo.traceback:" py.std.pprint.pprint(appexcinfo.traceback) - raise py.test.collect.Item.Failed(excinfo=appexcinfo) - raise py.test.collect.Item.Failed(excinfo=ilevelinfo) + raise Failed(excinfo=appexcinfo) + raise Failed(excinfo=ilevelinfo) # # compliance modules where we invoke test_main() usually call into @@ -425,17 +426,18 @@ RegrTest('test_class.py', enabled=True, oldstyle=True, core=True), RegrTest('test_cmath.py', enabled=True, dumbtest=1, core=True), RegrTest('test_codeccallbacks.py', enabled=True, core=True), - RegrTest('test_codecencodings_cn.py', enabled=False), - RegrTest('test_codecencodings_hk.py', enabled=False), - RegrTest('test_codecencodings_jp.py', enabled=False), - RegrTest('test_codecencodings_kr.py', enabled=False), - RegrTest('test_codecencodings_tw.py', enabled=False), - - RegrTest('test_codecmaps_cn.py', enabled=False), - RegrTest('test_codecmaps_hk.py', enabled=False), - RegrTest('test_codecmaps_jp.py', enabled=False), - RegrTest('test_codecmaps_kr.py', enabled=False), - RegrTest('test_codecmaps_tw.py', enabled=False), + RegrTest('test_coding.py', enabled=True), + RegrTest('test_codecencodings_cn.py', enabled=True), + RegrTest('test_codecencodings_hk.py', enabled=True), + RegrTest('test_codecencodings_jp.py', enabled=True), + RegrTest('test_codecencodings_kr.py', enabled=True), + RegrTest('test_codecencodings_tw.py', enabled=True), + + RegrTest('test_codecmaps_cn.py', enabled=True), + RegrTest('test_codecmaps_hk.py', enabled=True), + RegrTest('test_codecmaps_jp.py', enabled=True), + RegrTest('test_codecmaps_kr.py', enabled=True), + RegrTest('test_codecmaps_tw.py', enabled=True), RegrTest('test_codecs.py', enabled=True, core=True), RegrTest('test_codeop.py', enabled=True, core=True), RegrTest('test_coercion.py', enabled=True, oldstyle=True, core=True), From fijal at codespeak.net Mon Sep 22 22:03:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 22:03:12 +0200 (CEST) Subject: [pypy-svn] r58344 - pypy/branch/cross-compilation/pypy/translator/benchmark Message-ID: <20080922200312.3D0AD169E36@codespeak.net> Author: fijal Date: Mon Sep 22 22:03:11 2008 New Revision: 58344 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Log: Bah, forgotten checkin Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/bench_mem.py Mon Sep 22 22:03:11 2008 @@ -10,9 +10,7 @@ import py import time import re -import signal import sys -import socket class Result(object): def __init__(self, priv_map, shared_map, starttime=None): @@ -57,12 +55,14 @@ realos = os def __init__(self, name, args): + import signal self.pid = -1 signal.signal(signal.SIGCHLD, lambda a,b: self.close()) self.pid = run_child(name, args) self.results = [] def loop(self, logfile, interval): + import signal if isinstance(logfile, basestring): logfile = open(logfile, 'w') counter = 0 Modified: pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py ============================================================================== --- pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py (original) +++ pypy/branch/cross-compilation/pypy/translator/benchmark/microbench_mem.py Mon Sep 22 22:03:11 2008 @@ -8,7 +8,7 @@ def measure_func(num, pid): res = smaps_measure_func(pid) - print num, res.private + print 2**num, res.private def tuples(read, write, coeff): import gc @@ -48,12 +48,52 @@ read() write('e') +def linked_list_with_floats(read, write, coeff): + import gc + + class A(object): + def __init__(self, other, i): + self.i = float(i) + self.other = other + + x = None + for i in range(1000 * coeff): + x = A(x, i) + if i % 1000 == 0: + gc.collect() + gc.collect() + write('x') + read() + write('e') + +def empty_instances(read, write, coeff): + import gc + class A(object): + pass + + x = [A() for i in range(1000*coeff)] + gc.collect() + write('x') + read() + write('e') + +def lists(read, write, coeff): + import gc + x = [] + for i in range(1000 * coeff): + x = [x] + gc.collect() + write('x') + read() + write('e') + if __name__ == '__main__': coeff = 1 i = 0 funcs = [] - while i < 10: - funcs.append(lambda r, w, coeff=coeff: list_of_instances_with_int(r, w, coeff)) + while i < 9: + funcs.append(lambda r, w, coeff=coeff: + linked_list_with_floats(r, w, coeff)) coeff *= 2 i += 1 res = measure(measure_func, funcs) From fijal at codespeak.net Mon Sep 22 22:04:53 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 22:04:53 +0200 (CEST) Subject: [pypy-svn] r58345 - in pypy/branch/2.5-features/lib-python: 2.5.1 2.5.1/bsddb 2.5.1/bsddb/test 2.5.1/compiler 2.5.1/ctypes 2.5.1/ctypes/macholib 2.5.1/ctypes/test 2.5.1/curses 2.5.1/distutils 2.5.1/distutils/command 2.5.1/distutils/tests 2.5.1/email 2.5.1/email/mime 2.5.1/email/test 2.5.1/email/test/data 2.5.1/encodings 2.5.1/hotshot 2.5.1/idlelib 2.5.1/lib-tk 2.5.1/logging 2.5.1/msilib 2.5.1/plat-aix3 2.5.1/plat-aix4 2.5.1/plat-atheos 2.5.1/plat-beos5 2.5.1/plat-darwin 2.5.1/plat-freebsd2 2.5.1/plat-freebsd3 2.5.1/plat-freebsd4 2.5.1/plat-freebsd5 2.5.1/plat-freebsd6 2.5.1/plat-freebsd7 2.5.1/plat-irix5 2.5.1/plat-irix6 2.5.1/plat-linux2 2.5.1/plat-mac 2.5.1/plat-mac/Carbon 2.5.1/plat-mac/lib-scriptpackages/CodeWarrior 2.5.1/plat-mac/lib-scriptpackages/Explorer 2.5.1/plat-mac/lib-scriptpackages/Finder 2.5.1/plat-mac/lib-scriptpackages/Netscape 2.5.1/plat-mac/lib-scriptpackages/StdSuites 2.5.1/plat-mac/lib-scriptpackages/SystemEvents 2.5.1/plat-mac/lib-scriptpackages/Terminal 2.5.1/plat-mac/lib-scriptpackages/_builtinSuites 2.5.1/plat-netbsd1 2.5.1/plat-os2emx 2.5.1/plat-riscos 2.5.1/plat-sunos5 2.5.1/plat-unixware7 2.5.1/sqlite3 2.5.1/sqlite3/test 2.5.1/test 2.5.1/test/crashers 2.5.1/test/leakers 2.5.1/wsgiref 2.5.1/xml 2.5.1/xml/dom 2.5.1/xml/etree 2.5.1/xml/parsers 2.5.1/xml/sax modified-2.5.1 modified-2.5.1/encodings modified-2.5.1/test Message-ID: <20080922200453.523FA169E36@codespeak.net> Author: fijal Date: Mon Sep 22 22:04:52 2008 New Revision: 58345 Modified: pypy/branch/2.5-features/lib-python/2.5.1/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/bsddb/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/bsddb/test/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/compiler/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/ctypes/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/ctypes/macholib/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/ctypes/test/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/curses/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/distutils/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/distutils/command/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/distutils/tests/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/email/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/email/mime/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/email/test/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/email/test/data/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/encodings/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/hotshot/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/idlelib/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/lib-tk/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/logging/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/msilib/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-aix3/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-aix4/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-atheos/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-beos5/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-darwin/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd2/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd3/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd4/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd5/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd6/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-freebsd7/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-irix5/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-irix6/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-linux2/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/Carbon/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/CodeWarrior/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/Explorer/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/Finder/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/Netscape/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/StdSuites/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/SystemEvents/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/Terminal/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-mac/lib-scriptpackages/_builtinSuites/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-netbsd1/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-os2emx/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-riscos/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-sunos5/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/plat-unixware7/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/sqlite3/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/sqlite3/test/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/test/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/test/crashers/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/test/leakers/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/wsgiref/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/xml/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/xml/dom/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/xml/etree/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/xml/parsers/ (props changed) pypy/branch/2.5-features/lib-python/2.5.1/xml/sax/ (props changed) pypy/branch/2.5-features/lib-python/modified-2.5.1/ (props changed) pypy/branch/2.5-features/lib-python/modified-2.5.1/encodings/ (props changed) pypy/branch/2.5-features/lib-python/modified-2.5.1/test/ (props changed) Log: fixeol From hpk at codespeak.net Mon Sep 22 22:13:21 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 22:13:21 +0200 (CEST) Subject: [pypy-svn] r58346 - pypy/build/benchmem Message-ID: <20080922201321.55BDA169E2A@codespeak.net> Author: hpk Date: Mon Sep 22 22:13:20 2008 New Revision: 58346 Modified: pypy/build/benchmem/runbench.py Log: remove benchlog by default, use "--append" to add to it Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 22:13:20 2008 @@ -181,6 +181,8 @@ help="Python executable to use for recording benchmarks") parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", help="logfile for recording benchmark measurements") +parser.add_option("-a", "--append", action="store_true", dest="append", default="bench.log", + help="append to logfile") def getbenchfiles(options, args): if args: @@ -199,12 +201,15 @@ if benchlog is None: benchlog = "bench.log" return py.path.local(benchlog) + if __name__ == '__main__': (options, args) = parser.parse_args() names = getbenchfiles(options, args) benchlog = getbenchlog(options) + if not options.append and benchlog.check(): + benchlog.remove() for executable in options.executable.split(","): if not executable: continue From hpk at codespeak.net Mon Sep 22 22:16:07 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 22:16:07 +0200 (CEST) Subject: [pypy-svn] r58347 - pypy/build/benchmem Message-ID: <20080922201607.B0B07169E36@codespeak.net> Author: hpk Date: Mon Sep 22 22:16:07 2008 New Revision: 58347 Modified: pypy/build/benchmem/runbench.py Log: set correct default Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 22:16:07 2008 @@ -181,7 +181,7 @@ help="Python executable to use for recording benchmarks") parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", help="logfile for recording benchmark measurements") -parser.add_option("-a", "--append", action="store_true", dest="append", default="bench.log", +parser.add_option("-a", "--append", action="store_true", dest="append", default=False, help="append to logfile") def getbenchfiles(options, args): From fijal at codespeak.net Mon Sep 22 22:16:49 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 22:16:49 +0200 (CEST) Subject: [pypy-svn] r58348 - pypy/build/benchmem Message-ID: <20080922201649.88C20169E36@codespeak.net> Author: fijal Date: Mon Sep 22 22:16:49 2008 New Revision: 58348 Modified: pypy/build/benchmem/report.py pypy/build/benchmem/runbench.py Log: Use private instead of private dirty. Most of the time this stays the same, but in case something get swapped out, we really want to measure total memory. Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 22:16:49 2008 @@ -29,11 +29,11 @@ for name, results in reader.name2results.items(): tw.sep("=", name) - row0 = "executable maxpdirty maxrss".split() + row0 = "executable maxprivate maxrss".split() rows = [row0] for result in results: rows.append([result.executable, - result.max("private_dirty"), + result.max("private"), result.max("rss"), ]) tw.line(asciitable(rows)) @@ -49,7 +49,7 @@ for i in range(numsnapshosts): row = [i] for result in results: - row.append(result.snapshots[i].private_dirty) + row.append(result.snapshots[i].private) rows.append(row) tw.line(asciitable(rows)) Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 22:16:49 2008 @@ -168,9 +168,11 @@ self.mappings = mappings for name in smaps.Mapping._attrnames: setattr(self, name, sum([getattr(x, name) for x in mappings])) + self.private = self.private_dirty + self.private_clean + self.shared = self.shared_dirty + self.shared_clean def memusage(self): - return "privdirty: %d, shadirty: %d" %(self.private_dirty, self.shared_dirty) + return "private: %d, shared: %d" %(self.private, self.shared) # # ============================================================================== From hpk at codespeak.net Mon Sep 22 22:33:46 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Mon, 22 Sep 2008 22:33:46 +0200 (CEST) Subject: [pypy-svn] r58349 - pypy/build/benchmem Message-ID: <20080922203346.0823C169F09@codespeak.net> Author: hpk Date: Mon Sep 22 22:33:44 2008 New Revision: 58349 Modified: pypy/build/benchmem/report.py pypy/build/benchmem/runbench.py Log: always collect at checkpoints for the current benchmarks don't print max overview, does not fully make sense. Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 22:33:44 2008 @@ -58,7 +58,7 @@ reader = runbench.LogReader() reader.parse_logfile(benchlog) - maxtable_overview(reader) + #maxtable_overview(reader) checkpointdetails(reader) #for name, results in reader.name2results.items(): Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 22:33:44 2008 @@ -51,10 +51,9 @@ import os, sys, gc pid = os.getpid() write(str(pid) + "\\n") - checkpoint() + checkpoint(collect=True) %s(checkpoint, %s) - gc.collect() # XXX - checkpoint() + checkpoint(collect=True) write("F") sys.stdin.close() """ %(name, arglist)) From fijal at codespeak.net Mon Sep 22 22:40:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 22:40:03 +0200 (CEST) Subject: [pypy-svn] r58352 - in pypy/build/benchmem: . benchmark Message-ID: <20080922204003.B1063169F17@codespeak.net> Author: fijal Date: Mon Sep 22 22:40:01 2008 New Revision: 58352 Added: pypy/build/benchmem/benchmark/empty_instances.py (contents, props changed) pypy/build/benchmem/benchmark/linked_list_with_floats.py (contents, props changed) pypy/build/benchmem/benchmark/lists.py (contents, props changed) Modified: pypy/build/benchmem/runbench.py Log: * Add a couple of benchmarks * Also add a coefficient to multiply default number of iterations Added: pypy/build/benchmem/benchmark/empty_instances.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchmark/empty_instances.py Mon Sep 22 22:40:01 2008 @@ -0,0 +1,6 @@ +def bench_empty_instances(checkpoint, iter1): + class A(object): + pass + + x = [A() for i in range(iter1)] + checkpoint(collect=True) Added: pypy/build/benchmem/benchmark/linked_list_with_floats.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchmark/linked_list_with_floats.py Mon Sep 22 22:40:01 2008 @@ -0,0 +1,12 @@ +def bench_linked_list(checkpoint, iter1): + + class A(object): + def __init__(self, other, i): + self.i = float(i) + self.other = other + + x = None + for i in range(iter1): + x = A(x, i) + checkpoint(collect=True) + Added: pypy/build/benchmem/benchmark/lists.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchmark/lists.py Mon Sep 22 22:40:01 2008 @@ -0,0 +1,6 @@ + +def bench_lists(checkpoint, iter1): + x = [] + for i in range(iter1): + x = [x] + checkpoint(collect=True) Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Mon Sep 22 22:40:01 2008 @@ -184,6 +184,8 @@ help="logfile for recording benchmark measurements") parser.add_option("-a", "--append", action="store_true", dest="append", default=False, help="append to logfile") +parser.add_option("-c", "--coefficient", action="store", dest="coeff", + default=1, help="Coefficient of number of iterations") def getbenchfiles(options, args): if args: @@ -221,5 +223,6 @@ raise SystemExit("could not find %r"% (executable)) runner = BenchRunner(executable, benchlog) for name in names: - runner.run_checkpointed_bench(name, (100000,)) + iter1 = int(100000*float(options.coeff)) + runner.run_checkpointed_bench(name, (iter1,)) print "bench results append to -->>>", benchlog From fijal at codespeak.net Mon Sep 22 23:12:08 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 22 Sep 2008 23:12:08 +0200 (CEST) Subject: [pypy-svn] r58353 - pypy/build/benchmem Message-ID: <20080922211208.B2C9C169E7F@codespeak.net> Author: fijal Date: Mon Sep 22 23:12:07 2008 New Revision: 58353 Modified: pypy/build/benchmem/report.py Log: A very rudimentary gnuplot support. I don't know how to do various operations, but tried a bit Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Mon Sep 22 23:12:07 2008 @@ -6,7 +6,7 @@ xxx extend, introduce options """ -import py +import py, os import smaps, runbench def asciitable(table): @@ -38,10 +38,9 @@ ]) tw.line(asciitable(rows)) -def checkpointdetails(reader): - tw = py.io.TerminalWriter() +def _process_rows(reader, starthook, endhook): for name, results in reader.name2results.items(): - tw.sep("=", "private_dirty at checkpoints: %s" %(name,)) + starthook(name) row0 = ["num"] + [result.executable for result in results] numsnapshosts = len(results[0].snapshots) @@ -51,8 +50,43 @@ for result in results: row.append(result.snapshots[i].private) rows.append(row) - tw.line(asciitable(rows)) - + endhook(rows) + +def checkpointdetails(reader): + tw = py.io.TerminalWriter() + def starthook(name): + tw.sep("=", "private_dirty at checkpoints: %s" %(name,)) + _process_rows(reader, starthook, lambda rows: tw.line(asciitable(rows))) + +def gnuplot_output(reader): + output = [] + pythons = [] + + def endhook(rows): + if not pythons: + pythons.extend(rows[0][1:]) + output.append(rows[2][1:]) + + _process_rows(reader, lambda name : None, endhook) + runbench.mydir.join("gnuplotdata").write( + "\n".join([" ".join([str(j) for j in i]) for i in output])) + + def new_cmd(num, name): + s = ("plot 'gnuplotdata' using ($%d) with histograms title '%s'" % + (num + 1,name)) + if num > 0: + s = "re" + s + return s + + plotcmds = "\n".join([new_cmd(i, name) for i, name in enumerate(pythons)]) + runbench.mydir.join("gnuplotcmd").write( + """set terminal postscript color + set output 'output.ps' + %s + """ % plotcmds + ) + os.system("gnuplot gnuplotcmd") + if __name__ == "__main__": benchlog = py.path.local("bench.log") reader = runbench.LogReader() @@ -60,6 +94,7 @@ #maxtable_overview(reader) checkpointdetails(reader) + gnuplot_output(reader) #for name, results in reader.name2results.items(): # tw.sep("=", name) From fijal at codespeak.net Tue Sep 23 00:54:26 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 00:54:26 +0200 (CEST) Subject: [pypy-svn] r58355 - pypy/branch/2.5-features/pypy/module/_codecs Message-ID: <20080922225426.F0A9F169E41@codespeak.net> Author: fijal Date: Tue Sep 23 00:54:24 2008 New Revision: 58355 Modified: pypy/branch/2.5-features/pypy/module/_codecs/__init__.py pypy/branch/2.5-features/pypy/module/_codecs/app_codecs.py pypy/branch/2.5-features/pypy/module/_codecs/interp_codecs.py Log: General whack-until-it-works coding. Does not behave exactly like in cpython, but the question is do we care? This opaque object seems to have no public interface anyway. Modified: pypy/branch/2.5-features/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/2.5-features/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/2.5-features/pypy/module/_codecs/__init__.py Tue Sep 23 00:54:24 2008 @@ -21,6 +21,7 @@ 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', '_register_existing_errors': 'app_codecs._register_existing_errors', + 'charmap_build' : 'app_codecs.charmap_build' } interpleveldefs = { 'encode': 'interp_codecs.encode', Modified: pypy/branch/2.5-features/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/2.5-features/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/2.5-features/pypy/module/_codecs/app_codecs.py Tue Sep 23 00:54:24 2008 @@ -93,7 +93,6 @@ def charmap_encode(obj, errors='strict', mapping='latin-1'): """None """ - res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors) res = ''.join(res) return res, len(res) @@ -600,13 +599,6 @@ p += p[1] return p - -def PyUnicode_DecodeMBCS(s, size, errors): - pass - -def PyUnicode_EncodeMBCS(p, size, errors): - pass - def unicode_call_errorhandler(errors, encoding, reason, input, startinpos, endinpos, decode=True): @@ -951,3 +943,13 @@ pos += count return p + +def charmap_build(somestring): + m = {} + num = 0 + for elem in somestring: + m[ord(elem)] = num + num += 1 + return m + + Modified: pypy/branch/2.5-features/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/2.5-features/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/2.5-features/pypy/module/_codecs/interp_codecs.py Tue Sep 23 00:54:24 2008 @@ -76,7 +76,6 @@ Looks up a codec tuple in the Python codec registry and returns a tuple of functions. """ - #import pdb; pdb.set_trace() state = space.fromcache(CodecState) normalized_encoding = encoding.replace(" ", "-").lower() w_result = state.codec_search_cache.get(normalized_encoding, None) From fijal at codespeak.net Tue Sep 23 01:04:21 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 01:04:21 +0200 (CEST) Subject: [pypy-svn] r58356 - pypy/branch/2.5-features/pypy/module/symbol Message-ID: <20080922230421.97455169E39@codespeak.net> Author: fijal Date: Tue Sep 23 01:04:20 2008 New Revision: 58356 Modified: pypy/branch/2.5-features/pypy/module/symbol/__init__.py Log: re-introduce evil line, which seems to be fairly necessary for things to work (still not sure why it's there) Modified: pypy/branch/2.5-features/pypy/module/symbol/__init__.py ============================================================================== --- pypy/branch/2.5-features/pypy/module/symbol/__init__.py (original) +++ pypy/branch/2.5-features/pypy/module/symbol/__init__.py Tue Sep 23 01:04:20 2008 @@ -38,3 +38,6 @@ Module.interpleveldefs[name] = 'space.wrap(%d)' % val sym_name[val] = name Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,) + +# This is very evil +_init_symbols('2.5') From fijal at codespeak.net Tue Sep 23 01:07:10 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 01:07:10 +0200 (CEST) Subject: [pypy-svn] r58357 - in pypy/branch/2.5-features/pypy: interpreter/pyparser/test rlib Message-ID: <20080922230710.08041169E4A@codespeak.net> Author: fijal Date: Tue Sep 23 01:07:10 2008 New Revision: 58357 Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_samples.py pypy/branch/2.5-features/pypy/rlib/rope.py Log: * Bump version number in a test (does not change much) * remove forgotten import pdb Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_samples.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_samples.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_samples.py Tue Sep 23 01:07:10 2008 @@ -11,7 +11,7 @@ from pypy.interpreter.pyparser.pythonlexer import TokenError grammar.DEBUG = False -_, PYPY_VERSION = get_grammar_file("2.4") +_, PYPY_VERSION = get_grammar_file("2.5") # these samples are skipped if the native version of Python does not match # the version of the grammar we use GRAMMAR_MISMATCH = PYTHON_VERSION != PYPY_VERSION Modified: pypy/branch/2.5-features/pypy/rlib/rope.py ============================================================================== --- pypy/branch/2.5-features/pypy/rlib/rope.py (original) +++ pypy/branch/2.5-features/pypy/rlib/rope.py Tue Sep 23 01:07:10 2008 @@ -735,7 +735,6 @@ return -1 # invariant: stack should only contain nodes that can contain the int what stack = [node] - #import pdb; pdb.set_trace() i = 0 while stack: curr = stack.pop() From fijal at codespeak.net Tue Sep 23 01:16:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 01:16:12 +0200 (CEST) Subject: [pypy-svn] r58358 - in pypy/branch/2.5-features/pypy/interpreter: pyparser test Message-ID: <20080922231612.14191169E16@codespeak.net> Author: fijal Date: Tue Sep 23 01:16:11 2008 New Revision: 58358 Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/astbuilder.py pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Log: Minor fix for yield expressions Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/astbuilder.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/pyparser/astbuilder.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/pyparser/astbuilder.py Tue Sep 23 01:16:11 2008 @@ -906,7 +906,10 @@ def build_yield_expr(builder, nb): atoms = get_atoms(builder, nb) - builder.push(ast.Yield(atoms[1], atoms[0].lineno)) + if len(atoms) == 1: + builder.push(ast.Yield(ast.Const(builder.wrap_none()), atoms[0].lineno)) + else: + builder.push(ast.Yield(atoms[1], atoms[0].lineno)) def build_continue_stmt(builder, nb): atoms = get_atoms(builder, nb) Modified: pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/test/test_generator.py Tue Sep 23 01:16:11 2008 @@ -35,7 +35,7 @@ g = f() g.next() """ in d - d = d['g'] + g = d['g'] assert g.send(42) == 42 def test_throw1(self): From fijal at codespeak.net Tue Sep 23 01:23:14 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 01:23:14 +0200 (CEST) Subject: [pypy-svn] r58359 - pypy/branch/2.5-features/pypy/interpreter/astcompiler Message-ID: <20080922232314.065FF169E30@codespeak.net> Author: fijal Date: Tue Sep 23 01:23:11 2008 New Revision: 58359 Modified: pypy/branch/2.5-features/pypy/interpreter/astcompiler/ast.py Log: Fix a bit repr of From Modified: pypy/branch/2.5-features/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/astcompiler/ast.py Tue Sep 23 01:23:11 2008 @@ -2508,7 +2508,7 @@ def __repr__(self): - return "From(%s, %s)" % (self.modname.__repr__(), self.names.__repr__()) + return "From(%s, %s, %d)" % (self.modname.__repr__(), self.names.__repr__(), self.level) def accept(self, visitor): return visitor.visitFrom(self) From hpk at codespeak.net Tue Sep 23 09:37:57 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 23 Sep 2008 09:37:57 +0200 (CEST) Subject: [pypy-svn] r58361 - pypy/build/benchmem Message-ID: <20080923073757.15D60169E11@codespeak.net> Author: hpk Date: Tue Sep 23 09:37:56 2008 New Revision: 58361 Modified: pypy/build/benchmem/report.py Log: adapt reporting to fijal's changes Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Tue Sep 23 09:37:56 2008 @@ -55,7 +55,7 @@ def checkpointdetails(reader): tw = py.io.TerminalWriter() def starthook(name): - tw.sep("=", "private_dirty at checkpoints: %s" %(name,)) + tw.sep("=", "private RSS at checkpoints: %s" %(name,)) _process_rows(reader, starthook, lambda rows: tw.line(asciitable(rows))) def gnuplot_output(reader): From pedronis at codespeak.net Tue Sep 23 10:03:01 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 10:03:01 +0200 (CEST) Subject: [pypy-svn] r58362 - in pypy/build/testrunner: . test Message-ID: <20080923080301.2958D169EA8@codespeak.net> Author: pedronis Date: Tue Sep 23 10:02:58 2008 New Revision: 58362 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: (iko, pedronis) - some more tests - factor main code into main(args) - logic to add to the log info about unexpected exits Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 10:02:58 2008 @@ -26,6 +26,19 @@ exitcode = run(args, cwd, out) return exitcode +def interpret_exitcode(exitcode, test): + extralog = "" + if exitcode: + failure = True + if exitcode != 1: + if exitcode > 0: + msg = "Exit code %d." % exitcode + else: + msg = "Killed by %s." % getsignalname(-exitcode) + extralog = "! %s\n %s\n" % (test, msg) + else: + failure = False + return failure, extralog def worker(num, n, run_param, testdirs, result_queue): sessdir = run_param.sessdir @@ -55,15 +68,12 @@ else: logdata = "" - if exitcode: - failure = True - if exitcode != 1: - pass # xxx unexpected exit cases - else: - failure = False + failure, extralog = interpret_exitcode(exitcode, test) - result_queue.put((failure, logdata, output)) + if extralog: + logdata += extralog + result_queue.put((failure, logdata, output)) def start_workers(n, run_param, testdirs): @@ -147,7 +157,7 @@ -if __name__ == '__main__': +def main(args): parser = optparse.OptionParser() parser.add_option("--logfile", dest="logfile", default=None, help="accumulated machine-readable logfile") @@ -162,7 +172,7 @@ type="int", help="number of parallel test runs") - opts, args = parser.parse_args() + opts, args = parser.parse_args(args) if opts.logfile is None: print "no logfile specified" @@ -178,7 +188,6 @@ testdirs = [] - run_param = RunParam(root) # the config files are python files whose run overrides the content # of the run_param instance namespace @@ -196,3 +205,7 @@ if res: sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv) Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Sep 23 10:02:58 2008 @@ -46,6 +46,29 @@ test_driver=['driver', 'darg']) assert res == -signal.SIGSEGV + def test_interpret_exitcode(self): + failure, extralog = runner.interpret_exitcode(0, "test_foo") + assert not failure + assert extralog == "" + + failure, extralog = runner.interpret_exitcode(1, "test_foo") + assert failure + assert extralog == "" + + + failure, extralog = runner.interpret_exitcode(2, "test_foo") + assert failure + assert extralog == """! test_foo + Exit code 2. +""" + + failure, extralog = runner.interpret_exitcode(-signal.SIGSEGV, + "test_foo") + assert failure + assert extralog == """! test_foo + Killed by SIGSEGV. +""" + class TestRunner(object): @@ -53,22 +76,45 @@ def setup_class(cls): cls.udir = py.path.local.make_numbered_dir(prefix='usession-runner-', keep=3) - cls.udir.join('test_normal').ensure(dir=1) - for p in py.path.local(__file__).dirpath( - 'examples', 'normal').listdir("*.py"): - p.copy(cls.udir.join('test_normal', 'test_'+p.basename)) + + def fill_test_dir(test_dir): + for p in py.path.local(__file__).dirpath( + 'examples', 'normal').listdir("*.py"): + p.copy(test_dir.join('test_'+p.basename)) + + + test_normal_dir0 = cls.udir.join('one', 'test_normal').ensure(dir=1) + cls.one_test_dir = cls.udir.join('one') + + fill_test_dir(test_normal_dir0) + + + test_normal_dir1 = cls.udir.join('two', 'test_normal1').ensure(dir=1) + test_normal_dir2 = cls.udir.join('two', 'pkg', + 'test_normal2').ensure(dir=1) + cls.two_test_dir = cls.udir.join('two') + + fill_test_dir(test_normal_dir1) + fill_test_dir(test_normal_dir2) + def teardown_class(cls): pass - def test_collect_testdirs_simple(self): + def test_collect_testdirs(self): res = [] - run_param = runner.RunParam(self.udir) + run_param = runner.RunParam(self.one_test_dir) run_param.collect_testdirs(res) assert res == ['test_normal'] + + res = [] + run_param = runner.RunParam(self.two_test_dir) + run_param.collect_testdirs(res) + + assert sorted(res) == ['pkg/test_normal2', 'test_normal1'] def test_one_dir(self): test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] @@ -76,11 +122,11 @@ log = cStringIO.StringIO() out = cStringIO.StringIO() - param = runner.RunParam(self.udir) - param.test_driver = test_driver - param.parallel_runs = 3 + run_param = runner.RunParam(self.one_test_dir) + run_param.test_driver = test_driver + run_param.parallel_runs = 3 - res = runner.execute_tests(param, ['test_normal'], log, out) + res = runner.execute_tests(run_param, ['test_normal'], log, out) assert res @@ -100,6 +146,39 @@ assert noutcomes == 107 assert nfailures == 6 - # xxx test with more than one dir - # xxx test for main logic + def test_many_dirs(self): + test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + + log = cStringIO.StringIO() + out = cStringIO.StringIO() + + run_param = runner.RunParam(self.udir) + run_param.test_driver = test_driver + run_param.parallel_runs = 3 + + testdirs = [] + run_param.collect_testdirs(testdirs) + + res = runner.execute_tests(run_param, testdirs, log, out) + + assert res + + assert out.getvalue() + + log_lines = log.getvalue().splitlines() + + nfailures = 0 + noutcomes = 0 + for line in log_lines: + if line[0] != ' ': + noutcomes += 1 + if line[0] != '.': + nfailures += 1 + + assert noutcomes == 3*107 + assert nfailures == 3*6 + + + + From pedronis at codespeak.net Tue Sep 23 10:28:44 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 10:28:44 +0200 (CEST) Subject: [pypy-svn] r58364 - pypy/branch/pypy-pytrunk Message-ID: <20080923082844.CA0FA169DB3@codespeak.net> Author: pedronis Date: Tue Sep 23 10:28:42 2008 New Revision: 58364 Modified: pypy/branch/pypy-pytrunk/ (props changed) Log: (iko, pedronis) get the testrunner through an external From pedronis at codespeak.net Tue Sep 23 11:26:44 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 11:26:44 +0200 (CEST) Subject: [pypy-svn] r58365 - in pypy/build/testrunner: . test Message-ID: <20080923092644.A3F5B169EAB@codespeak.net> Author: pedronis Date: Tue Sep 23 11:26:42 2008 New Revision: 58365 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: (iko, pedronis) --dry-run option for debugging setups Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 11:26:42 2008 @@ -10,6 +10,13 @@ finally: f.close() +def dry_run(args, cwd, out): + f = out.open('w') + try: + f.write("run %s with cwd='%s'\n" % (args, cwd)) + finally: + f.close() + return 0 def getsignalname(n): for name, value in signal.__dict__.items(): @@ -17,13 +24,19 @@ return name return 'signal %d' % (n,) -def execute_test(cwd, test, out, logfname, interp, test_driver): +def execute_test(cwd, test, out, logfname, interp, test_driver, + do_dry_run=False): args = interp+test_driver args += ['--resultlog=%s' % logfname, test] args = map(str, args) - - exitcode = run(args, cwd, out) + if do_dry_run: + runfunc = dry_run + else: + runfunc = run + + exitcode = runfunc(args, cwd, out) + return exitcode def interpret_exitcode(exitcode, test): @@ -45,6 +58,7 @@ root = run_param.root test_driver = run_param.test_driver interp = run_param.interp + dry_run = run_param.dry_run # xxx cfg thread start while 1: try: @@ -58,7 +72,7 @@ num += n exitcode = execute_test(root, test, one_output, logfname, - interp, test_driver) + interp, test_driver, do_dry_run=dry_run) # xxx cfg cleanup after testdir @@ -74,13 +88,14 @@ logdata += extralog result_queue.put((failure, logdata, output)) - + +invoke_in_thread = thread.start_new_thread def start_workers(n, run_param, testdirs): result_queue = Queue.Queue() for i in range(n): - thread.start_new_thread(worker, (i, n, run_param, testdirs, - result_queue)) + invoke_in_thread(worker, (i, n, run_param, testdirs, + result_queue)) return result_queue @@ -120,6 +135,7 @@ class RunParam(object): + dry_run = False interp = [os.path.abspath(sys.executable)] test_driver = [os.path.abspath(os.path.join('py', 'bin', 'py.test'))] parallel_runs = 1 @@ -170,8 +186,11 @@ help="root directory for the run") parser.add_option("--parallel-runs", dest="parallel_runs", default=0, type="int", - help="number of parallel test runs") - + help="number of parallel test runs") + parser.add_option("--dry-run", dest="dry_run", default=False, + action="store_true", + help="dry run") + opts, args = parser.parse_args(args) if opts.logfile is None: @@ -200,6 +219,10 @@ if opts.parallel_runs: run_param.parallel_runs = opts.parallel_runs + run_param.dry_run = opts.dry_run + + if run_param.dry_run: + print >>out, run_param.__dict__ res = execute_tests(run_param, testdirs, logfile, out) Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Sep 23 11:26:42 2008 @@ -70,10 +70,14 @@ """ -class TestRunner(object): - +class RunnerTests(object): + with_thread = True def setup_class(cls): + cls.real_invoke_in_thread = (runner.invoke_in_thread,) + if not cls.with_thread: + runner.invoke_in_thread = lambda func, args: func(*args) + cls.udir = py.path.local.make_numbered_dir(prefix='usession-runner-', keep=3) @@ -99,7 +103,7 @@ def teardown_class(cls): - pass + runner.invoke_in_thread = cls.real_invoke_in_thread[0] def test_collect_testdirs(self): res = [] @@ -146,6 +150,29 @@ assert noutcomes == 107 assert nfailures == 6 + def test_one_dir_dry_run(self): + test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + + log = cStringIO.StringIO() + out = cStringIO.StringIO() + + run_param = runner.RunParam(self.one_test_dir) + run_param.test_driver = test_driver + run_param.parallel_runs = 3 + run_param.dry_run = True + + res = runner.execute_tests(run_param, ['test_normal'], log, out) + + assert not res + + assert log.getvalue() == "" + + out_lines = out.getvalue().splitlines() + + assert len(out_lines) == 1 + + assert out_lines[0].startswith("run [") + assert "test_normal" in out_lines[0] def test_many_dirs(self): test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] @@ -180,5 +207,10 @@ assert nfailures == 3*6 - +class TestRunnerNoThreads(RunnerTests): + with_thread = False + + +class TestRunner(RunnerTests): + pass From pedronis at codespeak.net Tue Sep 23 11:43:35 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 11:43:35 +0200 (CEST) Subject: [pypy-svn] r58367 - in pypy/build/testrunner: . test Message-ID: <20080923094335.3AD1B169F3A@codespeak.net> Author: pedronis Date: Tue Sep 23 11:43:34 2008 New Revision: 58367 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: (iko, pedronis) - missing relto - adjusting tests Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 11:43:34 2008 @@ -163,7 +163,7 @@ for p1 in entries: if self.is_test_py_file(p1): self.collect_one_testdir(testdirs, reldir, - [t for t in entries + [t.relto(self.root) for t in entries if self.is_test_py_file(t)]) return Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Sep 23 11:43:34 2008 @@ -105,21 +105,6 @@ def teardown_class(cls): runner.invoke_in_thread = cls.real_invoke_in_thread[0] - def test_collect_testdirs(self): - res = [] - run_param = runner.RunParam(self.one_test_dir) - - run_param.collect_testdirs(res) - - assert res == ['test_normal'] - - res = [] - run_param = runner.RunParam(self.two_test_dir) - - run_param.collect_testdirs(res) - - assert sorted(res) == ['pkg/test_normal2', 'test_normal1'] - def test_one_dir(self): test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] @@ -210,6 +195,35 @@ class TestRunnerNoThreads(RunnerTests): with_thread = False + def test_collect_testdirs(self): + res = [] + seen = [] + run_param = runner.RunParam(self.one_test_dir) + real_collect_one_testdir = run_param.collect_one_testdir + + def witness_collect_one_testdir(testdirs, reldir, tests): + seen.append((reldir, sorted(map(str, tests)))) + real_collect_one_testdir(testdirs, reldir, tests) + + run_param.collect_one_testdir = witness_collect_one_testdir + + run_param.collect_testdirs(res) + + assert res == ['test_normal'] + assert len(seen) == 1 + reldir, tests = seen[0] + assert reldir == 'test_normal' + for test in tests: + assert test.startswith('test_normal/') + + run_param.collect_one_testdir = real_collect_one_testdir + res = [] + run_param = runner.RunParam(self.two_test_dir) + + run_param.collect_testdirs(res) + + assert sorted(res) == ['pkg/test_normal2', 'test_normal1'] + class TestRunner(RunnerTests): pass From pedronis at codespeak.net Tue Sep 23 11:46:59 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 11:46:59 +0200 (CEST) Subject: [pypy-svn] r58368 - pypy/branch/pypy-pytrunk/pypy Message-ID: <20080923094659.56904169F3A@codespeak.net> Author: pedronis Date: Tue Sep 23 11:46:58 2008 New Revision: 58368 Added: pypy/branch/pypy-pytrunk/pypy/testrunner_cfg.py (contents, props changed) Log: a configuration for the testrunner to explode translator/c into tests run one by one Added: pypy/branch/pypy-pytrunk/pypy/testrunner_cfg.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-pytrunk/pypy/testrunner_cfg.py Tue Sep 23 11:46:58 2008 @@ -0,0 +1,11 @@ +# nightly test configuration for the paraller runner + +def collect_one_testdir(testdirs, reldir, tests): + if (reldir.startswith('jit/codegen/i386/') or + reldir.startswith('jit/timeshifter/') or + reldir.startswith('translator/c/')): + testdirs.extend(tests) + else: + testdirs.append(reldir) + + From pedronis at codespeak.net Tue Sep 23 12:00:30 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 12:00:30 +0200 (CEST) Subject: [pypy-svn] r58369 - pypy/build/testrunner Message-ID: <20080923100030.C8E22169F4A@codespeak.net> Author: pedronis Date: Tue Sep 23 12:00:30 2008 New Revision: 58369 Modified: pypy/build/testrunner/runner.py Log: (iko, pedronis) don't die on missing configs, show which one are used Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 12:00:30 2008 @@ -213,7 +213,9 @@ # in that code function overriding method should not take self # though a self and self.__class__ are available if needed for config_py_file in opts.config: - execfile(config_py_file, run_param.__dict__) + if py.path.local(config_py_file).check(file=1): + print >>out, "using config", config_py_file + execfile(config_py_file, run_param.__dict__) run_param.collect_testdirs(testdirs) From pedronis at codespeak.net Tue Sep 23 12:08:52 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 12:08:52 +0200 (CEST) Subject: [pypy-svn] r58371 - pypy/build/testrunner Message-ID: <20080923100852.53AAD169E4D@codespeak.net> Author: pedronis Date: Tue Sep 23 12:08:51 2008 New Revision: 58371 Modified: pypy/build/testrunner/runner.py Log: (iko, pedronis) apply expanduser to the config file names for convenience Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 12:08:51 2008 @@ -213,6 +213,7 @@ # in that code function overriding method should not take self # though a self and self.__class__ are available if needed for config_py_file in opts.config: + config_py_file = os.path.expanduser(config_py_file) if py.path.local(config_py_file).check(file=1): print >>out, "using config", config_py_file execfile(config_py_file, run_param.__dict__) From pedronis at codespeak.net Tue Sep 23 12:11:28 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 12:11:28 +0200 (CEST) Subject: [pypy-svn] r58372 - pypy/build/bot2/pypybuildbot Message-ID: <20080923101128.7FA64169E4E@codespeak.net> Author: pedronis Date: Tue Sep 23 12:11:28 2008 New Revision: 58372 Modified: pypy/build/bot2/pypybuildbot/steps.py Log: (iko, pedronis) - experimental full own tests dry run setup - next step tweak this and set it up on wyvern Modified: pypy/build/bot2/pypybuildbot/steps.py ============================================================================== --- pypy/build/bot2/pypybuildbot/steps.py (original) +++ pypy/build/bot2/pypybuildbot/steps.py Tue Sep 23 12:11:28 2008 @@ -53,14 +53,17 @@ self.addStep(CondShellCommand( description="wcrevert", cond=not_first_time, - command = ["python", "py/bin/py.svnwcrevert", ".", - ".buildbot-sourcedata"], - haltOnFailure=True)) + command = ["python", "py/bin/py.svnwcrevert", + "-p.buildbot-sourcedata", "."], + )) self.addStep(source.SVN("https://codespeak.net/svn/pypy/" "branch/pypy-pytrunk")) self.addStep(shell.ShellCommand( description="pytest", - command=["python", "py/bin/py.test", - "pypy/module/__builtin__", "pypy/module/operator", - "--session=FileLogSession", "--filelog=pytest.log"], - logfiles={'pytestLog': 'pytest.log'})) + command=["python", "testrunner/runner.py", + "--logfile=testrun.log", "--dry-run", + "--config=pypy/testrunner_cfg.py", + "--config=~/machine_cfg.py", + "--root=pypy"], + logfiles={'pytestLog': 'testrun.log'}, + env={"PYTHONPATH": ['.']})) From fijal at codespeak.net Tue Sep 23 13:32:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 13:32:47 +0200 (CEST) Subject: [pypy-svn] r58376 - pypy/branch/2.5-features/pypy/interpreter/pyparser/test Message-ID: <20080923113247.4CEC2169EC6@codespeak.net> Author: fijal Date: Tue Sep 23 13:32:45 2008 New Revision: 58376 Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/test/expressions.py pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_astbuilder.py Log: A test for yield expression. Just in order to make sure this is tested Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/test/expressions.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/pyparser/test/expressions.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/pyparser/test/expressions.py Tue Sep 23 13:32:45 2008 @@ -484,6 +484,7 @@ CHANGES_25_INPUTS = [ ["class A(): pass"], + ["def f(): x = yield 3"] ] EXEC_INPUTS = [ Modified: pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_astbuilder.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_astbuilder.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/pyparser/test/test_astbuilder.py Tue Sep 23 13:32:45 2008 @@ -174,6 +174,7 @@ # yield changed in 2.5, so we can not trust the stablecompiler "def f(n):\n for i in range(n):\n yield n\n": "Module(None, Stmt([Function(None, 'f', [AssName('n', 0)], [], 0, None, Stmt([For(AssName('i', 0), CallFunc(Name('range'), [Name('n')], None, None), Stmt([Discard(Yield(Name('n')))]), None)]))]))", + "def f(): x = yield 3": "Module(None, Stmt([Function(None, 'f', [], [], 0, None, Stmt([Assign([AssName('x', 0)], Yield(Const(3)))]))]))", # stablecompiler produces a Pass statement which does not seem very consistent # (a module should only have a Stmt child) From fijal at codespeak.net Tue Sep 23 13:59:02 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 13:59:02 +0200 (CEST) Subject: [pypy-svn] r58377 - pypy/branch/2.5-features/pypy/interpreter/astcompiler Message-ID: <20080923115902.28A46169F06@codespeak.net> Author: fijal Date: Tue Sep 23 13:59:00 2008 New Revision: 58377 Modified: pypy/branch/2.5-features/pypy/interpreter/astcompiler/pyassem.py pypy/branch/2.5-features/pypy/interpreter/astcompiler/pycodegen.py Log: Port a couple of fixes from dist. This is necessary to run any tests. Modified: pypy/branch/2.5-features/pypy/interpreter/astcompiler/pyassem.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/astcompiler/pyassem.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/astcompiler/pyassem.py Tue Sep 23 13:59:00 2008 @@ -51,7 +51,8 @@ self.argcount = self.argcount - 1 def checkFlag(self, flag): - return self.flags & flag + if self.flags & flag: + return 1 def setFreeVars(self, names): self.freevars = list(names) @@ -257,6 +258,8 @@ depths[i] = stackdepth else: if previous_value != stackdepth: + import pdb + pdb.set_trace() raise InternalCompilerError("inconsistent stack depth") def computeStackDepth(self): @@ -264,6 +267,7 @@ co_code = self.co_code self._stackdepths = [UNREACHABLE] * len(co_code) self._stackdepths[0] = 0 + just_loaded_const = None consts_w = self.getConsts() finally_targets = {} largestsize = 0 @@ -291,6 +295,7 @@ i += 1 if curstackdepth == UNREACHABLE: + just_loaded_const = None continue # ignore unreachable instructions if opcode in DEPTH_OP_EFFECT_ALONG_JUMP: @@ -308,12 +313,28 @@ except KeyError: pass else: - effect = tracker(oparg) + if opcode == pythonopcode.opmap['MAKE_CLOSURE']: + # only supports "LOAD_CONST co / MAKE_CLOSURE n" + if just_loaded_const is None: + raise InternalCompilerError("MAKE_CLOSURE not " + "following LOAD_CONST") + codeobj = self.space.interp_w(PyCode, just_loaded_const) + nfreevars = len(codeobj.co_freevars) + effect = - nfreevars - oparg + else: + effect = tracker(oparg) + curstackdepth += effect if i in finally_targets: curstackdepth += 2 # see pyopcode.FinallyBlock.cleanup() self._setdepth(i, curstackdepth) + if opcode == pythonopcode.opmap['LOAD_CONST']: + just_loaded_const = consts_w[oparg] + else: + just_loaded_const = None + + self.stacksize = largestsize def fixLabelTargets(self): @@ -464,9 +485,8 @@ def depth_MAKE_FUNCTION(argc): return -argc def depth_MAKE_CLOSURE(argc): - if argc == 0: - return -1 - return -argc + raise InternalCompilerError("must special-case this in order to account" + " for the free variables") def depth_BUILD_SLICE(argc): if argc == 2: return -1 Modified: pypy/branch/2.5-features/pypy/interpreter/astcompiler/pycodegen.py ============================================================================== --- pypy/branch/2.5-features/pypy/interpreter/astcompiler/pycodegen.py (original) +++ pypy/branch/2.5-features/pypy/interpreter/astcompiler/pycodegen.py Tue Sep 23 13:59:00 2008 @@ -335,21 +335,18 @@ self.set_lineno(node) for default in node.defaults: default.accept( self ) - self._makeClosure(gen, len(node.defaults)) - for i in range(ndecorators): - self.emitop_int('CALL_FUNCTION', 1) - - def _makeClosure(self, gen, args): frees = gen.scope.get_free_vars_in_parent() if frees: for name in frees: self.emitop('LOAD_CLOSURE', name) - self.emitop_int('BUILD_TUPLE', len(frees)) self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_CLOSURE', args) + self.emitop_int('MAKE_CLOSURE', len(node.defaults)) else: self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_FUNCTION', args) + self.emitop_int('MAKE_FUNCTION', len(node.defaults)) + for i in range(ndecorators): + self.emitop_int('CALL_FUNCTION', 1) + def visitClass(self, node): gen = ClassCodeGenerator(self.space, node, @@ -361,7 +358,16 @@ for base in node.bases: base.accept( self ) self.emitop_int('BUILD_TUPLE', len(node.bases)) - self._makeClosure(gen, 0) + frees = gen.scope.get_free_vars_in_parent() + if frees: + for name in frees: + self.emitop('LOAD_CLOSURE', name) + self.emitop_code('LOAD_CONST', gen) + self.emitop_int('MAKE_CLOSURE', 0) + else: + self.emitop_code('LOAD_CONST', gen) + self.emitop_int('MAKE_FUNCTION', 0) + self.emitop_int('CALL_FUNCTION', 0) self.emit('BUILD_CLASS') self.storeName(node.name, node.lineno) @@ -593,7 +599,7 @@ def visitListComp(self, node): self.set_lineno(node) # setup list - tmpname = "$list%d" % self.__list_count + tmpname = "_[%d]" % self.__list_count self.__list_count = self.__list_count + 1 self.emitop_int('BUILD_LIST', 0) self.emit('DUP_TOP') @@ -655,7 +661,16 @@ inner.accept( gen ) gen.finish() self.set_lineno(node) - self._makeClosure(gen, 0) + frees = gen.scope.get_free_vars_in_parent() + if frees: + for name in frees: + self.emitop('LOAD_CLOSURE', name) + self.emitop_code('LOAD_CONST', gen) + self.emitop_int('MAKE_CLOSURE', 0) + else: + self.emitop_code('LOAD_CONST', gen) + self.emitop_int('MAKE_FUNCTION', 0) + # precomputation of outmost iterable qual0 = inner.quals[0] assert isinstance(qual0, ast.GenExprFor) From fijal at codespeak.net Tue Sep 23 15:12:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 15:12:20 +0200 (CEST) Subject: [pypy-svn] r58378 - pypy/branch/2.5-merge Message-ID: <20080923131220.84245169F5F@codespeak.net> Author: fijal Date: Tue Sep 23 15:12:18 2008 New Revision: 58378 Added: pypy/branch/2.5-merge/ - copied from r58377, pypy/branch/2.5-features/ Log: Create a new branch which is a copy of 2.5 for merging dist into it From pedronis at codespeak.net Tue Sep 23 15:16:23 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 15:16:23 +0200 (CEST) Subject: [pypy-svn] r58379 - pypy/build/bot2/pypybuildbot Message-ID: <20080923131623.18DAA169F60@codespeak.net> Author: pedronis Date: Tue Sep 23 15:16:22 2008 New Revision: 58379 Modified: pypy/build/bot2/pypybuildbot/master.py pypy/build/bot2/pypybuildbot/steps.py Log: (iko, pedronis) the configuration as it looks right now on wyvern Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Tue Sep 23 15:16:22 2008 @@ -27,8 +27,7 @@ 'change_source': [], 'schedulers': [Nightly("nightly", - ["pypy-own-linux", "pypy-own-other-linux", - "pypy-own-win"], hour=19)], + ["own-linux-x86-32"], hour=19)], # xxx time 'status': [status], 'slaves': [BuildSlave(name, password) @@ -36,23 +35,14 @@ in passwords.iteritems()], 'builders': [ - {"name": "pypy-own-linux", - "slavenames": ["vitaly"], - "builddir": "pypy-own-linux", + {"name": "own-linux-x86-32", + "slavenames": ["wyvern"], + "builddir": "own-linux-x86-32", "factory": pypyOwnTestFactory }, - {"name": "pypy-own-other-linux", - "slavenames": ["fido"], - "builddir": "pypy-own-other-linux", - "factory": pypyOwnTestFactory - }, - {"name": "pypy-own-win", - "slavenames": ['ebgoc'], - "builddir": "pypy-own-win", - "factory": pypyOwnTestFactoryWin} ], - 'buildbotURL': 'http://localhost:%d/' % (httpPortNumber,), + 'buildbotURL': 'http://wyvern.cs.uni-duesseldorf.de:%d/' % (httpPortNumber,), 'projectURL': 'http://codespeak.net/pypy/', 'projectName': 'PyPy'} Modified: pypy/build/bot2/pypybuildbot/steps.py ============================================================================== --- pypy/build/bot2/pypybuildbot/steps.py (original) +++ pypy/build/bot2/pypybuildbot/steps.py Tue Sep 23 15:16:22 2008 @@ -61,7 +61,7 @@ self.addStep(shell.ShellCommand( description="pytest", command=["python", "testrunner/runner.py", - "--logfile=testrun.log", "--dry-run", + "--logfile=testrun.log", "--config=pypy/testrunner_cfg.py", "--config=~/machine_cfg.py", "--root=pypy"], From pedronis at codespeak.net Tue Sep 23 15:31:03 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 15:31:03 +0200 (CEST) Subject: [pypy-svn] r58380 - in pypy/build/bot2: . pypybuildbot Message-ID: <20080923133103.E7C2C169F3B@codespeak.net> Author: pedronis Date: Tue Sep 23 15:31:00 2008 New Revision: 58380 Added: pypy/build/bot2/pypybuildbot/util.py (contents, props changed) pypy/build/bot2/vitaly_master.cfg - copied, changed from r58379, pypy/build/bot2/master.cfg Modified: pypy/build/bot2/master.cfg pypy/build/bot2/pypybuildbot/master.py Log: (iko, pedronis) - use a load helper instead of the three lines import/reload/= dance - reorganize things such that master.py reflects wyvern setups while there's still around the vilaly experimental setup on which new stuff can be tried out by us Modified: pypy/build/bot2/master.cfg ============================================================================== --- pypy/build/bot2/master.cfg (original) +++ pypy/build/bot2/master.cfg Tue Sep 23 15:31:00 2008 @@ -1,13 +1,13 @@ import sys, os +# checkout bot2 in the home dir of the master +sys.path.append(os.path.expanduser('~/bot2/')) +from pypybuildbot.util import load slavePortnum = "tcp:10407" httpPortNumber = 8099 # slavename -> password -import slaveinfo -reload(slaveinfo) -passwords = slaveinfo.passwords +passwords = load('slaveinfo').passwords + -# checkout bot2 in the home dir of the master -sys.path.append(os.path.expanduser('~/bot2/')) execfile(os.path.expanduser('~/bot2/pypybuildbot/master.py')) Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Tue Sep 23 15:31:00 2008 @@ -15,9 +15,12 @@ status = WebStatus(httpPortNumber, allowForce=True) -import pypybuildbot.steps -reload(pypybuildbot.steps) -pypysteps = pypybuildbot.steps +# pypy test summary page +summary = load('pypybuildbot.summary') +status.putChild('summary', summary.Summary()) + + +pypysteps = load('pypybuildbot.steps') pypyOwnTestFactory = pypysteps.PyPyOwnTestFactory() pypyOwnTestFactoryWin = pypysteps.PyPyOwnTestFactory(platform="win32") @@ -42,13 +45,7 @@ }, ], - 'buildbotURL': 'http://wyvern.cs.uni-duesseldorf.de:%d/' % (httpPortNumber,), + 'buildbotURL': 'http://wyvern.cs.uni-duesseldorf.de:%d/'%(httpPortNumber), 'projectURL': 'http://codespeak.net/pypy/', 'projectName': 'PyPy'} -import pypybuildbot.summary -reload(pypybuildbot.summary) -summary = pypybuildbot.summary - -# pypy test summary page -status.putChild('summary', summary.Summary()) Added: pypy/build/bot2/pypybuildbot/util.py ============================================================================== --- (empty file) +++ pypy/build/bot2/pypybuildbot/util.py Tue Sep 23 15:31:00 2008 @@ -0,0 +1,5 @@ + +def load(name): + mod = __import__(name, {}, {}, ['__all__']) + reload(mod) + return mod Copied: pypy/build/bot2/vitaly_master.cfg (from r58379, pypy/build/bot2/master.cfg) ============================================================================== --- pypy/build/bot2/master.cfg (original) +++ pypy/build/bot2/vitaly_master.cfg Tue Sep 23 15:31:00 2008 @@ -1,13 +1,39 @@ import sys, os +# checkout bot2 in the home dir of the master +sys.path.append(os.path.expanduser('~/bot2/')) +from pypybuildbot.util import load slavePortnum = "tcp:10407" httpPortNumber = 8099 # slavename -> password -import slaveinfo -reload(slaveinfo) -passwords = slaveinfo.passwords +passwords = load('slaveinfo').passwords + -# checkout bot2 in the home dir of the master -sys.path.append(os.path.expanduser('~/bot2/')) execfile(os.path.expanduser('~/bot2/pypybuildbot/master.py')) + + +# tweaks for vitaly experimental setup +BuildmasterConfig['schedulers'] = [Nightly("nightly", + ["own-linux-x86-32", + "own-other-linux-x86-32", + "own-win-32"], hour=19)] + +BuildmasterConfig['builders'] = [ + {"name": "own-linux-x86-32", + "slavenames": ["vitaly"], + "builddir": "own-linux-x86-32", + "factory": pypyOwnTestFactory + }, + {"name": "own-other-linux-x86-32", + "slavenames": ["fido"], + "builddir": "own-other-linux-x86-32", + "factory": pypyOwnTestFactory + }, + {"name": "own-win-32", + "slavenames": ['ebgoc'], + "builddir": "own-win-32", + "factory": pypyOwnTestFactoryWin} + ] + +BuildmasterConfig['buildbotURL'] = 'http://localhost:8099/' From pedronis at codespeak.net Tue Sep 23 15:39:35 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 15:39:35 +0200 (CEST) Subject: [pypy-svn] r58381 - pypy/build/bot2/pypybuildbot Message-ID: <20080923133935.15E3F169F87@codespeak.net> Author: pedronis Date: Tue Sep 23 15:39:34 2008 New Revision: 58381 Modified: pypy/build/bot2/pypybuildbot/master.py Log: (pedronis, iko) run the tests through buildbot for pypy-pytrunk just a bit after the autotest run for trunk. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Tue Sep 23 15:39:34 2008 @@ -30,7 +30,8 @@ 'change_source': [], 'schedulers': [Nightly("nightly", - ["own-linux-x86-32"], hour=19)], # xxx time + ["own-linux-x86-32"], + hour=4, minute=45)], 'status': [status], 'slaves': [BuildSlave(name, password) From pedronis at codespeak.net Tue Sep 23 16:03:12 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 16:03:12 +0200 (CEST) Subject: [pypy-svn] r58382 - pypy/build/testrunner Message-ID: <20080923140312.BE39F169F4A@codespeak.net> Author: pedronis Date: Tue Sep 23 16:03:10 2008 New Revision: 58382 Modified: pypy/build/testrunner/runner.py Log: (iko, pedronis) normalize dir separators in the test collection code to slashes, given how test are used it should not matter on windows but it will make it easier to write portable config files. Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Sep 23 16:03:10 2008 @@ -148,6 +148,10 @@ name = p.basename return name.startswith('test_') and name.endswith('.py') + def reltoroot(self, p): + rel = p.relto(self.root) + return rel.replace(os.sep, '/') + def collect_one_testdir(self, testdirs, reldir, tests): testdirs.append(reldir) return @@ -156,14 +160,14 @@ if p is None: p = self.root - reldir = p.relto(self.root) + reldir = self.reltoroot(p) entries = [p1 for p1 in p.listdir() if p1.check(dotfile=0)] if p != self.root: for p1 in entries: if self.is_test_py_file(p1): self.collect_one_testdir(testdirs, reldir, - [t.relto(self.root) for t in entries + [self.reltoroot(t) for t in entries if self.is_test_py_file(t)]) return From pedronis at codespeak.net Tue Sep 23 16:16:18 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 23 Sep 2008 16:16:18 +0200 (CEST) Subject: [pypy-svn] r58383 - pypy/build/bot2 Message-ID: <20080923141618.BFE08169EED@codespeak.net> Author: pedronis Date: Tue Sep 23 16:16:18 2008 New Revision: 58383 Modified: pypy/build/bot2/TODO Log: (iko, pedronis) update TODO, the parallel runner is done (minus tweaks) and seems to work on wyvern setup Modified: pypy/build/bot2/TODO ============================================================================== --- pypy/build/bot2/TODO (original) +++ pypy/build/bot2/TODO Tue Sep 23 16:16:18 2008 @@ -1,13 +1,12 @@ -- runner for running tests per directory and optionally in parallel - (this should be inspired by the current autotest.py running strategy minus the htmlconftest - details) [IN-PROGRESS] - +- tweak setup for operating on branches - buildbot Nightly scheduler that can pick a uniform revision server side and with support for specifying possibly branches - improve summary page (defined in pypybuildbot/summary.page), support querying/slicing by builders/revisions(/branches), formatting and css +- some kind of progress information for a run + - build/support for running cpython on tests, adaptations to produce structured logs and be compatible with the py.lib trunk are probably necessary for lib-python/conftest.py From fijal at codespeak.net Tue Sep 23 16:59:11 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 16:59:11 +0200 (CEST) Subject: [pypy-svn] r58388 - pypy/branch/2.5-merge/pypy/objspace/flow Message-ID: <20080923145911.39EA5169FB2@codespeak.net> Author: fijal Date: Tue Sep 23 16:59:10 2008 New Revision: 58388 Modified: pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py Log: fix flow objspace tests Modified: pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py Tue Sep 23 16:59:10 2008 @@ -218,6 +218,18 @@ return None + def abstract_issubclass(self, w_obj, w_cls, failhard=False): + return self.issubtype(w_obj, w_cls) + + def abstract_isinstance(self, w_obj, w_cls): + return self.isinstance(w_obj, w_cls) + + def abstract_isclass(self, w_obj): + return self.isinstance(w_obj, self.w_type) + + def abstract_getclass(self, w_obj): + return self.type(w_obj) + def build_flow(self, func, constargs={}): """ """ From fijal at codespeak.net Tue Sep 23 18:09:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 18:09:20 +0200 (CEST) Subject: [pypy-svn] r58390 - in pypy/branch/2.5-merge/pypy: interpreter objspace/flow Message-ID: <20080923160920.4D769169FC8@codespeak.net> Author: fijal Date: Tue Sep 23 18:09:19 2008 New Revision: 58390 Modified: pypy/branch/2.5-merge/pypy/interpreter/baseobjspace.py pypy/branch/2.5-merge/pypy/interpreter/error.py pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py Log: Hack differently, more the way it is on trunk Modified: pypy/branch/2.5-merge/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/baseobjspace.py Tue Sep 23 18:09:19 2008 @@ -806,7 +806,6 @@ # module when it is loaded. return self.type(w_obj) - def eval(self, expression, w_globals, w_locals): "NOT_RPYTHON: For internal debugging." import types Modified: pypy/branch/2.5-merge/pypy/interpreter/error.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/error.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/error.py Tue Sep 23 18:09:19 2008 @@ -175,7 +175,7 @@ while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if (space.is_true(space.abstract_isclass(w_type)) and + if (space.abstract_isclass_w(w_type) and is_valid_exception_class(space, w_type)): # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): Modified: pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/2.5-merge/pypy/objspace/flow/objspace.py Tue Sep 23 18:09:19 2008 @@ -217,19 +217,6 @@ return ecls return None - - def abstract_issubclass(self, w_obj, w_cls, failhard=False): - return self.issubtype(w_obj, w_cls) - - def abstract_isinstance(self, w_obj, w_cls): - return self.isinstance(w_obj, w_cls) - - def abstract_isclass(self, w_obj): - return self.isinstance(w_obj, self.w_type) - - def abstract_getclass(self, w_obj): - return self.type(w_obj) - def build_flow(self, func, constargs={}): """ """ From fijal at codespeak.net Tue Sep 23 18:15:55 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 18:15:55 +0200 (CEST) Subject: [pypy-svn] r58391 - pypy/branch/2.5-merge/lib-python/modified-2.5.1 Message-ID: <20080923161555.87985169FA1@codespeak.net> Author: fijal Date: Tue Sep 23 18:15:54 2008 New Revision: 58391 Removed: pypy/branch/2.5-merge/lib-python/modified-2.5.1/types.py Log: This is gone for a while From fijal at codespeak.net Tue Sep 23 18:56:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 18:56:58 +0200 (CEST) Subject: [pypy-svn] r58392 - pypy/branch/2.5-merge/pypy/interpreter/astcompiler Message-ID: <20080923165658.27377169FD4@codespeak.net> Author: fijal Date: Tue Sep 23 18:56:56 2008 New Revision: 58392 Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pycodegen.py Log: Revert 58377 as it was bogus Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py Tue Sep 23 18:56:56 2008 @@ -51,8 +51,7 @@ self.argcount = self.argcount - 1 def checkFlag(self, flag): - if self.flags & flag: - return 1 + return self.flags & flag def setFreeVars(self, names): self.freevars = list(names) @@ -258,8 +257,6 @@ depths[i] = stackdepth else: if previous_value != stackdepth: - import pdb - pdb.set_trace() raise InternalCompilerError("inconsistent stack depth") def computeStackDepth(self): @@ -267,7 +264,6 @@ co_code = self.co_code self._stackdepths = [UNREACHABLE] * len(co_code) self._stackdepths[0] = 0 - just_loaded_const = None consts_w = self.getConsts() finally_targets = {} largestsize = 0 @@ -295,7 +291,6 @@ i += 1 if curstackdepth == UNREACHABLE: - just_loaded_const = None continue # ignore unreachable instructions if opcode in DEPTH_OP_EFFECT_ALONG_JUMP: @@ -313,28 +308,12 @@ except KeyError: pass else: - if opcode == pythonopcode.opmap['MAKE_CLOSURE']: - # only supports "LOAD_CONST co / MAKE_CLOSURE n" - if just_loaded_const is None: - raise InternalCompilerError("MAKE_CLOSURE not " - "following LOAD_CONST") - codeobj = self.space.interp_w(PyCode, just_loaded_const) - nfreevars = len(codeobj.co_freevars) - effect = - nfreevars - oparg - else: - effect = tracker(oparg) - + effect = tracker(oparg) curstackdepth += effect if i in finally_targets: curstackdepth += 2 # see pyopcode.FinallyBlock.cleanup() self._setdepth(i, curstackdepth) - if opcode == pythonopcode.opmap['LOAD_CONST']: - just_loaded_const = consts_w[oparg] - else: - just_loaded_const = None - - self.stacksize = largestsize def fixLabelTargets(self): @@ -485,8 +464,9 @@ def depth_MAKE_FUNCTION(argc): return -argc def depth_MAKE_CLOSURE(argc): - raise InternalCompilerError("must special-case this in order to account" - " for the free variables") + if argc == 0: + return -1 + return -argc def depth_BUILD_SLICE(argc): if argc == 2: return -1 Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pycodegen.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pycodegen.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pycodegen.py Tue Sep 23 18:56:56 2008 @@ -335,18 +335,21 @@ self.set_lineno(node) for default in node.defaults: default.accept( self ) + self._makeClosure(gen, len(node.defaults)) + for i in range(ndecorators): + self.emitop_int('CALL_FUNCTION', 1) + + def _makeClosure(self, gen, args): frees = gen.scope.get_free_vars_in_parent() if frees: for name in frees: self.emitop('LOAD_CLOSURE', name) + self.emitop_int('BUILD_TUPLE', len(frees)) self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_CLOSURE', len(node.defaults)) + self.emitop_int('MAKE_CLOSURE', args) else: self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_FUNCTION', len(node.defaults)) - for i in range(ndecorators): - self.emitop_int('CALL_FUNCTION', 1) - + self.emitop_int('MAKE_FUNCTION', args) def visitClass(self, node): gen = ClassCodeGenerator(self.space, node, @@ -358,16 +361,7 @@ for base in node.bases: base.accept( self ) self.emitop_int('BUILD_TUPLE', len(node.bases)) - frees = gen.scope.get_free_vars_in_parent() - if frees: - for name in frees: - self.emitop('LOAD_CLOSURE', name) - self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_CLOSURE', 0) - else: - self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_FUNCTION', 0) - + self._makeClosure(gen, 0) self.emitop_int('CALL_FUNCTION', 0) self.emit('BUILD_CLASS') self.storeName(node.name, node.lineno) @@ -661,16 +655,7 @@ inner.accept( gen ) gen.finish() self.set_lineno(node) - frees = gen.scope.get_free_vars_in_parent() - if frees: - for name in frees: - self.emitop('LOAD_CLOSURE', name) - self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_CLOSURE', 0) - else: - self.emitop_code('LOAD_CONST', gen) - self.emitop_int('MAKE_FUNCTION', 0) - + self._makeClosure(gen, 0) # precomputation of outmost iterable qual0 = inner.quals[0] assert isinstance(qual0, ast.GenExprFor) From fijal at codespeak.net Tue Sep 23 18:59:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 18:59:40 +0200 (CEST) Subject: [pypy-svn] r58393 - in pypy/branch/2.5-merge/pypy/interpreter: astcompiler test Message-ID: <20080923165940.1DCF2169FD4@codespeak.net> Author: fijal Date: Tue Sep 23 18:59:39 2008 New Revision: 58393 Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py Log: (fijal, arigo) Proper fix Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/astcompiler/pyassem.py Tue Sep 23 18:59:39 2008 @@ -464,9 +464,7 @@ def depth_MAKE_FUNCTION(argc): return -argc def depth_MAKE_CLOSURE(argc): - if argc == 0: - return -1 - return -argc + return -argc-1 def depth_BUILD_SLICE(argc): if argc == 2: return -1 Modified: pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py Tue Sep 23 18:59:39 2008 @@ -280,3 +280,14 @@ def f(x): return x assert f(666) == 666 + def test_stack_bug(self): + x = """class A: + def initialize(self): + # install all the MultiMethods into the space instance + if isinstance(mm, object): + def make_boundmethod(func=func): + def boundmethod(*args): + return func(self, *args) + """ + exec x in {} + # assert did not crash From fijal at codespeak.net Tue Sep 23 19:02:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 23 Sep 2008 19:02:12 +0200 (CEST) Subject: [pypy-svn] r58394 - in pypy/branch/2.5-merge/pypy/interpreter: astcompiler/test test Message-ID: <20080923170212.47659169FB2@codespeak.net> Author: fijal Date: Tue Sep 23 19:02:11 2008 New Revision: 58394 Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py Log: Move the test to a correct place Modified: pypy/branch/2.5-merge/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/astcompiler/test/test_compiler.py Tue Sep 23 19:02:11 2008 @@ -606,3 +606,17 @@ """) decl = str(decl) + '\n' yield self.simple_test, decl + 'r = test_read(X())', 'r', 42 + + def test_stack_depth_bug(self): + decl = py.code.Source(""" + class A: + def initialize(self): + # install all the MultiMethods into the space instance + if isinstance(mm, object): + def make_boundmethod(func=func): + def boundmethod(*args): + return func(self, *args) + """) + decl = str(decl) + '\n' + yield self.simple_test, decl, 'r', None + Modified: pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py ============================================================================== --- pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py (original) +++ pypy/branch/2.5-merge/pypy/interpreter/test/test_interpreter.py Tue Sep 23 19:02:11 2008 @@ -279,15 +279,3 @@ def test_identity(self): def f(x): return x assert f(666) == 666 - - def test_stack_bug(self): - x = """class A: - def initialize(self): - # install all the MultiMethods into the space instance - if isinstance(mm, object): - def make_boundmethod(func=func): - def boundmethod(*args): - return func(self, *args) - """ - exec x in {} - # assert did not crash From cami at codespeak.net Tue Sep 23 22:26:24 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 23 Sep 2008 22:26:24 +0200 (CEST) Subject: [pypy-svn] r58395 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080923202624.2677C169FE0@codespeak.net> Author: cami Date: Tue Sep 23 22:26:22 2008 New Revision: 58395 Modified: pypy/dist/pypy/lang/gameboy/cpu.py pypy/dist/pypy/lang/gameboy/test/test_cpu_2.py pypy/dist/pypy/lang/gameboy/video.py Log: small changes Modified: pypy/dist/pypy/lang/gameboy/cpu.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/cpu.py (original) +++ pypy/dist/pypy/lang/gameboy/cpu.py Tue Sep 23 22:26:22 2008 @@ -6,12 +6,13 @@ # --------------------------------------------------------------------------- -def process_2_complement(value): +def process_2s_complement(value): # check if the left most bit is set - if (value >> 7) == 1: - return -((~value) & 0xFF) - 1 - else : - return value + #if (value >> 7) == 1: + # return -((~value) & 0xFF) - 1 + #else : + # return value + return (value ^ 0x80) - 128 # --------------------------------------------------------------------------- class AbstractRegister(object): @@ -829,7 +830,7 @@ def get_fetchadded_sp(self): # 1 cycle - offset = process_2_complement(self.fetch()) # 1 cycle + offset = process_2s_complement(self.fetch()) # 1 cycle s = (self.sp.get() + offset) & 0xFFFF self.flag.reset() if (offset >= 0): @@ -869,7 +870,7 @@ def relative_jump(self): # JR +nn, 3 cycles - self.pc.add(process_2_complement(self.fetch())) # 3 + 1 cycles + self.pc.add(process_2s_complement(self.fetch())) # 3 + 1 cycles self.cycles += 1 def relative_conditional_jump(self, cc): Modified: pypy/dist/pypy/lang/gameboy/test/test_cpu_2.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_cpu_2.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_cpu_2.py Tue Sep 23 22:26:22 2008 @@ -662,15 +662,15 @@ assert cpu.flag.get() == 0xFF assert cpu.pc.get() == 0x1234+2 -def test_process_2_complement(): - assert process_2_complement(0x00) == 0 - assert process_2_complement(0xFF) == -1 +def test_process_2s_complement(): + assert process_2s_complement(0x00) == 0 + assert process_2s_complement(0xFF) == -1 for i in range(0x7E): - assert process_2_complement(i) == i + assert process_2s_complement(i) == i for i in range(1, 0x7E): - assert process_2_complement(0xFF - i+1) == -i + assert process_2s_complement(0xFF - i+1) == -i def test_relative_jump(): cpu = get_cpu() Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 23 22:26:22 2008 @@ -3,9 +3,10 @@ constants.LCD Video Display Processor """ import math +import operator from pypy.lang.gameboy import constants from pypy.lang.gameboy.ram import iMemory -from pypy.lang.gameboy.cpu import process_2_complement +from pypy.lang.gameboy.cpu import process_2s_complement # ----------------------------------------------------------------------------- class VideoCallWraper(object): @@ -486,9 +487,6 @@ def get_tile_number(self): return self.tile.id - def set_tile_number(self, tile_number): - self.tile = self.video.tiles[tile_number] - def get_width(self): return 8 @@ -521,6 +519,9 @@ def set_tile_data(self, data): pass + def get_pixel_data(self): + return self.pixel_data + def get_selected_tile_map_space(self): pass # ----------------------------------------------------------------------------- @@ -963,7 +964,8 @@ """ sets one byte of the object attribute memory. The object attribute memory stores the position and seme other - attributes of the sprites + attributes of the sprites, this works only during the v-blank and + the h-blank period. """ self.oam[address - constants.OAM_ADDR] = data & 0xFF #self.update_sprites(address) @@ -975,7 +977,7 @@ def set_vram(self, address, data): """ sets one byte of the video memory. - The video memroy contains the tiles used to display. + The video memory contains the tiles used to display. """ self.vram[address - constants.VRAM_ADDR] = data & 0xFF self.update_tile(address, data) @@ -1017,7 +1019,7 @@ self.draw_pixels_line() def draw_sprites_line_new(self): - sprites_on_line = self.get_active_sprites_on_line(self.line_y) + sprites_on_line = self.get_drawable_sprites_on_line(self.line_y) last_sprite = sprites_on_line[0] last_sprite.draw() @@ -1036,6 +1038,21 @@ found.append(self.sprites[i]) return found + def get_drawable_sprites_on_line(self, line_y): + sprites_on_line = self.get_active_sprites_on_line(self.line_y) + sprites_on_line = self.sort_drawable_sprites(sprites_on_line) + # only 10 sprites per line + return sprites_on_line[:constants.OBJECTS_PER_LINE] + + def sort_drawable_sprites(self, sprites): + """ + returns an ordered list of selected sprites. + The order rules are as following: + 1. order by x -coordinates, lower first + 2. order by id, lower first + """ + return sprites.sort(key=operator.itemgetter("x")) + def draw_sprites_line(self): count = self.scan_sprites() lastx = 176 From cami at codespeak.net Tue Sep 23 23:42:20 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Tue, 23 Sep 2008 23:42:20 +0200 (CEST) Subject: [pypy-svn] r58396 - pypy/dist/pypy/lang/gameboy Message-ID: <20080923214220.513AB16A048@codespeak.net> Author: cami Date: Tue Sep 23 23:42:18 2008 New Revision: 58396 Modified: pypy/dist/pypy/lang/gameboy/constants.py pypy/dist/pypy/lang/gameboy/video.py Log: introduced some more constants for sprite size and video size Modified: pypy/dist/pypy/lang/gameboy/constants.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/constants.py (original) +++ pypy/dist/pypy/lang/gameboy/constants.py Tue Sep 23 23:42:18 2008 @@ -10,6 +10,8 @@ GAMEBOY_SCREEN_WIDTH = 160 GAMEBOY_SCREEN_HEIGHT = 144 +SPRITE_SIZE = 8 + #___________________________________________________________________________ # CATRIGE TYPES # ___________________________________________________________________________ Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Tue Sep 23 23:42:18 2008 @@ -5,6 +5,7 @@ import math import operator from pypy.lang.gameboy import constants +from pypy.lang.gameboy.constants import SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH, GAMEBOY_SCREEN_HEIGHT from pypy.lang.gameboy.ram import iMemory from pypy.lang.gameboy.cpu import process_2s_complement @@ -168,7 +169,7 @@ def emulate_hblank(self): self.video.line_y += 1 self.emulate_hblank_line_y_compare() - if self.video.line_y < 144: + if self.video.line_y < GAMEBOY_SCREEN_HEIGHT: self.video.status.set_mode(2) else: self.emulate_hblank_part_2() @@ -417,6 +418,9 @@ self.hidden = True + def get_data(self): + return [self.y, self.x, self.tile_number, self.get_attributes_and_flags()] + def set_data(self, byte0=-1, byte1=-1, byte2=-1, byte3=-1): """ extracts the sprite data from an oam entry @@ -475,11 +479,18 @@ self.y_flipped = bool(data & (1 << 5)) self.palette_number = bool(data & (1 << 3)) + def get_attributes_and_flags(self): + value = 0 + value += int(self.object_behind_background) << 7 + value += int(self.x_flipped) << 6 + value += int(self.y_flipped) << 5 + value += int(self.palette_number ) << 3 + return value def hide_check(self): - if self.y <= 0 or self.y >= 160: + if self.y <= 0 or self.y >= GAMEBOY_SCREEN_WIDTH: self.hidden = True - elif self.x <= 0 or self.x >= 168: + elif self.x <= 0 or self.x >= GAMEBOY_SCREEN_WIDTH+SPRITE_SIZE: self.hidden = True else: self.hidden = False @@ -488,13 +499,13 @@ return self.tile.id def get_width(self): - return 8 + return SPRITE_SIZE def get_height(self): if self.big_size: - return 2*8 + return 2*SPRITE_SIZE else: - return 8 + return SPRITE_SIZE def overlaps_on_line(self, sprite, line): return False @@ -514,8 +525,11 @@ class Tile(object): def __init__(self): + self.reset() + + def reset(self): pass - + def set_tile_data(self, data): pass @@ -543,7 +557,7 @@ # don't draw window if it was not enabled and not being drawn before if not self.enabled and (data & 0x20) != 0 and \ self.line_y == 0 and self.video.line_y > self.y: - self.line_y = 144 + self.line_y = GAMEBOY_SCREEN_HEIGHT def get_tile_map_space(self): #if (self.control.read() & mask) != 0: @@ -554,7 +568,7 @@ def draw_line(self, line_y): if line_y < self.y or self.x >= 167 or \ - self.line_y >= 144: + self.line_y >= GAMEBOY_SCREEN_HEIGHT: return else: tile_map, tile_data = self.prepare_window_data() @@ -592,7 +606,7 @@ return constants.VRAM_MAP_A def draw_clean_line(self, line_y): - for x in range(8+160+8): + for x in range(8+GAMEBOY_SCREEN_WIDTH+8): self.video.line[x] = 0x00 def draw_line(self, line_y): @@ -636,6 +650,18 @@ def update_tile(self, address, data): # XXX to implement + #self.get_tile(address).set_data(); + pass + + def get_tile(self, address): + # XXX to implement + pass + + def reset_all_tiles(self): + #for tile in self.tile_map_0: + # tile.reset() + #for tile in self.tile_map_1: + # tile.reset() pass def create_sprites(self): @@ -652,17 +678,23 @@ self.oam[address + 3]) def update_sprite(self, address, data): - address -= constants.OAM_ADDR - # address divided by 4 gives the correct sprite, each sprite has 4 - # bytes of attributes - sprite_id = int(math.floor(address / 4)) # XXX why cant I use None here attribute = [-1] * 4 # assign the data to the correct attribute attribute[address % 4] = data - self.sprites[sprite_id].set_data(attribute[0], attribute[1], - attribute[2], attribute[3]) + self.get_sprite(address).set_data(attribute[0], attribute[1], + attribute[2], attribute[3]) + def get_sprite(self, address): + address -= constants.OAM_ADDR + # address divided by 4 gives the correct sprite, each sprite has 4 + # bytes of attributes + return self.sprites[ int(math.floor(address / 4)) ] + + def reset_all_sprites(self): + #for sprite in self.sprites: + # sprite.reset() + pass def reset(self): self.control.reset() @@ -684,10 +716,13 @@ self.dirty = True self.vram = [0] * constants.VRAM_SIZE + self.reset_all_tiles() # Object Attribute Memory self.oam = [0] * constants.OAM_SIZE + self.reset_all_sprites() - self.line = [0] * (8 + 160 + 8) + #XXX remove those dumb helper "objects" + self.line = [0] * (SPRITE_SIZE + GAMEBOY_SCREEN_WIDTH + SPRITE_SIZE) self.objects = [0] * constants.OBJECTS_PER_LINE self.palette = [0] * 1024 @@ -972,6 +1007,7 @@ self.update_sprite(address, data) def get_oam(self, address): + #return self.get_sprite(address).get_data()[address % 4]; return self.oam[address - constants.OAM_ADDR] def set_vram(self, address, data): @@ -983,6 +1019,7 @@ self.update_tile(address, data) def get_vram(self, address): + #self.get_tile(address).get_data()[address % 4] return self.vram[address - constants.VRAM_ADDR] # emulation ---------------------------------------------------------------- @@ -1061,7 +1098,7 @@ x = (data >> 24) & 0xFF flags = (data >> 12) & 0xFF address = data & 0xFFF - if (x + 8 <= lastx): + if (x + SPRITE_SIZE <= lastx): self.draw_object_tile(x, address, flags) else: self.draw_overlapped_object_tile(x, address, flags) @@ -1073,7 +1110,8 @@ for offset in range(0, 4*40, 4): y = self.oam[offset + 0] x = self.oam[offset + 1] - if (y <= 0 or y >= 144 + 16 or x <= 0 or x >= 168): + if (y <= 0 or y >= SPRITE_SIZE + GAMEBOY_SCREEN_HEIGHT + SPRITE_SIZE + or x <= 0 or x >= GAMEBOY_SCREEN_WIDTH + SPRITE_SIZE): continue tile = self.oam[offset + 2] flags = self.oam[offset + 3] @@ -1115,14 +1153,14 @@ self.objects[rightmost] = data def draw_tiles(self, x, tile_map, tile_data): - while x < 168: + while x < GAMEBOY_SCREEN_WIDTH+SPRITE_SIZE: if self.control.background_and_window_lower_tile_data_selected: tile = self.vram[tile_map] else: tile = (self.vram[tile_map] ^ 0x80) & 0xFF self.draw_tile(x, tile_data + (tile << 4)) tile_map = (tile_map & 0x1FE0) + ((tile_map + 1) & 0x001F) - x += 8 + x += SPRITE_SIZE def draw_tile(self, x, address): pattern = self.get_pattern(address) @@ -1182,7 +1220,7 @@ self.update_palette() pixels = self.driver.get_pixels() offset = self.line_y * self.driver.get_width() - for x in range(8, 168, 4): + for x in range(SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH+SPRITE_SIZE, 4): for i in range(0,4): pixels[offset + i] = self.palette[self.line[x + i]] offset += 4 From antocuni at codespeak.net Wed Sep 24 11:02:16 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 24 Sep 2008 11:02:16 +0200 (CEST) Subject: [pypy-svn] r58398 - pypy/extradoc/sprintinfo/october-2008 Message-ID: <20080924090216.70B7A169FFA@codespeak.net> Author: antocuni Date: Wed Sep 24 11:02:14 2008 New Revision: 58398 Modified: pypy/extradoc/sprintinfo/october-2008/people.txt Log: my dates Modified: pypy/extradoc/sprintinfo/october-2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/october-2008/people.txt (original) +++ pypy/extradoc/sprintinfo/october-2008/people.txt Wed Sep 24 11:02:14 2008 @@ -9,7 +9,7 @@ ==================== ============== ============================ Name Arrive/Depart Accomodation ==================== ============== ============================ -Antonio Cuni 5-15 possible ??? +Antonio Cuni 2-sys.maxint private Holger Krekel 5-13th interested in sharing room Maciej Fijalkowski 5-15 possible ??? Armin Rigo 3-inf private From fijal at codespeak.net Wed Sep 24 14:10:42 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 24 Sep 2008 14:10:42 +0200 (CEST) Subject: [pypy-svn] r58402 - pypy/branch/2.5-merge/pypy/lib Message-ID: <20080924121042.E1229168529@codespeak.net> Author: fijal Date: Wed Sep 24 14:10:42 2008 New Revision: 58402 Added: pypy/branch/2.5-merge/pypy/lib/pstats.py - copied unchanged from r58401, pypy/dist/lib-python/modified-2.4.1/pstats.py Log: Put pstats in a correct place From pedronis at codespeak.net Wed Sep 24 15:24:02 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 24 Sep 2008 15:24:02 +0200 (CEST) Subject: [pypy-svn] r58404 - in pypy/build/bot2: . pypybuildbot pypybuildbot/test Message-ID: <20080924132402.DAAA516A05D@codespeak.net> Author: pedronis Date: Wed Sep 24 15:23:59 2008 New Revision: 58404 Added: pypy/build/bot2/wyvern_index.html (contents, props changed) Modified: pypy/build/bot2/pypybuildbot/steps.py pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: (iko, pedronis) - tweak setup to enable branch runs - added failed/skipped counts to the summary page - added some branch support to the summary page - adjusted index.html for wyvern next step: applying these on wyvern proper Modified: pypy/build/bot2/pypybuildbot/steps.py ============================================================================== --- pypy/build/bot2/pypybuildbot/steps.py (original) +++ pypy/build/bot2/pypybuildbot/steps.py Wed Sep 24 15:23:59 2008 @@ -56,8 +56,8 @@ command = ["python", "py/bin/py.svnwcrevert", "-p.buildbot-sourcedata", "."], )) - self.addStep(source.SVN("https://codespeak.net/svn/pypy/" - "branch/pypy-pytrunk")) + self.addStep(source.SVN(baseURL="http://codespeak.net/svn/pypy/", + defaultBranch="branch/pypy-pytrunk")) self.addStep(shell.ShellCommand( description="pytest", command=["python", "testrunner/runner.py", Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Sep 24 15:23:59 2008 @@ -64,7 +64,7 @@ return (self.key, namekey) def get_run_stdios(self): - return {self.key: self._run_stdio} + return {self.key: (self, self._run_stdio)} class RevisionOutcomeSetCache(object): CACHESIZE = 10 @@ -196,22 +196,32 @@ def make_stdio_anchors_for(self, outcome_set): anchors = [] stdios = sorted(outcome_set.get_run_stdios().items()) - for cachekey, url in stdios: + for cachekey, (run, url) in stdios: builder = cachekey[0] anchors.append(' ') - anchors.append(html.a(builder, href=url)) + text = "%s [%d failed; %d skipped]" % (builder, + len(run.failed), + len(run.skipped)) + anchors.append(html.a(text, href=url)) return anchors + + def add_title(self, title): + self.sections.append(html.h2(title)) def add_section(self, outcome_sets): + revs = sorted(outcome_set.revision for outcome_set in outcome_sets) by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set in outcome_sets) lines = [] + + align = 2*len(revs)-1+len(str(revs[-1])) def bars(): return ' |'*len(lines) for rev, outcome_set in by_rev: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) line = ["%s %d" % (bars(),rev)] + line.append((align-len(line[0]))*" ") line.append(self.make_stdio_anchors_for(outcome_set)) line.append('\n') lines.append(line) @@ -242,7 +252,8 @@ self.sections.append(section) def add_no_revision_builds(self, status, no_revision_builds): - section = html.div(html.p("builds aborted without getting a revision")) + section = html.div(html.p("builds aborted without getting" + " a revision:")) for build in no_revision_builds: builderName = build.getBuilder().getName() @@ -305,40 +316,65 @@ return default class Summary(HtmlResource): - title="Summary" # xxx def __init__(self): HtmlResource.__init__(self) self.putChild('longrepr', LongRepr()) + def getTitle(self, request): + status = self.getStatus(request) + return "%s: summaries of last %d revisions" % (status.getProjectName(), + N) + + @staticmethod + def _prune_revs(revs, cutnum): + if len(revs) > cutnum: + for rev in sorted(revs.keys())[:-cutnum]: + del revs[rev] + def recentRevisions(self, status): - # xxx branches - revs = {} - no_revision_builds = [] + branches = {} + for builderName in status.getBuilderNames(): builderStatus = status.getBuilder(builderName) - for build in builderStatus.generateFinishedBuilds(num_builds=N): + for build in builderStatus.generateFinishedBuilds(num_builds=5*N): + branch = getProp(build, 'branch') got_rev = getProp(build, 'got_revision', None) + + revs, no_revision_builds = branches.setdefault(branch, + ({}, [])) + if got_rev is None: no_revision_builds.append(build) else: rev = int(got_rev) revBuilds = revs.setdefault(rev, {}) - if builderName not in revBuilds: # pick the most recent or ? + # pick the most recent or ? + if builderName not in revBuilds: key = (builderName, build.getNumber()) outcome_set = outcome_set_cache.get(status, key) revBuilds[builderName] = outcome_set + + for branch, (revs, no_revision_builds) in branches.items(): + self._prune_revs(revs, N) - return revs, no_revision_builds + return branches def body(self, request): status = self.getStatus(request) - - revs, no_revision_builds = self.recentRevisions(status) - outcome_sets = [] - for rev, by_build in revs.items(): - outcome_sets.append(GatherOutcomeSet(by_build)) + page = SummaryPage() - page.add_section(outcome_sets) - page.add_no_revision_builds(status, no_revision_builds) + + branches = self.recentRevisions(status) + + for branch, (revs, no_revision_builds) in sorted(branches.iteritems()): + outcome_sets = [] + for rev, by_build in revs.items(): + outcome_sets.append(GatherOutcomeSet(by_build)) + if branch is None: + branch = "" + page.add_title(branch) + page.add_section(outcome_sets) + page.add_no_revision_builds(status, no_revision_builds) + return page.render() Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Wed Sep 24 15:23:59 2008 @@ -198,11 +198,21 @@ res = goutcome_top.get_longrepr(('what', 'foo', 'a.b', 'test_one')) assert res == '' - def test_colsizes(self): - failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'), - ('ab', '', 'cd')] +def test_colsizes(): + failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'), + ('ab', '', 'cd')] + + res = summary.colsizes(failed) + + assert res == [2,3,2] - res = summary.colsizes(failed) +def test__prune_revs(): + revs = dict(zip(range(100), range(100, 200))) - assert res == [2,3,2] + summary.Summary._prune_revs(revs, 4) + + assert len(revs) == 4 + + assert revs == {99: 199, 98: 198, 97: 197, 96: 196} + Added: pypy/build/bot2/wyvern_index.html ============================================================================== --- (empty file) +++ pypy/build/bot2/wyvern_index.html Wed Sep 24 15:23:59 2008 @@ -0,0 +1,35 @@ + + + + +Welcome to Wyvern's PyPy Buildbot + + + +

Welcome to Wyvern's PyPy Buildbot!

+ + + + + From pedronis at codespeak.net Wed Sep 24 15:30:02 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 24 Sep 2008 15:30:02 +0200 (CEST) Subject: [pypy-svn] r58405 - pypy/build/bot2 Message-ID: <20080924133002.75C4F16A0B3@codespeak.net> Author: pedronis Date: Wed Sep 24 15:30:01 2008 New Revision: 58405 Modified: pypy/build/bot2/TODO Log: (iko, pedronis) update todo Modified: pypy/build/bot2/TODO ============================================================================== --- pypy/build/bot2/TODO (original) +++ pypy/build/bot2/TODO Wed Sep 24 15:30:01 2008 @@ -1,4 +1,3 @@ -- tweak setup for operating on branches - buildbot Nightly scheduler that can pick a uniform revision server side and with support for specifying possibly branches From witulski at codespeak.net Wed Sep 24 18:20:52 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Wed, 24 Sep 2008 18:20:52 +0200 (CEST) Subject: [pypy-svn] r58409 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080924162052.1C20C168527@codespeak.net> Author: witulski Date: Wed Sep 24 18:20:50 2008 New Revision: 58409 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: Push and pop works (new test added) CMP has an bug with imm32 values (e.g. CMP 42,42 == false) JNE works (new Test added) neg. Displacement can now be convertet some test modified to test if there is a problem with neg. values Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Wed Sep 24 18:20:50 2008 @@ -122,14 +122,36 @@ # FIXME: rexB? if isinstance(arg1,Register64): rexB, modrm1 = self.get_register_bits(arg1.reg) - + # exchange the two arguments (modrm2/modrm1) if isinstance(arg2,Immediate64): + new_opcode = hex(int(opcode,16)+modrm1) + assert len(new_opcode[2:len(new_opcode)]) == 2 self.write_rex_byte(rexW, rexR, rexX, rexB) - self.write(opcode+chr(modrm1)) + self.write(new_opcode[2:len(new_opcode)]) self.writeImm64(arg2.value) return quadreg_instr +def make_one_operand_instr_with_alternate_encoding(W = None, R = None, X = None, B = None, opcode =None, md1 = None, md2 = None): + def quadreg_instr(self, arg1): + # move the parameter + # to the inner function + modrm1 = md1 + modrm2 = md2 + rexW = W + rexR = R + rexX = X + rexB = B + # TODO: other cases e.g memory as operand + # FIXME: rexB? + if isinstance(arg1,Register64): + rexB, modrm1 = self.get_register_bits(arg1.reg) + new_opcode = hex(int(opcode,16)+modrm1) + assert len(new_opcode[2:len(new_opcode)]) == 2 + self.write_rex_byte(rexW, rexR, rexX, rexB) + self.write(new_opcode[2:len(new_opcode)]) + return quadreg_instr + class X86_64CodeBuilder(object): """ creats x86_64 opcodes""" def write(self, data): @@ -143,7 +165,7 @@ # The opcodes differs depending on the operands # Params: - # W (64bit Operands), R (permits R8-R15), X (extend Index field), B (extend r/m field), + # W (64bit Operands), R (extends reg field), X (extend Index(SIB) field), B (extends r/m field, Base(SIB) field, opcode reg field), # Opcode, mod, modrm1, modrm2, tttn(JUMPS), extraopcode # FIXME: rexB is set @@ -153,6 +175,7 @@ _AND_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x21", 3, None, None) # FIXME: rexB is set + # maybe a bug _CMP_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 1, "\x81", 3, None, 7) _CMP_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x39", 3, None, None) # FIXME: rexB is set @@ -164,7 +187,7 @@ _MOV_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, None, "\xC7", 3, None, 0) _MOV_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x89", 3, None, None) - _MOV_QWREG_IMM64 = make_two_operand_instr_with_alternate_encoding(1,0,0,None,"\xB8",None,None) + _MOV_QWREG_IMM64 = make_two_operand_instr_with_alternate_encoding(1,0,0,None,"B8",None,None) _IMUL_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x0F", 3, None, None, None, "\xAF") _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") @@ -181,6 +204,9 @@ _POP_QWREG = make_one_operand_instr( 1, 0, 0, None, "\x8F", 3, None, 0) _PUSH_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xFF", 3, None, 6) + #_POP_QWREG = make_one_operand_instr_with_alternate_encoding(1,0,0,None,"58",None,None) + #_PUSH_QWREG = make_one_operand_instr_with_alternate_encoding(1,0,0,None,"50",None,None) + _SETE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,4) _SETG_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,15) _SETGE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,13) @@ -223,21 +249,23 @@ # op1 is and 32bit displ def JNE(self,op1): + print hex(self.tell()),": JNE to",hex(op1) self.write("\x0F") self.write("\x85") self.writeImm32(op1) - print self.tell(),": JNE to",op1 + def OR(self, op1, op2): method = getattr(self, "_OR"+op1.to_string()+op2.to_string()) method(op1, op2) + #fixme:none def POP(self, op1): method = getattr(self, "_POP"+op1.to_string()) method(op1) def PUSH(self, op1): - method = getattr(self, "_POP"+op1.to_string()) + method = getattr(self, "_PUSH"+op1.to_string()) method(op1) def MOV(self, op1, op2): @@ -302,16 +330,24 @@ def get_register_bits_8Bit(self, register): return REGISTER_MAP_8BIT[register] - # TODO: sign extention? + # Parse the integervalue to an charakter # and write it def writeImm32(self, imm32): x = hex(imm32) if x[0]=='-': - x = self.cast_to_neg_hex(int(x,16)) - # parse to string and cut "0x" off - # fill with zeros if to short - y = "0"*(10-len(x))+x[2:len(x)] + # parse to string and cut "0x" off + # fill with Fs if to short + #print "x before:",x + x = self.cast_neg_hex32(int(x,16)) + #print "x after:",x + y = "F"*(8-len(x))+x[0:len(x)] + else: + # parse to string and cut "0x" off + # fill with zeros if to short + y = "0"*(10-len(x))+x[2:len(x)] + #print "y:",y + #y = "00000000" assert len(y) == 8 self.write(chr(int(y[6:8],16))) self.write(chr(int(y[4:6],16))) @@ -347,10 +383,9 @@ byte = mod << 6 | (reg << 3) | rm self.write(chr(byte)) - # TODO: write me - def cast_to_neg_hex(self,a_hex): - #FIXME: segfault to compliment - x = hex(18446744073709551616 +a_hex) - y = x[16:len(x)-1] + # TODO: write comment + def cast_neg_hex32(self,a_int): + x = hex(int("FFFFFFFF",16)+1 +a_int) + y = x[2:len(x)] return y - + Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Wed Sep 24 18:20:50 2008 @@ -88,19 +88,19 @@ @specialize.arg(1) def genop1(self, opname, gv_arg): genmethod = getattr(self, 'op_' + opname) - # print self.mc.tell(),":",opname + print hex(self.mc.tell()),":",opname," ",gv_arg.to_string() return genmethod(gv_arg) @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): genmethod = getattr(self, 'op_' + opname) - # print self.mc.tell(),":",opname + print hex(self.mc.tell()),":",opname," ",gv_arg1.to_string()," ",gv_arg2.to_string() return genmethod(gv_arg1, gv_arg2) op_int_add = make_two_argument_method("ADD") op_int_and = make_two_argument_method("AND") op_int_dec = make_one_argument_method("DEC") - op_int_inc = make_one_argument_method("INC") + op_int_inc = make_one_argument_method("INC") #for debuging op_int_mul = make_two_argument_method("IMUL") op_int_neg = make_one_argument_method("NEG") op_int_not = make_one_argument_method("NOT") @@ -108,15 +108,16 @@ op_int_push = make_one_argument_method("PUSH") op_int_pop = make_one_argument_method("POP") op_int_sub = make_two_argument_method("SUB") - op_int_xor = make_two_argument_method("XOR") + op_int_xor = make_two_argument_method("XOR") #FIXME: can only jump 32bit - def jump_if_true(self, gv_condition, args_for_jump_gv): + #FIXME: -6 displacement: the displ+ rip of next instr + def jump_if_true(self, gv_condition, args_for_jump_gv): targetbuilder = Builder() self.mc.CMP(gv_condition, Immediate32(0)) displ = self.calc_32bit_displacement(self.mc.tell(),targetbuilder.mc.tell()) - self.mc.JNE(displ) - #targetbuilder.come_from(self.mc, 'JNE') + self.mc.JNE(displ-6) + #targetbuilder.come_from(self.mc, 'JNE') return targetbuilder def op_int_gt(self, gv_x, gv_y): @@ -161,19 +162,21 @@ def finish_and_return(self, sigtoken, gv_returnvar): #self.mc.write("\xB8\x0F\x00\x00\x00") + print hex(self.mc.tell()),": RET" self._open() self.mc.MOV(Register64("rax"), gv_returnvar) self.mc.RET() self._close() + # if the label is greater than 32bit # it must be in a register def finish_and_goto(self, outputargs_gv, target): + print hex(self.mc.tell()),": JMP to",hex(target.startaddr) self._open() gv_x = self.allocate_register() self.mc.MOV(gv_x,Immediate64(target.startaddr)) self.mc.JMP(gv_x) - print self.mc.tell(),": JMP to",target.startaddr self._close() def allocate_register(self, register=None): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Wed Sep 24 18:20:50 2008 @@ -1,6 +1,6 @@ import py from pypy.rpython.lltypesystem import lltype -from pypy.jit.codegen.x86_64.rgenop import RX86_64GenOp +from pypy.jit.codegen.x86_64.rgenop import RX86_64GenOp, Label from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTestsDirect #from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTestsCompile @@ -9,6 +9,47 @@ def skip(self): py.test.skip("not implemented yet") + +def make_push_pop(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) + builder, gv_push_pop, [gv_x] = rgenop.newgraph(sigtoken, "push_pop") + builder.start_writing() + gv_x = builder.genop1("int_push", gv_x) + gv_x = builder.genop1("int_inc", gv_x) + gv_x = builder.genop1("int_inc", gv_x) + gv_x = builder.genop1("int_pop", gv_x) + builder.finish_and_return(sigtoken, gv_x) + builder.end() + return gv_push_pop + +#FIXME: Jumps ever +def make_jne(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_jne, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "jne") + builder.start_writing() + gv_z = builder.genop2("int_gt", gv_x, gv_y) + builder.mc.CMP(gv_z, rgenop.genconst(1)) + builder.mc.JNE(6) + builder.genop1("int_inc",gv_y)#not executed if x<=y + builder.genop1("int_inc",gv_y)#not executed if x<=y + builder.genop1("int_inc",gv_y) + builder.finish_and_return(sigtoken, gv_y) + builder.end() + return gv_jne + +def make_jmp(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_jmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "jmp") + builder.start_writing() + gv_z = builder.genop2("int_add", gv_x, gv_y) + builder.finish_and_goto("",Label(builder.mc.tell()+6, [], 0)) + gv_w = builder.genop2("int_add", gv_z, gv_y) + gv_v = builder.genop2("int_sub", gv_w, gv_x) + gv_a = builder.genop1("int_inc",gv_v) + gv_b = builder.genop1("int_inc",gv_a) + builder.finish_and_return(sigtoken, gv_a) + builder.end() + return gv_jmp def make_one_op_instr(rgenop, instr_name): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed], lltype.Signed)) @@ -30,12 +71,16 @@ builder.end() return gv_bool_op -def make_cmp(rgenop, which_cmp): +def make_cmp(rgenop, which_cmp, const=None): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_cmp, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "cmp") builder.start_writing() - gv_result = builder.genop2(which_cmp, gv_x, gv_y) + if not const == None: + gv_result = builder.genop2(which_cmp, gv_x, rgenop.genconst(const)) + else: + gv_result = builder.genop2(which_cmp, gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_cmp @@ -95,6 +140,20 @@ fnptr = self.cast(mul_function,2) res = fnptr(1200,300) assert res == 360000 + res = fnptr(12345,42) + assert res == 518490 + res = fnptr(12345,-42) + assert res == -518490 + res = fnptr(-12345,42) + assert res == -518490 + res = fnptr(-12345,-42) + assert res == 518490 + res = fnptr(-12345,-9876) + assert res == 121919220 + res = fnptr(-12345,9876) + assert res == -121919220 + res = fnptr(12345,-9876) + assert res == -121919220 def test_greater(self): rgenop = self.RGenOp() @@ -138,6 +197,8 @@ assert res == 1 res = fnptr(4,0) assert res == 0 + res = fnptr(0,-4) + assert res == 0 res = fnptr(-4,0) assert res == 1 @@ -153,11 +214,23 @@ assert res == 1 res = fnptr(4,0) assert res == 1 + res = fnptr(512,256) + assert res == 1 + res = fnptr(256,512) + assert res == 0 + res = fnptr(-4,253) #-4>=253 + assert res == 0 # false res = fnptr(-4,0) assert res == 0 def test__equal(self): rgenop = self.RGenOp() + cmp_function = make_cmp(rgenop, "int_eq",42) + fnptr = self.cast(cmp_function,1) + res = fnptr(42) + assert res == 1 + res = fnptr(23) + assert res == 0 cmp_function = make_cmp(rgenop, "int_eq") fnptr = self.cast(cmp_function,2) res = fnptr(3,4) # 3==4? @@ -170,6 +243,12 @@ assert res == 0 res = fnptr(-4,0) assert res == 0 + res = fnptr(252,-4) + assert res == 0 + res = fnptr(-4,252) + assert res == 0 + res = fnptr(244,756) + assert res == 0 def test_not_equal(self): rgenop = self.RGenOp() @@ -264,7 +343,29 @@ result = fnptr(-43) assert result == 42 - #TODO: Test push/pop + # if x>y: + # return y+1 + # else: + # return y+3 + def test_jne(self): + jne_func = make_jne(self.RGenOp()) + fnptr = self.cast(jne_func,2) + result = fnptr(4,1) + assert result == 4 + result = fnptr(1,4) + assert result == 5 + + #def test_jmp(self): + # jmp_function = make_jmp(self.RGenOp()) + # fnptr = self.cast(jmp_function,2) + # result = fnptr(1,2) + # assert result == 5 + + def test_push_pop(self): + pp_func = make_push_pop(self.RGenOp()) + fnptr = self.cast(pp_func,1) + result = fnptr(1) + assert result == 1 test_directtesthelper_direct = skip test_dummy_compile = skip @@ -275,7 +376,7 @@ test_dummy_direct = skip test_largedummy_direct = skip test_branching_direct = skip - ##test_goto_direct = skip## + test_goto_direct = skip## test_if_direct = skip test_switch_direct = skip test_large_switch_direct = skip Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Wed Sep 24 18:20:50 2008 @@ -21,20 +21,24 @@ return builder, fp, inputargs_gv, token class TestSimple(): - + def test_add_big_num(self): builder, fp, inputargs_gv, token = make_testbuilder(2) genv0 = inputargs_gv[0] #the first argument "location" genv1 = inputargs_gv[1] genv_result = builder.genop2("int_add", genv0, genv1) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) + num = fp(1280, 20) + assert num == 1300 + num = fp(1280, -80) + assert num == 1200 num = fp(1280, 1000) assert num == 2280 print num def test_add_twice(self): builder, fp, inputargs_gv, token = make_testbuilder(2) - genv0 = inputargs_gv[0] #the first argument "place" + genv0 = inputargs_gv[0] #the first argument "location" genv1 = inputargs_gv[1] genv2 = builder.genop2("int_add", genv0, genv1) genv_result = builder.genop2("int_add", genv2, genv1) @@ -49,7 +53,33 @@ assert result == -4 result = fp(0,-4) # 0+(-4)+(-4) = -8 assert result == -8 + result = fp(1280,500) # 1280+500+500=2280 + assert result == 2280 + result = fp(0,252) # 0+252+252= 504 + assert result == 504 #==0000:0001:1111:1000 + def test_tripple_add(self): + builder, fp, inputargs_gv, token = make_testbuilder(2) + genv0 = inputargs_gv[0] + genv1 = inputargs_gv[1] + genv2 = builder.genop2("int_add", genv0, genv1) + genv3 = builder.genop2("int_add", genv2, genv1) + genv_result = builder.genop2("int_add", genv3, genv1) + builder.finish_and_return(token, genv_result) + result = fp(4, 6) # 4+6+6+6= 22 + assert result == 22 + result = fp(2,12) # 2+12+12+12= 38 + assert result == 38 + result = fp(10,-2) # 10+(-2)+(-2)+(-2) = 4 + assert result == 4 + result = fp(-4,0) # -4 +0+0+0 = -4 + assert result == -4 + result = fp(0,-4) # 0+(-4)+(-4)+(-4) = -12 + assert result == -12 + result = fp(1280,500) # 1280+500+500+500=2780 + assert result == 2780 + result = fp(0,252) # 0+252+252= 756 + assert result == 756 #==0000:0001:1111:1000 def test_add(self): builder, fp, inputargs_gv, token = make_testbuilder(2) @@ -69,7 +99,7 @@ builder.finish_and_return(token, genv_result) ten = fp(-4, -6) assert ten == -10 - print ten + print ten four = fp(-4,0) assert four == -4 print four @@ -86,11 +116,32 @@ def test_add_imm32(self): builder, fp, inputargs_gv, token = make_testbuilder(1) genv0 = inputargs_gv[0] #the first argument "location" + genv_result = builder.genop2("int_add", genv0, rgenop.genconst(-100000)) #creates the addition and returns the place(register) of the result in genv_result + builder.finish_and_return(token, genv_result) + num = fp(-1000) # -1000+(-100000) = -101000 + assert num == -101000 + print num + num = fp(1000) # 1000+(-100000) = -99000 + assert num == -99000 + print num + num = fp(50) # 50+(-100000) = -99950 + assert num == -99950 + print num + num = fp(-1024) # -1024+(-100000) = -1124 + assert num == -101024 + print num + builder, fp, inputargs_gv, token = make_testbuilder(1) + genv0 = inputargs_gv[0] #the first argument "location" genv_result = builder.genop2("int_add", genv0, rgenop.genconst(1000)) #creates the addition and returns the place(register) of the result in genv_result builder.finish_and_return(token, genv_result) num = fp(1111) # 1111+1000 = 2111 assert num == 2111 print num + num = fp(-100) # -100+1000 = 900 + assert num == 900 + print num + + def test_ret(self): builder, fp, inputargs_gv, token = make_testbuilder(1) From hpk at codespeak.net Thu Sep 25 11:41:46 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 25 Sep 2008 11:41:46 +0200 (CEST) Subject: [pypy-svn] r58423 - in pypy/build/benchmem: . benchmark testing Message-ID: <20080925094146.595F316A053@codespeak.net> Author: hpk Date: Thu Sep 25 11:41:42 2008 New Revision: 58423 Added: pypy/build/benchmem/benchmark/sizes.py pypy/build/benchmem/readme.txt Removed: pypy/build/benchmem/benchmark/create_recursive_tuples.py pypy/build/benchmem/benchmark/empty_instances.py pypy/build/benchmem/benchmark/linked_list_with_floats.py pypy/build/benchmem/benchmark/list_of_instances_with_ints.py pypy/build/benchmem/benchmark/lists.py pypy/build/benchmem/benchmark/simple_linked_instances.py Modified: pypy/build/benchmem/report.py pypy/build/benchmem/runbench.py pypy/build/benchmem/testing/test_benchtool.py Log: * put all "size" benchmarks into sizes.py * add a readme.txt with some XXX * add functional tests for runbench and reporting * add option parsing to reporting Added: pypy/build/benchmem/benchmark/sizes.py ============================================================================== --- (empty file) +++ pypy/build/benchmem/benchmark/sizes.py Thu Sep 25 11:41:42 2008 @@ -0,0 +1,60 @@ + +def bench_list_of_None(iter1): + l = [None for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_int(iter1): + l = [int(i+1000) for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_emptylist(iter1): + l = [[] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_with_int(iter1): + l = [[i] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_with_float(iter1): + l = [[float(i)] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_emptydict(iter1): + l = [{} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_intkey(iter1): + l = [{i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_empty_instances_old(iter1): + class A: + pass + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_empty_instances_new(iter1): + class A(object): + pass + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_linked_instances(iter1): + class A(object): + def __init__(self, prev): + self.prev = prev + l = [] + prev = None + for i in range(iter1): + x = A(prev) + l.append(prev) + prev = x + + checkpoint(collect=True) + +def bench_recursive_list(iter1): + x = [] + for i in range(iter1): + x = [x] + checkpoint(collect=True) + Added: pypy/build/benchmem/readme.txt ============================================================================== --- (empty file) +++ pypy/build/benchmem/readme.txt Thu Sep 25 11:41:42 2008 @@ -0,0 +1,26 @@ +memory benchmarks running/reporting +----------------------------------------- + +``runbench.py`` runs all available and stores resulting information in a log file. + +``report.py`` takes a logfile and produces reporting information. + +In order to run benchmarks with multiple python interpreters:: + + python runbench.py -e python2.5,pypy-c-opt3 + + +Benchmark types / files +------------------------- + +sizes.py contains explicitely checkpointed benchmarks used to measure sizes of several object types and constellations. + all 'bench_' functions take an NUMITER parameter which indicates how often an object + structure should be put in a list. This parameter can be set from ``runbench.py`` command line. + +XXX In order to run interpreter size benchmarks:: + + XXX python runbench.py -e python2.5,pypy-c-opt3 --interpsize + +XXX todo: +app*.py contains time-checkpointed benchmarks used to measure memory usage of an application scenario. + --runwithparams=500,1000,1500 will run application benchmarks with with several paramat Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Thu Sep 25 11:41:42 2008 @@ -9,6 +9,10 @@ import py, os import smaps, runbench +parser = py.compat.optparse.OptionParser(usage=__doc__) +parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", + help="logfile to read results from") + def asciitable(table): colmax = [] for row in table: @@ -88,9 +92,11 @@ os.system("gnuplot gnuplotcmd") if __name__ == "__main__": - benchlog = py.path.local("bench.log") + options, args = parser.parse_args() + + benchlog = py.path.local(options.benchlog) reader = runbench.LogReader() - reader.parse_logfile(benchlog) + reader.parse(benchlog) #maxtable_overview(reader) checkpointdetails(reader) Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Thu Sep 25 11:41:42 2008 @@ -13,31 +13,26 @@ mydir = py.magic.autopath().dirpath() benchmarkdir = mydir.join("benchmark") - -class BenchRunner(object): +class BenchRunner: SEPBENCH = "=" * 80 - - def __init__(self, executable, benchlog): - self.benchlogpath = py.path.local(benchlog) - self.logstream = self.benchlogpath.open("a") + +class BenchRunnerSizes(BenchRunner): + def __init__(self, executable, fsname, logpath, options): self.executable = executable + self.benchpath = benchmarkdir.join(fsname) + assert self.benchpath.check() + self.logpath = py.path.local(logpath) + self.logstream = self.logpath.open("a") self.tmpdir = py.path.local.make_numbered_dir(prefix="bench") + self.options = options def log(self, *args): print " ".join(map(str, args)) - def _preparebench(self, path, args): - path = py.path.local(path) - for name, obj in vars(path.pyimport()).items(): - if name.startswith("bench") and callable(obj): - break # xxx only first bench function is considered - else: - raise LookupError("no benchmark found in %s" %(path,)) - - arglist = ",".join(map(str, args)) - source = py.code.Source(path.read(), """ + def makebench(self, name): + arglist = str(self.options.numiter) + source = py.code.Source(self.benchpath.read(), """ import gc - def write(c): sys.stdout.write(c) sys.stdout.flush() @@ -47,35 +42,47 @@ gc.collect() write("c") sys.stdin.read(1) + if __name__ == "__main__": import os, sys, gc pid = os.getpid() write(str(pid) + "\\n") checkpoint(collect=True) - %s(checkpoint, %s) + %s(%s) checkpoint(collect=True) write("F") sys.stdin.close() """ %(name, arglist)) - p = self.tmpdir.join(path.basename) + p = self.tmpdir.join(self.benchpath.basename) p.write(source) return p def write_benchheader(self, benchname, args): print >>self.logstream, self.SEPBENCH print >>self.logstream, "#executable=%r" %(str(self.executable ),) + print >>self.logstream, "#benchpath=%r" %(self.benchpath.basename,) print >>self.logstream, "#benchname=%r" %(benchname,) print >>self.logstream, "#benchargs=%r" %(args,) - def run_checkpointed_bench(self, filename, args): - benchpyfile = self._preparebench(filename, args) - self.log("created", benchpyfile) + def getnames(self): + l = [] + for name, obj in vars(self.benchpath.pyimport()).items(): + if name.startswith("bench") and callable(obj): + l.append(name) + return l + + def run(self): + for name in self.getnames(): + self.run_checkpointed(name) + + def run_checkpointed(self, name): + benchpyfile = self.makebench(name) + #self.log("created", benchpyfile) cmd = "%s -u %s" %(self.executable, benchpyfile) - self.log("exec", cmd) + self.log("running %s(%s)" %(name, self.options.numiter)) stdout, stdin = os.popen2(cmd) pid = int(stdin.readline()) - - self.write_benchheader(benchpyfile.basename, args) + self.write_benchheader(name, self.options.numiter) rec = smaps.SmapsRecorder(pid, self.logstream) self.interact_with_child_checkpoints(rec, stdout, stdin) @@ -103,7 +110,7 @@ def __init__(self): self.name2results = {} - def parse_logfile(self, logpath): + def parse(self, logpath): f = logpath.open() for result in BenchmarkResult.parse(f): #print "parsed", result @@ -132,9 +139,12 @@ if not line.strip(): break if not line.startswith("#executable"): - print "ignoring", line + if line != BenchRunner.SEPBENCH: + print "ignoring", line continue + # see write_benchheader executable = eval(line.split("=", 1)[1]) + benchbasename = eval(f.readline().split("=", 1)[1]) benchname = eval(f.readline().split("=", 1)[1]) benchargs = eval(f.readline().split("=", 1)[1]) @@ -184,35 +194,20 @@ help="logfile for recording benchmark measurements") parser.add_option("-a", "--append", action="store_true", dest="append", default=False, help="append to logfile") -parser.add_option("-c", "--coefficient", action="store", dest="coeff", - default=1, help="Coefficient of number of iterations") - -def getbenchfiles(options, args): - if args: - benchfiles = [py.path.local(x) for x in args] - else: - benchfiles = [x for x in benchmarkdir.listdir('*.py') - if x.basename[0] != "_"] - for x in benchfiles: - if not x.check(): - raise LookupError("%s does not exist" %(x,)) - assert benchfiles - return benchfiles +parser.add_option("-n", "--numiter", action="store", dest="numiter", + default=100000, help="number of iterations") def getbenchlog(options): benchlog = options.benchlog if benchlog is None: benchlog = "bench.log" - return py.path.local(benchlog) - - -if __name__ == '__main__': - (options, args) = parser.parse_args() - - names = getbenchfiles(options, args) - benchlog = getbenchlog(options) + benchlog = py.path.local(benchlog) if not options.append and benchlog.check(): benchlog.remove() + return benchlog + +def getexecutables(options): + l = [] for executable in options.executable.split(","): if not executable: continue @@ -221,8 +216,22 @@ p = py.path.local.sysfind(executable) if not p.check(): raise SystemExit("could not find %r"% (executable)) - runner = BenchRunner(executable, benchlog) - for name in names: - iter1 = int(100000*float(options.coeff)) - runner.run_checkpointed_bench(name, (iter1,)) + l.append(executable) + return l + +def getrunnerclass(fsname): + if fsname == "sizes.py": + return BenchRunnerSizes + +if __name__ == '__main__': + (options, args) = parser.parse_args() + + benchlog = getbenchlog(options) + if not args: + args = ("sizes.py",) + for executable in getexecutables(options): + for fsname in args: + Runner = getrunnerclass(fsname) + runner = Runner(executable, fsname, benchlog, options) + runner.run() print "bench results append to -->>>", benchlog Modified: pypy/build/benchmem/testing/test_benchtool.py ============================================================================== --- pypy/build/benchmem/testing/test_benchtool.py (original) +++ pypy/build/benchmem/testing/test_benchtool.py Thu Sep 25 11:41:42 2008 @@ -10,7 +10,7 @@ def test_smapsrecorder(): tmpdir = py.test.ensuretemp("smapsrecorder") - logpath = tmpdir.join("logfile") + logpath = tmpdir.join("benchlog") pid = os.getpid() f = logpath.open("w") rec = smaps.SmapsRecorder(pid=pid, stream=f) @@ -22,27 +22,38 @@ s = logpath.read() assert s.count(smaps.SmapsRecorder.SEPSNAPSHOT) == 2 -def test_benchrunner_functional(): +def test_sizes_runbench_and_read_results(): tmpdir = py.test.ensuretemp("benchrunner") - log=tmpdir.join("log") - def checker(path, *args): - if log.check(): - log.remove() - runner = runbench.BenchRunner(executable="python2.5", benchlog=log) - runner.run_checkpointed_bench(path, args) - assert log.check() - benchresult = runbench.LogReader() - benchresult.parse_logfile(log) - #assert reader.executable - #assert reader.executable - assert len(benchresult.name2results) == 1 - results = benchresult.name2results.values()[0] - assert len(results) == 1 + benchlog=tmpdir.join("log") + class options: + numiter = 10 + runner = runbench.BenchRunnerSizes("python2.5", "sizes.py", benchlog, options) + assert runner.benchpath.basename == "sizes.py" + runner.run() + benchresult = runbench.LogReader() + benchresult.parse(benchlog) + + names = runner.getnames() + assert len(benchresult.name2results) == len(names) + for results in benchresult.name2results.values(): + assert len(results) ==1 assert len(results[0].snapshots) == 2 + 1 - for path in runbench.benchmarkdir.listdir("*.py"): - if path.basename[0] != "_": - yield checker, path, 10 +def test_runbench_functional(): + script = py.path.local(runbench.__file__).dirpath("runbench.py") + output = py.process.cmdexec("python %s --numiter=10" %(script)) + +def test_report_functional(): + tmpdir = py.test.ensuretemp("test_report_functional") + script = py.path.local(runbench.__file__).dirpath("runbench.py") + benchlog = tmpdir.join("benchlog") + py.process.cmdexec("%s --benchlog=%s --numiter=10" %(script, benchlog)) + report = script.dirpath("report.py") + old = tmpdir.chdir() + try: + py.process.cmdexec("%s --benchlog %s" %(report, benchlog,)) + finally: + old.chdir() def test_log_mapping(): s = py.std.textwrap.dedent("""\ @@ -114,3 +125,4 @@ val1 = getattr(snap.mappings[0], name) val2 = getattr(snap.mappings[1], name) assert sumval == val1 + val2 + From hpk at codespeak.net Thu Sep 25 12:18:51 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 25 Sep 2008 12:18:51 +0200 (CEST) Subject: [pypy-svn] r58425 - pypy/build/benchmem Message-ID: <20080925101851.C285D16A0CD@codespeak.net> Author: hpk Date: Thu Sep 25 12:18:51 2008 New Revision: 58425 Modified: pypy/build/benchmem/report.py Log: unscramble code a bit (but gnuplot still doesnt produce results for me) Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Thu Sep 25 12:18:51 2008 @@ -42,54 +42,66 @@ ]) tw.line(asciitable(rows)) -def _process_rows(reader, starthook, endhook): - for name, results in reader.name2results.items(): - starthook(name) - row0 = ["num"] + [result.executable for result in results] - - numsnapshosts = len(results[0].snapshots) - rows = [row0] - for i in range(numsnapshosts): - row = [i] - for result in results: - row.append(result.snapshots[i].private) - rows.append(row) - endhook(rows) - -def checkpointdetails(reader): - tw = py.io.TerminalWriter() - def starthook(name): - tw.sep("=", "private RSS at checkpoints: %s" %(name,)) - _process_rows(reader, starthook, lambda rows: tw.line(asciitable(rows))) - -def gnuplot_output(reader): - output = [] - pythons = [] - - def endhook(rows): - if not pythons: - pythons.extend(rows[0][1:]) - output.append(rows[2][1:]) - - _process_rows(reader, lambda name : None, endhook) - runbench.mydir.join("gnuplotdata").write( - "\n".join([" ".join([str(j) for j in i]) for i in output])) - - def new_cmd(num, name): - s = ("plot 'gnuplotdata' using ($%d) with histograms title '%s'" % - (num + 1,name)) - if num > 0: - s = "re" + s - return s - - plotcmds = "\n".join([new_cmd(i, name) for i, name in enumerate(pythons)]) - runbench.mydir.join("gnuplotcmd").write( - """set terminal postscript color - set output 'output.ps' - %s - """ % plotcmds - ) - os.system("gnuplot gnuplotcmd") +class TableReporter: + def __init__(self, reader): + self.reader = reader + + begin = report = end = lambda x: None # hooks + + def run(self): + self.begin() + for name, results in reader.name2results.items(): + row0 = ["num"] + [result.executable for result in results] + numsnapshosts = len(results[0].snapshots) + rows = [row0] + for i in range(numsnapshosts): + row = [i] + for result in results: + row.append(result.snapshots[i].private) + rows.append(row) + self.report(name, rows) + self.end() + +class CheckpointDetails(TableReporter): + def begin(self): + self.tw = py.io.TerminalWriter() + + def report(self, name, rows): + self.tw.sep("=", "private RSS at checkpoints: %s" %(name,)) + self.tw.line(asciitable(rows)) + +class Gnuplot(TableReporter): + def begin(self): + self.output = [] + self.pythons = [] + + def report(self, name, rows): + if not self.pythons: + self.pythons.extend(rows[0][1:]) + self.output.append(rows[2][1:]) + + def end(self): + pythons = self.pythons + output = self.output + + runbench.mydir.join("gnuplotdata").write( + "\n".join([" ".join([str(j) for j in i]) for i in output])) + + def new_cmd(num, name): + s = ("plot 'gnuplotdata' using ($%d) with histograms title '%s'" % + (num + 1,name)) + if num > 0: + s = "re" + s + return s + + plotcmds = "\n".join([new_cmd(i, name) for i, name in enumerate(pythons)]) + runbench.mydir.join("gnuplotcmd").write( + """set terminal postscript color + set output 'output.ps' + %s + """ % plotcmds + ) + os.system("gnuplot gnuplotcmd") if __name__ == "__main__": options, args = parser.parse_args() @@ -99,8 +111,8 @@ reader.parse(benchlog) #maxtable_overview(reader) - checkpointdetails(reader) - gnuplot_output(reader) + CheckpointDetails(reader).run() + Gnuplot(reader).run() #for name, results in reader.name2results.items(): # tw.sep("=", name) From fijal at codespeak.net Thu Sep 25 12:58:17 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 25 Sep 2008 12:58:17 +0200 (CEST) Subject: [pypy-svn] r58427 - pypy/branch/gc-experiments/pypy/rpython/memory/gc Message-ID: <20080925105817.6FBC8169FE4@codespeak.net> Author: fijal Date: Thu Sep 25 12:58:16 2008 New Revision: 58427 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Log: * add a comment "how I would like this to be" * shuffle code a bit around, doesn't really matter. Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Thu Sep 25 12:58:16 2008 @@ -4,6 +4,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict +from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage class GCBase(object): _alloc_flavor_ = "raw" @@ -252,6 +253,7 @@ return size + def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers @@ -304,7 +306,6 @@ self.objects_with_finalizers = new_with_finalizer return scan - def _append_if_nonnull(pointer, stack): if pointer.address[0] != NULL: stack.append(pointer.address[0]) Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Thu Sep 25 12:58:16 2008 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rpython.memory.gc.base import MovingGCBase, GCFLAG_FORWARDED,\ - TYPEID_MASK +from pypy.rpython.memory.gc.base import MovingGCBase, \ + TYPEID_MASK, GCFLAG_FINALIZATION_ORDERING from pypy.rlib.debug import ll_assert from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -14,6 +14,25 @@ memoryError = MemoryError() +# Mark'n'compact garbage collector +# +# main point of this GC is to save as much memory as possible +# (not to be worse than semispace), but avoid having peaks of +# memory during collection. Inspired, at least partly by squeak's +# garbage collector + +# so, the idea as now is: + +# we allocate space (full of zeroes) which is big enough to handle +# all possible cases. Because it's full of zeroes, we never allocate +# it really, unless we start using it + +# for each collection we mark objects which are alive, also marking all +# for which we want to run finalizers. we mark them by storing forward +# pointer, which will be a place to copy them. After that, we copy all +# using memmove to another view of the same space, hence compacting +# everything + class MarkCompactGC(MovingGCBase): HDR = lltype.Struct('header', ('tid', lltype.Signed), ('forward_ptr', llmemory.Address)) @@ -87,36 +106,16 @@ if self.run_finalizers.non_empty(): self.update_run_finalizers() if self.objects_with_finalizers.non_empty(): - self.deal_with_objects_with_finalizers(self.space) + self.deal_with_objects_with_finalizers(None) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.debug_check_consistency() toaddr = llarena.arena_new_view(self.space) - self.create_forward_pointers(toaddr) self.debug_check_consistency() - self.update_forward_refs() self.compact(toaddr) self.space = toaddr self.debug_check_consistency() - def create_forward_pointers(self, toaddr): - fromaddr = self.space - size_gc_header = self.gcheaderbuilder.size_gc_header - while fromaddr < self.spaceptr: - hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) - obj = fromaddr + size_gc_header - objsize = self.get_size(obj) - totalsize = size_gc_header + objsize - if hdr.tid & GCFLAG_MARKBIT: - # this objects survives, clear MARKBIT - if fromaddr.offset != toaddr.offset: - # this object is forwarded, set forward bit and address - hdr.tid |= GCFLAG_FORWARDED - llarena.arena_reserve(toaddr, totalsize) - hdr.forward_ptr = toaddr + size_gc_header - toaddr += size_gc_header + objsize - fromaddr += size_gc_header + objsize - def get_type_id(self, addr): return self.header(addr).tid & TYPEID_MASK @@ -178,8 +177,7 @@ if addr != NULL: hdr = llmemory.cast_adr_to_ptr(addr - size_gc_header, lltype.Ptr(self.HDR)) - if hdr.tid & GCFLAG_FORWARDED: - pointer.address[0] = hdr.forward_ptr + pointer.address[0] = hdr.forward_ptr def mark(self): self.root_walker.walk_roots( @@ -193,6 +191,44 @@ self.header(obj).tid |= GCFLAG_MARKBIT self.trace(obj, self._mark_object, None) + def deal_with_objects_with_finalizers(self, ignored): + new_with_finalizer = self.AddressDeque() + while self.objects_with_finalizers.non_empty(): + obj = self.objects_with_finalizers.popleft() + if self.surviving(obj): + new_with_finalizer.append(obj) + break + finalizers_to_run.append(obj) + xxxx + def debug_check_object(self, obj): # XXX write it down pass + + def surviving(self, obj): + hdr = self.header(obj) + return hdr.tid & GCFLAG_MARKBIT + + def _finalization_state(self, obj): + hdr = self.header(obj) + if self.surviving(obj): + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 2 + else: + return 3 + else: + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 1 + else: + return 0 + + def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): + # it's enough to leave mark bit here + hdr = self.header(obj) + if hdr.tid & GCFLAG_MARKBIT: + return # cycle + self.header(obj).tid |= GCFLAG_MARKBIT + self.trace(obj, self._mark_object, None) + + def get_forwarding_address(self, obj): + return obj From hpk at codespeak.net Thu Sep 25 15:45:46 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 25 Sep 2008 15:45:46 +0200 (CEST) Subject: [pypy-svn] r58432 - pypy/build/benchmem Message-ID: <20080925134546.9857416A0BA@codespeak.net> Author: hpk Date: Thu Sep 25 15:45:44 2008 New Revision: 58432 Modified: pypy/build/benchmem/report.py pypy/build/benchmem/runbench.py Log: more specific reporting: * incremental sizes of size benchmarks * detailed base interpreter sizes Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Thu Sep 25 15:45:44 2008 @@ -12,6 +12,8 @@ parser = py.compat.optparse.OptionParser(usage=__doc__) parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", help="logfile to read results from") +parser.add_option("-g", "--gnuplot", action="store_true", dest="gnuplot", default=False, + help="generate gnuplot output") def asciitable(table): colmax = [] @@ -27,6 +29,7 @@ line.append("%%-%ds" %(width) %(col)) lines.append(" ".join(line)) return "\n".join(lines) + def maxtable_overview(reader): tw = py.io.TerminalWriter() @@ -103,6 +106,81 @@ ) os.system("gnuplot gnuplotcmd") +class Sorter: + def __init__(self, name2results): + self.name2results = name2results + + def filter(self, benchprefix=""): + d = {} + for name, results in self.name2results.iteritems(): + if name.startswith(benchprefix): + d[name] = results + return Sorter(d) + + def getsorted(self): + l = self.name2results.items() + l.sort() + return l + + def getexecutables(self): + ret = None + for name, results in self.name2results.iteritems(): + executables = [result.executable for result in results] + if ret is None: + ret = executables + if ret != executables: + raise ValueError("resultset has incompatible list of executables" + "%s != %s" %(ret, executables)) + return ret + +class IncrementalSizePerBench: + def __init__(self, reader): + self.reader = reader + + def run(self): + tw = py.io.TerminalWriter() + tw.sep("=", "Incremental private RSS of size benchmarks") + sorter = Sorter(self.reader.name2results) + sorter = sorter.filter(benchprefix="sizes.") + executables = sorter.getexecutables() + row0 = ["name"] + [str(x) for x in executables] + rows = [row0] + for name, results in sorter.getsorted(): + basesize = self.get_incremental_size(results[0]) + row = [name] + for result in results: + incsize = self.get_incremental_size(result) + # colors work but messup table layout + #color = incsize <= basesize and "green" or "red" + #row.append(tw.markup(str(incsize), **{color:True})) + row.append(incsize) + rows.append(row) + tw.line(asciitable(rows)) + + def get_incremental_size(self, result): + # checkpoint 0: state before benchmark + # checkpoint 1: state at end of benchmark func + # checkpoint 2: state after benchmark func finished + basesize = result.snapshots[0].private + inbench = result.snapshots[1].private + return inbench - basesize + +class BaseSizeOfInterpreters: + def __init__(self, reader): + self.reader = reader + def run(self): + tw = py.io.TerminalWriter() + tw.sep("=", "Base Size of interpreters (using sizes.bench_list_of_None)") + sorter = Sorter(self.reader.name2results).filter(benchprefix="sizes.") + row0 = "executable rss shared_clean shared_dirty private_clean private_dirty".split() + rows = [row0] + for result in sorter.name2results['sizes.bench_list_of_None']: + row = [result.executable] + for name in row0[1:]: + row.append(getattr(result.snapshots[0], name)) + rows.append(row) + tw.line(asciitable(rows)) + if __name__ == "__main__": options, args = parser.parse_args() @@ -112,7 +190,11 @@ #maxtable_overview(reader) CheckpointDetails(reader).run() - Gnuplot(reader).run() + IncrementalSizePerBench(reader).run() + BaseSizeOfInterpreters(reader).run() + + if options.gnuplot: + Gnuplot(reader).run() #for name, results in reader.name2results.items(): # tw.sep("=", name) @@ -122,3 +204,4 @@ # result.max("shared_dirty"), # result.max("private_dirty") ### ) + Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Thu Sep 25 15:45:44 2008 @@ -145,7 +145,8 @@ # see write_benchheader executable = eval(line.split("=", 1)[1]) benchbasename = eval(f.readline().split("=", 1)[1]) - benchname = eval(f.readline().split("=", 1)[1]) + benchfuncname = eval(f.readline().split("=", 1)[1]) + benchname = "%s.%s" % (benchbasename[:-3], benchfuncname) benchargs = eval(f.readline().split("=", 1)[1]) snapshots = [] From hpk at codespeak.net Thu Sep 25 16:21:34 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 25 Sep 2008 16:21:34 +0200 (CEST) Subject: [pypy-svn] r58433 - pypy/build/benchmem/benchmark Message-ID: <20080925142134.A662516A0D9@codespeak.net> Author: hpk Date: Thu Sep 25 16:21:31 2008 New Revision: 58433 Modified: pypy/build/benchmem/benchmark/sizes.py Log: adding some benchmarks Modified: pypy/build/benchmem/benchmark/sizes.py ============================================================================== --- pypy/build/benchmem/benchmark/sizes.py (original) +++ pypy/build/benchmem/benchmark/sizes.py Thu Sep 25 16:21:31 2008 @@ -7,6 +7,14 @@ l = [int(i+1000) for i in range(iter1)] checkpoint(collect=True) +def bench_list_of_emptytuple(iter1): + l = [() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_tupleofNone(iter1): + l = [(None,) for i in range(iter1)] + checkpoint(collect=True) + def bench_list_of_emptylist(iter1): l = [[] for i in range(iter1)] checkpoint(collect=True) @@ -58,3 +66,8 @@ x = [x] checkpoint(collect=True) +def bench_recursive_tuple(iter1): + x = () + for i in range(iter1): + x = (x,) + checkpoint(collect=True) From witulski at codespeak.net Fri Sep 26 13:39:15 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Fri, 26 Sep 2008 13:39:15 +0200 (CEST) Subject: [pypy-svn] r58442 - pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test Message-ID: <20080926113915.8EF5416A17D@codespeak.net> Author: witulski Date: Fri Sep 26 13:39:13 2008 New Revision: 58442 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: Added Shitf right/left operations + Tests Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Fri Sep 26 13:39:13 2008 @@ -61,6 +61,16 @@ builder.end() return gv_one_op_instr +def make_two_op_instr(rgenop, instr_name): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_two_op_instr, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "two_op_instr") + builder.start_writing() + + gv_result = builder.genop2(instr_name, gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_two_op_instr + def make_bool_op(rgenop, which_bool_op): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_bool_op, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "bool_op") @@ -93,6 +103,15 @@ builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_mul + +def make_div(rgenop): + sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) + builder, gv_div, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "div") + builder.start_writing() + gv_result = builder.genop2("int_div", gv_x, gv_y) + builder.finish_and_return(sigtoken, gv_result) + builder.end() + return gv_div def make_mul_imm(rgenop, num): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) @@ -117,6 +136,32 @@ fnptr = self.cast(dec_function,1) res = fnptr(2) assert res == 1 + + def test_shift_left(self): + shift_func = make_two_op_instr(self.RGenOp(),"int_lshift") + fp = self.cast(shift_func, 2) + res = fp(128,2) + assert res == 512 + res = fp(16,2) + assert res == 64 + res = fp(21,1) + assert res == 42 + res = fp(16,-1) + assert res == 8 + + def test_shift_right(self): + shift_func = make_two_op_instr(self.RGenOp(),"int_rshift") + fp = self.cast(shift_func, 2) + res = fp(16,2) + assert res == 4 + res = fp(64,3) + assert res == 8 + res = fp(-64,2) + assert res == -16 + res = fp(84,1) + assert res == 42 + res = fp(32,-2) + assert res == 128 def test_mul_im32(self): rgenop = self.RGenOp() @@ -124,8 +169,12 @@ fnptr = self.cast(mul_function,1) res = fnptr(210) assert res == 42000 + mul_function = make_mul_imm(rgenop,-9876) + fnptr = self.cast(mul_function,1) + res = fnptr(12345) + assert res == -121919220 - # segmentation fault at mov(qwreg,imm64) + # Illegal instruction at mov(qwreg,imm64) #def test_mul_im64(self): # rgenop = self.RGenOp() @@ -134,7 +183,7 @@ # res = fnptr(2) # assert res == int("123456789",16)*2 - def test_mul(self): + def test_imul(self): rgenop = self.RGenOp() mul_function = make_mul(rgenop) fnptr = self.cast(mul_function,2) @@ -155,6 +204,14 @@ res = fnptr(12345,-9876) assert res == -121919220 + #Floating point exception + #def test_idiv(self): + # rgenop = self.RGenOp() + # div_function = make_div(rgenop) + # fnptr = self.cast(div_function,2) + # res = fnptr(100,2) + # assert res == 50 + def test_greater(self): rgenop = self.RGenOp() cmp_function = make_cmp(rgenop, "int_gt") From witulski at codespeak.net Fri Sep 26 13:40:46 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Fri, 26 Sep 2008 13:40:46 +0200 (CEST) Subject: [pypy-svn] r58443 - pypy/branch/oo-jit/pypy/jit/codegen/x86_64 Message-ID: <20080926114046.DF98F16850C@codespeak.net> Author: witulski Date: Fri Sep 26 13:40:46 2008 New Revision: 58443 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Log: Added shift left/right operations + Tests Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Fri Sep 26 13:40:46 2008 @@ -189,6 +189,8 @@ _MOV_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x89", 3, None, None) _MOV_QWREG_IMM64 = make_two_operand_instr_with_alternate_encoding(1,0,0,None,"B8",None,None) + _IDIV_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xF7", 3, None, 7) + _IMUL_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x0F", 3, None, None, None, "\xAF") _IMUL_QWREG_IMM32 = make_two_operand_instr( 1, None, 0, None, "\x69", 3, None, "sameReg") @@ -214,6 +216,9 @@ _SETLE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,14) _SETNE_8REG = make_one_operand_instr( 0, 0, 0, 0, "\x0F", 3, None, 0,5) + _SHL_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xD3", 3, None, 4) + _SHR_QWREG = make_one_operand_instr( 1, 0, 0, None, "\xD3", 3, None, 5) + _SUB_QWREG_QWREG = make_two_operand_instr( 1, None, 0, None, "\x28", 3, None, None) _SUB_QWREG_IMM32 = make_two_operand_instr( 1, 0, 0, 0, "\x81", 3, None, 5) @@ -272,6 +277,11 @@ method = getattr(self, "_MOV"+op1.to_string()+op2.to_string()) method(op1, op2) + + def IDIV(self, op1): + method = getattr(self, "_IDIV"+op1.to_string()) + method(op1) + def IMUL(self, op1, op2): method = getattr(self, "_IMUL"+op1.to_string()+op2.to_string()) # exchange the two arguments because @@ -300,6 +310,10 @@ method = getattr(self, "_SETL"+op1.to_string()) method(op1) + def SETR(self, op1): + method = getattr(self, "_SETR"+op1.to_string()) + method(op1) + def SETGE(self, op1): method = getattr(self, "_SETGE"+op1.to_string()) method(op1) @@ -316,6 +330,14 @@ method = getattr(self, "_SETNE"+op1.to_string()) method(op1) + def SHL(self, op1): + method = getattr(self, "_SHL"+op1.to_string()) + method(op1) + + def SHR(self, op1): + method = getattr(self, "_SHR"+op1.to_string()) + method(op1) + def SUB(self, op1, op2): method = getattr(self, "_SUB"+op1.to_string()+op2.to_string()) method(op1, op2) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Fri Sep 26 13:40:46 2008 @@ -99,17 +99,42 @@ op_int_add = make_two_argument_method("ADD") op_int_and = make_two_argument_method("AND") - op_int_dec = make_one_argument_method("DEC") + op_int_dec = make_one_argument_method("DEC") #for debuging op_int_inc = make_one_argument_method("INC") #for debuging op_int_mul = make_two_argument_method("IMUL") op_int_neg = make_one_argument_method("NEG") op_int_not = make_one_argument_method("NOT") op_int_or = make_two_argument_method("OR") - op_int_push = make_one_argument_method("PUSH") - op_int_pop = make_one_argument_method("POP") + op_int_push = make_one_argument_method("PUSH") #for debuging + op_int_pop = make_one_argument_method("POP") #for debuging op_int_sub = make_two_argument_method("SUB") op_int_xor = make_two_argument_method("XOR") + # FIXME: uses rcx insted of cl + def op_int_lshift(self, gv_x, gv_y): + gv_z = self.allocate_register("rcx") + #self.mc.XOR(gv_z, gv_z) + self.mc.MOV(gv_z, gv_y) + self.mc.SHL(gv_x) + return gv_x + + # FIXME: uses rcx insted of cl + def op_int_rshift(self, gv_x, gv_y): + gv_z = self.allocate_register("rcx") + #self.mc.XOR(gv_z, gv_z) + self.mc.MOV(gv_z, gv_y) + self.mc.SHR(gv_x) + return gv_x + + # IDIV RDX:RAX with QWREG + # FIXME: supports only RAX with QWREG + def op_int_div(self, gv_x, gv_y): + gv_z = self.allocate_register("rax") + self.mc.MOV(gv_z, gv_x) + self.mc.IDIV(gv_y) + return gv_y #FIXME: return gv_x? + + #FIXME: can only jump 32bit #FIXME: -6 displacement: the displ+ rip of next instr def jump_if_true(self, gv_condition, args_for_jump_gv): From witulski at codespeak.net Fri Sep 26 14:22:47 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Fri, 26 Sep 2008 14:22:47 +0200 (CEST) Subject: [pypy-svn] r58445 - in pypy/branch/oo-jit/pypy/jit/codegen: . x86_64 x86_64/test Message-ID: <20080926122247.108ED16A14F@codespeak.net> Author: witulski Date: Fri Sep 26 14:22:45 2008 New Revision: 58445 Modified: pypy/branch/oo-jit/pypy/jit/codegen/model.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: (cfbolz, witulski): AAAAAAARGH!!! c_int is 4 byte long on 64 bit machines, leading to arbitrarily much confusion with big and negative integers. Fix this. Modified: pypy/branch/oo-jit/pypy/jit/codegen/model.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/model.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/model.py Fri Sep 26 14:22:45 2008 @@ -399,13 +399,13 @@ Python callable object, for testing purposes. """ from pypy.rpython.lltypesystem import lltype - from ctypes import cast, c_void_p, CFUNCTYPE, c_int, c_double + from ctypes import cast, c_void_p, CFUNCTYPE, c_long, c_double def _to_ctypes(t): #limited type support for now if t == lltype.Float: return c_double if t == lltype.Void: return None - return c_int + return c_long ctypestypes = [_to_ctypes(t) for t in FUNC.TO.ARGS] ctypesres = _to_ctypes(FUNC.TO.RESULT) return cast(c_void_p(gv.value), CFUNCTYPE(ctypesres, *ctypestypes)) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Fri Sep 26 14:22:45 2008 @@ -110,10 +110,10 @@ op_int_sub = make_two_argument_method("SUB") op_int_xor = make_two_argument_method("XOR") + # FIXME: is that lshift val? # FIXME: uses rcx insted of cl def op_int_lshift(self, gv_x, gv_y): gv_z = self.allocate_register("rcx") - #self.mc.XOR(gv_z, gv_z) self.mc.MOV(gv_z, gv_y) self.mc.SHL(gv_x) return gv_x @@ -121,7 +121,6 @@ # FIXME: uses rcx insted of cl def op_int_rshift(self, gv_x, gv_y): gv_z = self.allocate_register("rcx") - #self.mc.XOR(gv_z, gv_z) self.mc.MOV(gv_z, gv_y) self.mc.SHR(gv_x) return gv_x Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Fri Sep 26 14:22:45 2008 @@ -275,6 +275,8 @@ assert res == 1 res = fnptr(256,512) assert res == 0 + res = fnptr(-4,18446744073709551615)#-4>=18446744073709551615 + assert res == 0 #false res = fnptr(-4,253) #-4>=253 assert res == 0 # false res = fnptr(-4,0) @@ -284,8 +286,8 @@ rgenop = self.RGenOp() cmp_function = make_cmp(rgenop, "int_eq",42) fnptr = self.cast(cmp_function,1) - res = fnptr(42) - assert res == 1 + # res = fnptr(42) + # assert res == 1 res = fnptr(23) assert res == 0 cmp_function = make_cmp(rgenop, "int_eq") @@ -300,12 +302,16 @@ assert res == 0 res = fnptr(-4,0) assert res == 0 + res = fnptr(184467440737095516,184467440737095516) + assert res == 1 res = fnptr(252,-4) assert res == 0 res = fnptr(-4,252) assert res == 0 res = fnptr(244,756) assert res == 0 + res = fnptr(-1,18446744073709551615) + assert res == 0 def test_not_equal(self): rgenop = self.RGenOp() From witulski at codespeak.net Fri Sep 26 14:34:09 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Fri, 26 Sep 2008 14:34:09 +0200 (CEST) Subject: [pypy-svn] r58446 - pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test Message-ID: <20080926123409.CA91916A06A@codespeak.net> Author: witulski Date: Fri Sep 26 14:34:08 2008 New Revision: 58446 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: Fixed some tests Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Fri Sep 26 14:34:08 2008 @@ -146,8 +146,8 @@ assert res == 64 res = fp(21,1) assert res == 42 - res = fp(16,-1) - assert res == 8 + res = fp(16,1) + assert res == 32 def test_shift_right(self): shift_func = make_two_op_instr(self.RGenOp(),"int_rshift") @@ -156,12 +156,8 @@ assert res == 4 res = fp(64,3) assert res == 8 - res = fp(-64,2) - assert res == -16 res = fp(84,1) assert res == 42 - res = fp(32,-2) - assert res == 128 def test_mul_im32(self): rgenop = self.RGenOp() @@ -310,7 +306,7 @@ assert res == 0 res = fnptr(244,756) assert res == 0 - res = fnptr(-1,18446744073709551615) + res = fnptr(-1,9223372036854775807) assert res == 0 def test_not_equal(self): @@ -430,6 +426,7 @@ result = fnptr(1) assert result == 1 + test_switch_many_args_direct = skip test_directtesthelper_direct = skip test_dummy_compile = skip test_cast_raising = skip From pedronis at codespeak.net Sat Sep 27 14:06:59 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 27 Sep 2008 14:06:59 +0200 (CEST) Subject: [pypy-svn] r58452 - pypy/build/bot2 Message-ID: <20080927120659.B224C16A1EA@codespeak.net> Author: pedronis Date: Sat Sep 27 14:06:57 2008 New Revision: 58452 Modified: pypy/build/bot2/TODO Log: another todo Modified: pypy/build/bot2/TODO ============================================================================== --- pypy/build/bot2/TODO (original) +++ pypy/build/bot2/TODO Sat Sep 27 14:06:57 2008 @@ -5,6 +5,7 @@ by builders/revisions(/branches), formatting and css - some kind of progress information for a run +- support for better information in case of runner-level killed hanging tests - build/support for running cpython on tests, adaptations to produce structured logs and be compatible with the py.lib trunk are probably necessary for lib-python/conftest.py From pedronis at codespeak.net Sat Sep 27 15:27:45 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 27 Sep 2008 15:27:45 +0200 (CEST) Subject: [pypy-svn] r58453 - pypy/build/bot2/pypybuildbot Message-ID: <20080927132745.A5D9616A1E5@codespeak.net> Author: pedronis Date: Sat Sep 27 15:27:43 2008 New Revision: 58453 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: - slicing/querying results by recentrev and/or branch - links using this in the summary page itself Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Sat Sep 27 15:27:43 2008 @@ -181,6 +181,7 @@ def __init__(self): self.sections = [] + self.cur_branch=None def make_longrepr_url_for(self, outcome_set, namekey): cachekey, namekey = outcome_set.get_key_namekey(namekey) @@ -205,9 +206,16 @@ anchors.append(html.a(text, href=url)) return anchors - def add_title(self, title): - self.sections.append(html.h2(title)) - + def start_branch(self, branch): + self.cur_branch = branch + branch_anchor = html.a(branch, href="/summary?branch=%s" % branch) + self.sections.append(html.h2(branch_anchor)) + + def _rev_anchor(self, rev): + rev_anchor = html.a(str(rev), href="/summary?branch=%s&recentrev=%d" % + (self.cur_branch, rev)) + return rev_anchor + def add_section(self, outcome_sets): revs = sorted(outcome_set.revision for outcome_set in outcome_sets) by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set @@ -220,7 +228,7 @@ for rev, outcome_set in by_rev: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - line = ["%s %d" % (bars(),rev)] + line = [bars(), ' ', self._rev_anchor(rev)] line.append((align-len(line[0]))*" ") line.append(self.make_stdio_anchors_for(outcome_set)) line.append('\n') @@ -314,6 +322,24 @@ return obj.getProperty(name) except KeyError: return default + +def make_test(lst): + if lst is None: + return lambda v: True + else: + membs = set(lst) + return lambda v: v in membs + +def make_subst(v1, v2): + def subst(v): + if v == v1: + return v2 + return v + return subst + +trunk_name = make_subst(None, "") +trunk_value = make_subst("", None) + class Summary(HtmlResource): @@ -332,14 +358,21 @@ for rev in sorted(revs.keys())[:-cutnum]: del revs[rev] - def recentRevisions(self, status): + def recentRevisions(self, status, only_recentrevs=None, only_branches=None): + test_rev = make_test(only_recentrevs) + test_branch = make_test(only_branches) + branches = {} for builderName in status.getBuilderNames(): builderStatus = status.getBuilder(builderName) for build in builderStatus.generateFinishedBuilds(num_builds=5*N): branch = getProp(build, 'branch') + if not test_branch(branch): + continue got_rev = getProp(build, 'got_revision', None) + if not test_rev(got_rev): + continue revs, no_revision_builds = branches.setdefault(branch, ({}, [])) @@ -364,16 +397,23 @@ status = self.getStatus(request) page = SummaryPage() + #page.sections.append(repr(request.args)) - branches = self.recentRevisions(status) + only_branches = request.args.get('branch', None) + only_recentrevs = request.args.get('recentrev', None) + if only_branches is not None: + only_branches = map(trunk_value, only_branches) + + branches = self.recentRevisions(status, + only_recentrevs=only_recentrevs, + only_branches=only_branches) for branch, (revs, no_revision_builds) in sorted(branches.iteritems()): outcome_sets = [] for rev, by_build in revs.items(): outcome_sets.append(GatherOutcomeSet(by_build)) - if branch is None: - branch = "" - page.add_title(branch) + branch = trunk_name(branch) + page.start_branch(branch) page.add_section(outcome_sets) page.add_no_revision_builds(status, no_revision_builds) From cami at codespeak.net Sun Sep 28 00:39:25 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 28 Sep 2008 00:39:25 +0200 (CEST) Subject: [pypy-svn] r58455 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080927223925.4CD7616A17F@codespeak.net> Author: cami Date: Sun Sep 28 00:39:24 2008 New Revision: 58455 Added: pypy/dist/pypy/lang/gameboy/video_mode.py pypy/dist/pypy/lang/gameboy/video_register.py pypy/dist/pypy/lang/gameboy/video_sprite.py Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py pypy/dist/pypy/lang/gameboy/video.py Log: created separated files for video elements to cleanup adapted tests according to split up adapted imports for the different files Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Sun Sep 28 00:39:24 2008 @@ -1,8 +1,8 @@ from pypy.lang.gameboy import constants -from pypy.lang.gameboy.video import ControlRegister -from pypy.lang.gameboy.video import StatusRegister -from pypy.lang.gameboy.video import Window -from pypy.lang.gameboy.video import Background +from pypy.lang.gameboy.video_register import ControlRegister +from pypy.lang.gameboy.video_register import StatusRegister +from pypy.lang.gameboy.video_sprite import Window +from pypy.lang.gameboy.video_sprite import Background from pypy.lang.gameboy.test.test_video import get_video import py @@ -22,7 +22,7 @@ control.write(0xFF) assert control.read() == 0xFF control.reset() - assert control.read() == 0x91 + assert control.read() == 0x91 def test_video_control_read_write_properties(): Modified: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Sun Sep 28 00:39:24 2008 @@ -1,5 +1,5 @@ from pypy.lang.gameboy import constants -from pypy.lang.gameboy.video import Sprite +from pypy.lang.gameboy.video_sprite import Sprite from pypy.lang.gameboy.video import Video from pypy.lang.gameboy.test.test_video import get_video import py Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 28 00:39:24 2008 @@ -5,9 +5,13 @@ import math import operator from pypy.lang.gameboy import constants -from pypy.lang.gameboy.constants import SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH, GAMEBOY_SCREEN_HEIGHT +from pypy.lang.gameboy.constants import SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH, \ + GAMEBOY_SCREEN_HEIGHT from pypy.lang.gameboy.ram import iMemory from pypy.lang.gameboy.cpu import process_2s_complement +from pypy.lang.gameboy.video_register import ControlRegister, StatusRegister +from pypy.lang.gameboy.video_sprite import Sprite, Tile, Background, Window +from pypy.lang.gameboy.video_mode import Mode0, Mode1, Mode2, Mode3 # ----------------------------------------------------------------------------- class VideoCallWraper(object): @@ -29,600 +33,7 @@ def call(self, pos, color, mask): self.video.set_tile_line(pos, color, mask) - - -# ----------------------------------------------------------------------------- - -class ControlRegister(object): - """ - used for enabled or disabled window or background - Bit 7 - LCD Display Enable (0=Off, 1=On) - Bit 6 - Window Tile Map Display Select (0=9800-9BFF, 1=9C00-9FFF) - Bit 5 - Window Display Enable (0=Off, 1=On) - Bit 4 - BG & Window Tile Data Select (0=8800-97FF, 1=8000-8FFF) - Bit 3 - BG Tile Map Display Select (0=9800-9BFF, 1=9C00-9FFF) - Bit 2 - OBJ (Sprite) Size (0=8x8, 1=8x16) - Bit 1 - OBJ (Sprite) Display Enable (0=Off, 1=On) - Bit 0 - BG Display (for CGB see below) (0=Off, 1=On) - """ - def __init__(self, window, background): - self.window = window - self.background = background - self.reset() - - def reset(self): - self.lcd_enabled = True - self.window.upper_tile_map_selected = False - self.window.enabled = False - self.background_and_window_lower_tile_data_selected = True - self.background.upper_tile_map_selected = False - self.big_sprite_size_selected = False - self.sprites_enabled = False - self.background.enabled = True - - def read(self): - value = 0 - value += int(self.lcd_enabled) << 7 - value += int(self.window.upper_tile_map_selected) << 6 - value += int(self.window.enabled) << 5 - value += int(self.background_and_window_lower_tile_data_selected) << 4 - value += int(self.background.upper_tile_map_selected) << 3 - value += int(self.big_sprite_size_selected) << 2 - value += int(self.sprites_enabled) << 1 - value += int(self.background.enabled) << 0 - return value - - def write(self, value): - self.lcd_enabled = bool(value & (1 << 7)) - self.window.upper_tile_map_selected = bool(value & (1 << 6)) - self.window.enabled = bool(value & (1 << 5)) - self.background_and_window_lower_tile_data_selected = \ - bool(value & (1 << 4)) - self.background.upper_tile_map_selected = bool(value & (1 << 3)) - self.big_sprite_size_selected = bool(value & (1 << 2)) - self.sprites_enabled = bool(value & (1 << 1)) - self.background.enabled = bool(value & (1 << 0)) - - - def get_selected_tile_data_space(self): - if self.window.upper_tile_map_selected: - return constants.VRAM_DATA_A - else: - return constants.VRAM_DATA_B - -# ----------------------------------------------------------------------------- -class InvalidModeOrderException(Exception): - def __init__(self, mode, previous_mode): - Exception.__init__(self, \ - "Wrong Mode order! No valid transition %i => %i" \ - % (previous_mode.id(), mode.id())) - -class HandleSubclassException(Exception): - def __init__(self): - Exception.__init__(self, "") - -class Mode(object): - - """ - The two lower STAT bits show the current status of the LCD controller. - """ - - def __init__(self, video): - self.video = video - self.reset() - - def reset(self): - raise Exception("unimplemented method") - - def id(self): - raise Exception("unimplemented method") - - def activate(self, previous_mode): - raise Exception("unimplemented method") - - def emulate(self): - raise Exception("unimplemented method") - - def emulate_hblank_line_y_compare(self, stat_check=False): - if self.video.line_y == self.video.line_y_compare: - if not (stat_check and self.video.status.line_y_compare_flag): - self.line_y_line_y_compare_interrupt_check() - else: - self.video.status.line_y_compare_flag = False - - def line_y_line_y_compare_interrupt_check(self): - self.video.status.line_y_compare_flag = True - if self.video.status.line_y_compare_interrupt: - self.video.lcd_interrupt_flag.set_pending() - -class Mode0(Mode): - """ - Mode 0: The LCD controller is in the H-Blank period and - the CPU can access both the display RAM (8000h-9FFFh) - and OAM (FE00h-FE9Fh) - """ - - def reset(self): - self.h_blank_interrupt = False - - def id(self): - return 0 - - def activate(self, previous_mode): - #if previous_mode.id() == 3: - self.video.cycles += constants.MODE_0_TICKS - self.h_blank_interrupt_check() - #else: - # video.reset_control() can be called in any position - # pass - #raise InvalidModeOrderException(self, previous_mode) - - def h_blank_interrupt_check(self): - if self.h_blank_interrupt and \ - self.video.status.line_y_compare_check(): - self.video.lcd_interrupt_flag.set_pending() - - def emulate(self): - #self.video.emulate_hblank() - self.emulate_hblank() - - def emulate_hblank(self): - self.video.line_y += 1 - self.emulate_hblank_line_y_compare() - if self.video.line_y < GAMEBOY_SCREEN_HEIGHT: - self.video.status.set_mode(2) - else: - self.emulate_hblank_part_2() - - def emulate_hblank_part_2(self): - if self.video.display: - self.video.draw_frame() - self.video.frames += 1 - if self.video.frames >= self.video.frame_skip: - self.video.display = True - self.video.frames = 0 - else: - self.video.display = False - self.video.status.set_mode(1) - self.video.v_blank = True - -class Mode1(Mode): - """ - Mode 1: The LCD contoller is in the V-Blank period (or the - display is disabled) and the CPU can access both the - display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) - """ - - def reset(self): - self.v_blank_interrupt = False - - def id(self): - return 1 - - def activate(self, previous_mode): - #if previous_mode.id() == 0: - self.set_begin() - #else: - # pass - #raise InvalidModeOrderException(self, previous_mode) - - def set_begin(self): - self.video.cycles += constants.MODE_1_BEGIN_TICKS - - def set_between(self): - self.video.cycles += constants.MODE_1_TICKS - constants.MODE_1_BEGIN_TICKS - - def set_end(self): - self.video.cycles += constants.MODE_1_END_TICKS - - def emulate(self): - self.emulate_v_blank() - - def emulate_v_blank(self): - if self.video.v_blank: - self.emulate_v_blank_v_blank() - elif self.video.line_y == 0: - self.video.status.set_mode(2) - else: - self.emulate_v_blank_other() - - def emulate_v_blank_v_blank(self): - self.video.v_blank = False - self.set_between() - self.v_blank_interrupt_check() - - def v_blank_interrupt_check(self): - if self.v_blank_interrupt: - self.video.lcd_interrupt_flag.set_pending() - self.video.v_blank_interrupt_flag.set_pending() - - def emulate_v_blank_other(self): - if self.video.line_y < 153: - self.emulate_v_blank_mode_1() - else: - self.video.line_y = 0 - self.video.window.line_y = 0 - self.set_between() - self.emulate_hblank_line_y_compare() - - def emulate_v_blank_mode_1(self): - self.video.line_y += 1 - if self.video.line_y != 153: - self.video.cycles += constants.MODE_1_TICKS - else: - self.set_end() - -class Mode2(Mode): - """ - Mode 2: The LCD controller is reading from OAM memory. - The CPU access OAM memory (FE00h-FE9Fh) - during this period. - """ - - def reset(self): - self.oam_interrupt = False - - def id(self): - return 2 - - def activate(self, previous_mode): - #if previous_mode.id() == 0 or previous_mode.id() == 1: - self.video.cycles += constants.MODE_2_TICKS - self.oam_interrupt_check() - #else: - # pass - #raise InvalidModeOrderException(self, previous_mode) - - def oam_interrupt_check(self): - if self.oam_interrupt and \ - self.video.status.line_y_compare_check(): - self.video.lcd_interrupt_flag.set_pending() - - def emulate(self): - self.emulate_oam() - - def emulate_oam(self): - self.video.status.set_mode(3) - -class Mode3(Mode): - """ - Mode 3: The LCD controller is reading from both OAM and VRAM, - The CPU access OAM and VRAM during this period. - CGB Mode: Cannot access Palette Data (FF69,FF6B) either. - """ - - def reset(self): - pass - - def id(self): - return 3 - - def activate(self, previous_mode): - #if previous_mode.id() == 2: - self.set_begin() - #else: - # pass - # #raise InvalidModeOrderException(self, previous_mode) - - def set_begin(self): - self.video.cycles += constants.MODE_3_BEGIN_TICKS - self.video.transfer = True - - def set_end(self): - self.video.cycles += constants.MODE_3_END_TICKS - self.video.transfer = False - - def emulate(self): - self.emulate_transfer() - - def emulate_transfer(self): - if self.video.transfer: - if self.video.display: - self.video.draw_line() - #print "mode 3 ", self.video.status.get_mode() - self.set_end() - else: - self.video.status.set_mode(0) - -# ----------------------------------------------------------------------------- - -class StatusRegister(object): - """ - Bit 6 - LYC=LY Coincidence Interrupt (1=Enable) (Read/Write) - Bit 5 - Mode 2 OAM Interrupt (1=Enable) (Read/Write) - Bit 4 - Mode 1 V-Blank Interrupt (1=Enable) (Read/Write) - Bit 3 - Mode 0 H-Blank Interrupt (1=Enable) (Read/Write) - Bit 2 - Coincidence Flag (0:LYC<>LY, 1:LYC=LY) (Read Only) - Bit 1-0 - Mode Flag (Mode 0-3, see below) (Read Only) - 0: During H-Blank - 1: During V-Blank - 2: During Searching OAM-RAM - 3: During Transfering Data to LCD Driver - """ - def __init__(self, video): - self.create_modes(video) - self.reset() - - def create_modes(self, video): - self.mode0 = Mode0(video) - self.mode1 = Mode1(video) - self.mode2 = Mode2(video) - self.mode3 = Mode3(video) - self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] - - - - def reset(self): - self.current_mode = self.mode2 - self.line_y_compare_flag = False - self.line_y_compare_interrupt = False - for mode in self.modes: - mode.reset() - #self.mode_0_h_blank_interrupt = False - #self.mode_1_v_blank_interrupt = False - #self.mode_2_oam_interrupt = False - self.status = True - - - def read(self, extend=False): - value = self.get_mode() - value += self.line_y_compare_flag << 2 - value += self.mode0.h_blank_interrupt << 3 - value += self.mode1.v_blank_interrupt << 4 - value += self.mode2.oam_interrupt << 5 - value += self.line_y_compare_interrupt << 6 - if extend: - value += int(self.status) << 7 - return value - - - def write(self, value, write_all=False, - keep_mode_0_h_blank_interrupt=False): - if write_all: - self.current_mode = self.modes[value & 0x03] - self.line_y_compare_flag = bool(value & (1 << 2)) - self.status = bool(value & (1 << 7)) - self.mode0.h_blank_interrupt = bool(value & (1 << 3)) - self.mode1.v_blank_interrupt = bool(value & (1 << 4)) - self.mode2.oam_interrupt = bool(value & (1 << 5)) - self.line_y_compare_interrupt = bool(value & (1 << 6)) - - def get_mode(self): - return self.current_mode.id() - - def set_mode(self, mode): - old_mode = self.current_mode - self.current_mode = self.modes[mode & 0x03] - self.current_mode.activate(old_mode) - - def line_y_compare_check(self): - return not self.line_y_compare_flag or not self.line_y_compare_interrupt - -# ----------------------------------------------------------------------------- - -class Sprite(object): - - def __init__(self, video): - self.video = video - self.big_size = False - self.reset() - - def reset(self): - self.x = 0 - self.y = 0 - self.tile = None - self.object_behind_background = False - self.x_flipped = False - self.y_flipped = False - self.palette_number = 0 - self.hidden = True - - - def get_data(self): - return [self.y, self.x, self.tile_number, self.get_attributes_and_flags()] - - def set_data(self, byte0=-1, byte1=-1, byte2=-1, byte3=-1): - """ - extracts the sprite data from an oam entry - """ - if byte0 is not -1: - self.extract_y_position(byte0) - if byte0 is not -1: - self.extract_x_position(byte1) - if byte0 is not -1: - self.extract_tile_number(byte2) - if byte0 is not -1: - self.extract_attributes_and_flags(byte3) - - def extract_y_position(self, data): - """ - extracts the Y Position - Specifies the sprites vertical position on the screen (minus 16). - An offscreen value (for example, Y=0 or Y>=160) hides the sprite. - """ - self.y = data # - 16 - self.hide_check() - - def extract_x_position(self, data): - """ - extracts the X Position - Specifies the sprites horizontal position on the screen (minus 8). - An offscreen value (X=0 or X>=168) hides the sprite, but the sprite - still affects the priority ordering - a better way to hide a sprite is - to set its Y-coordinate offscreen. - """ - self.x = data # - 8 - self.hide_check() - - def extract_tile_number(self, data): - """ - extracts the Tile/Pattern Number - Specifies the sprites Tile Number (00-FF). This (unsigned) value selects - a tile from memory at 8000h-8FFFh. In CGB Mode this could be either in - VRAM Bank 0 or 1, depending on Bit 3 of the following byte. - In 8x16 mode, the lower bit of the tile number is ignored. Ie. the - upper 8x8 tile is "NN AND FEh", and the lower 8x8 tile is "NN OR 01h". - """ - self.tile_number = data - - def extract_attributes_and_flags(self, data): - """ - extracts the Attributes/Flags: - Bit7 OBJ-to-BG Priority (0=OBJ Above BG, 1=OBJ Behind BG color 1-3) - (Used for both BG and Window. BG color 0 is always behind OBJ) - Bit6 Y flip (0=Normal, 1=Vertically mirrored) - Bit5 X flip (0=Normal, 1=Horizontally mirrored) - Bit4 Palette number **Non CGB Mode Only** (0=OBP0, 1=OBP1) - """ - self.object_behind_background = bool(data & (1 << 7)) - self.x_flipped = bool(data & (1 << 6)) - self.y_flipped = bool(data & (1 << 5)) - self.palette_number = bool(data & (1 << 3)) - - def get_attributes_and_flags(self): - value = 0 - value += int(self.object_behind_background) << 7 - value += int(self.x_flipped) << 6 - value += int(self.y_flipped) << 5 - value += int(self.palette_number ) << 3 - return value - - def hide_check(self): - if self.y <= 0 or self.y >= GAMEBOY_SCREEN_WIDTH: - self.hidden = True - elif self.x <= 0 or self.x >= GAMEBOY_SCREEN_WIDTH+SPRITE_SIZE: - self.hidden = True - else: - self.hidden = False - - def get_tile_number(self): - return self.tile.id - - def get_width(self): - return SPRITE_SIZE - - def get_height(self): - if self.big_size: - return 2*SPRITE_SIZE - else: - return SPRITE_SIZE - - def overlaps_on_line(self, sprite, line): - return False - - def intersects_line(self, line): - return line >= self.y and line <= self.y + self.get_height() - - def draw(self): - pass - - def draw_overlapped(self): - pass - -# ----------------------------------------------------------------------------- - - -class Tile(object): - - def __init__(self): - self.reset() - - def reset(self): - pass - - def set_tile_data(self, data): - pass - - def get_pixel_data(self): - return self.pixel_data - - def get_selected_tile_map_space(self): - pass -# ----------------------------------------------------------------------------- - -class Window(object): - - def __init__(self, video): - self.video = video - self.reset() - - def reset(self): - self.x = 0 - self.y = 0 - self.line_y = 0 - self.enabled = False - self.upper_tile_map_selected = False - - def update_line_y(self, data): - # don't draw window if it was not enabled and not being drawn before - if not self.enabled and (data & 0x20) != 0 and \ - self.line_y == 0 and self.video.line_y > self.y: - self.line_y = GAMEBOY_SCREEN_HEIGHT - - def get_tile_map_space(self): - #if (self.control.read() & mask) != 0: - if self.upper_tile_map_selected: - return constants.VRAM_MAP_B - else: - return constants.VRAM_MAP_A - - def draw_line(self, line_y): - if line_y < self.y or self.x >= 167 or \ - self.line_y >= GAMEBOY_SCREEN_HEIGHT: - return - else: - tile_map, tile_data = self.prepare_window_data() - self.video.draw_tiles(self.x + 1, tile_map, tile_data) - self.line_y += 1 - - def prepare_window_data(self): - tile_map = self.get_tile_map_space() - tile_map += (self.line_y >> 3) << 5 - tile_data = self.video.control.get_selected_tile_data_space() - tile_data += (self.line_y & 7) << 1 - return tile_map, tile_data; - -# ----------------------------------------------------------------------------- -class Background(object): - - def __init__(self, video): - self.video = video - self.reset() - - def reset(self): - # SCROLLX and SCROLLY hold the coordinates of background to - # be displayed in the left upper corner of the screen. - self.scroll_x = 0 - self.scroll_y = 0 - self.enabled = True - self.upper_tile_map_selected = False - - def get_tile_map_space(self): - #if (self.control.read() & mask) != 0: - if self.upper_tile_map_selected: - return constants.VRAM_MAP_B - else: - return constants.VRAM_MAP_A - - def draw_clean_line(self, line_y): - for x in range(8+GAMEBOY_SCREEN_WIDTH+8): - self.video.line[x] = 0x00 - - def draw_line(self, line_y): - y = (self.scroll_y + line_y) & 0xFF - x = self.scroll_x & 0xFF - tile_map, tile_data = self.prepare_background_data(x, y) - self.video.draw_tiles(8 - (x & 7), tile_map, tile_data) - - def prepare_background_data(self, x, y): - tile_map = self.get_tile_map_space() - tile_map += ((y >> 3) << 5) + (x >> 3) - tile_data = self.video.control.get_selected_tile_data_space() - tile_data += (y & 7) << 1 - return tile_map, tile_data - - # ----------------------------------------------------------------------------- class Video(iMemory): Added: pypy/dist/pypy/lang/gameboy/video_mode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/video_mode.py Sun Sep 28 00:39:24 2008 @@ -0,0 +1,239 @@ + +from pypy.lang.gameboy import constants +from pypy.lang.gameboy.constants import SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH, \ + GAMEBOY_SCREEN_HEIGHT + +# ----------------------------------------------------------------------------- +class InvalidModeOrderException(Exception): + def __init__(self, mode, previous_mode): + Exception.__init__(self, \ + "Wrong Mode order! No valid transition %i => %i" \ + % (previous_mode.id(), mode.id())) + +class HandleSubclassException(Exception): + def __init__(self): + Exception.__init__(self, "") + +class Mode(object): + + """ + The two lower STAT bits show the current status of the LCD controller. + """ + + def __init__(self, video): + self.video = video + self.reset() + + def reset(self): + raise Exception("unimplemented method") + + def id(self): + raise Exception("unimplemented method") + + def activate(self, previous_mode): + raise Exception("unimplemented method") + + def emulate(self): + raise Exception("unimplemented method") + + def emulate_hblank_line_y_compare(self, stat_check=False): + if self.video.line_y == self.video.line_y_compare: + if not (stat_check and self.video.status.line_y_compare_flag): + self.line_y_line_y_compare_interrupt_check() + else: + self.video.status.line_y_compare_flag = False + + def line_y_line_y_compare_interrupt_check(self): + self.video.status.line_y_compare_flag = True + if self.video.status.line_y_compare_interrupt: + self.video.lcd_interrupt_flag.set_pending() + +class Mode0(Mode): + """ + Mode 0: The LCD controller is in the H-Blank period and + the CPU can access both the display RAM (8000h-9FFFh) + and OAM (FE00h-FE9Fh) + """ + + def reset(self): + self.h_blank_interrupt = False + + def id(self): + return 0 + + def activate(self, previous_mode): + #if previous_mode.id() == 3: + self.video.cycles += constants.MODE_0_TICKS + self.h_blank_interrupt_check() + #else: + # video.reset_control() can be called in any position + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def h_blank_interrupt_check(self): + if self.h_blank_interrupt and \ + self.video.status.line_y_compare_check(): + self.video.lcd_interrupt_flag.set_pending() + + def emulate(self): + #self.video.emulate_hblank() + self.emulate_hblank() + + def emulate_hblank(self): + self.video.line_y += 1 + self.emulate_hblank_line_y_compare() + if self.video.line_y < GAMEBOY_SCREEN_HEIGHT: + self.video.status.set_mode(2) + else: + self.emulate_hblank_part_2() + + def emulate_hblank_part_2(self): + if self.video.display: + self.video.draw_frame() + self.video.frames += 1 + if self.video.frames >= self.video.frame_skip: + self.video.display = True + self.video.frames = 0 + else: + self.video.display = False + self.video.status.set_mode(1) + self.video.v_blank = True + +class Mode1(Mode): + """ + Mode 1: The LCD contoller is in the V-Blank period (or the + display is disabled) and the CPU can access both the + display RAM (8000h-9FFFh) and OAM (FE00h-FE9Fh) + """ + + def reset(self): + self.v_blank_interrupt = False + + def id(self): + return 1 + + def activate(self, previous_mode): + #if previous_mode.id() == 0: + self.set_begin() + #else: + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def set_begin(self): + self.video.cycles += constants.MODE_1_BEGIN_TICKS + + def set_between(self): + self.video.cycles += constants.MODE_1_TICKS - constants.MODE_1_BEGIN_TICKS + + def set_end(self): + self.video.cycles += constants.MODE_1_END_TICKS + + def emulate(self): + self.emulate_v_blank() + + def emulate_v_blank(self): + if self.video.v_blank: + self.emulate_v_blank_v_blank() + elif self.video.line_y == 0: + self.video.status.set_mode(2) + else: + self.emulate_v_blank_other() + + def emulate_v_blank_v_blank(self): + self.video.v_blank = False + self.set_between() + self.v_blank_interrupt_check() + + def v_blank_interrupt_check(self): + if self.v_blank_interrupt: + self.video.lcd_interrupt_flag.set_pending() + self.video.v_blank_interrupt_flag.set_pending() + + def emulate_v_blank_other(self): + if self.video.line_y < 153: + self.emulate_v_blank_mode_1() + else: + self.video.line_y = 0 + self.video.window.line_y = 0 + self.set_between() + self.emulate_hblank_line_y_compare() + + def emulate_v_blank_mode_1(self): + self.video.line_y += 1 + if self.video.line_y != 153: + self.video.cycles += constants.MODE_1_TICKS + else: + self.set_end() + +class Mode2(Mode): + """ + Mode 2: The LCD controller is reading from OAM memory. + The CPU access OAM memory (FE00h-FE9Fh) + during this period. + """ + + def reset(self): + self.oam_interrupt = False + + def id(self): + return 2 + + def activate(self, previous_mode): + #if previous_mode.id() == 0 or previous_mode.id() == 1: + self.video.cycles += constants.MODE_2_TICKS + self.oam_interrupt_check() + #else: + # pass + #raise InvalidModeOrderException(self, previous_mode) + + def oam_interrupt_check(self): + if self.oam_interrupt and \ + self.video.status.line_y_compare_check(): + self.video.lcd_interrupt_flag.set_pending() + + def emulate(self): + self.emulate_oam() + + def emulate_oam(self): + self.video.status.set_mode(3) + +class Mode3(Mode): + """ + Mode 3: The LCD controller is reading from both OAM and VRAM, + The CPU access OAM and VRAM during this period. + CGB Mode: Cannot access Palette Data (FF69,FF6B) either. + """ + + def reset(self): + pass + + def id(self): + return 3 + + def activate(self, previous_mode): + #if previous_mode.id() == 2: + self.set_begin() + #else: + # pass + # #raise InvalidModeOrderException(self, previous_mode) + + def set_begin(self): + self.video.cycles += constants.MODE_3_BEGIN_TICKS + self.video.transfer = True + + def set_end(self): + self.video.cycles += constants.MODE_3_END_TICKS + self.video.transfer = False + + def emulate(self): + self.emulate_transfer() + + def emulate_transfer(self): + if self.video.transfer: + if self.video.display: + self.video.draw_line() + #print "mode 3 ", self.video.status.get_mode() + self.set_end() + else: + self.video.status.set_mode(0) + \ No newline at end of file Added: pypy/dist/pypy/lang/gameboy/video_register.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/video_register.py Sun Sep 28 00:39:24 2008 @@ -0,0 +1,134 @@ + +from pypy.lang.gameboy import constants +from pypy.lang.gameboy.video_mode import Mode0, Mode1, Mode2, Mode3 +# ----------------------------------------------------------------------------- +class StatusRegister(object): + """ + Bit 6 - LYC=LY Coincidence Interrupt (1=Enable) (Read/Write) + Bit 5 - Mode 2 OAM Interrupt (1=Enable) (Read/Write) + Bit 4 - Mode 1 V-Blank Interrupt (1=Enable) (Read/Write) + Bit 3 - Mode 0 H-Blank Interrupt (1=Enable) (Read/Write) + Bit 2 - Coincidence Flag (0:LYC<>LY, 1:LYC=LY) (Read Only) + Bit 1-0 - Mode Flag (Mode 0-3, see below) (Read Only) + 0: During H-Blank + 1: During V-Blank + 2: During Searching OAM-RAM + 3: During Transfering Data to LCD Driver + """ + def __init__(self, video): + self.create_modes(video) + self.reset() + + def create_modes(self, video): + self.mode0 = Mode0(video) + self.mode1 = Mode1(video) + self.mode2 = Mode2(video) + self.mode3 = Mode3(video) + self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] + + + + def reset(self): + self.current_mode = self.mode2 + self.line_y_compare_flag = False + self.line_y_compare_interrupt = False + for mode in self.modes: + mode.reset() + #self.mode_0_h_blank_interrupt = False + #self.mode_1_v_blank_interrupt = False + #self.mode_2_oam_interrupt = False + self.status = True + + + def read(self, extend=False): + value = self.get_mode() + value += self.line_y_compare_flag << 2 + value += self.mode0.h_blank_interrupt << 3 + value += self.mode1.v_blank_interrupt << 4 + value += self.mode2.oam_interrupt << 5 + value += self.line_y_compare_interrupt << 6 + if extend: + value += int(self.status) << 7 + return value + + + def write(self, value, write_all=False, + keep_mode_0_h_blank_interrupt=False): + if write_all: + self.current_mode = self.modes[value & 0x03] + self.line_y_compare_flag = bool(value & (1 << 2)) + self.status = bool(value & (1 << 7)) + self.mode0.h_blank_interrupt = bool(value & (1 << 3)) + self.mode1.v_blank_interrupt = bool(value & (1 << 4)) + self.mode2.oam_interrupt = bool(value & (1 << 5)) + self.line_y_compare_interrupt = bool(value & (1 << 6)) + + def get_mode(self): + return self.current_mode.id() + + def set_mode(self, mode): + old_mode = self.current_mode + self.current_mode = self.modes[mode & 0x03] + self.current_mode.activate(old_mode) + + def line_y_compare_check(self): + return not self.line_y_compare_flag or not self.line_y_compare_interrupt + +# ----------------------------------------------------------------------------- + +class ControlRegister(object): + """ + used for enabled or disabled window or background + Bit 7 - LCD Display Enable (0=Off, 1=On) + Bit 6 - Window Tile Map Display Select (0=9800-9BFF, 1=9C00-9FFF) + Bit 5 - Window Display Enable (0=Off, 1=On) + Bit 4 - BG & Window Tile Data Select (0=8800-97FF, 1=8000-8FFF) + Bit 3 - BG Tile Map Display Select (0=9800-9BFF, 1=9C00-9FFF) + Bit 2 - OBJ (Sprite) Size (0=8x8, 1=8x16) + Bit 1 - OBJ (Sprite) Display Enable (0=Off, 1=On) + Bit 0 - BG Display (for CGB see below) (0=Off, 1=On) + """ + def __init__(self, window, background): + self.window = window + self.background = background + self.reset() + + def reset(self): + self.lcd_enabled = True + self.window.upper_tile_map_selected = False + self.window.enabled = False + self.background_and_window_lower_tile_data_selected = True + self.background.upper_tile_map_selected = False + self.big_sprite_size_selected = False + self.sprites_enabled = False + self.background.enabled = True + + def read(self): + value = 0 + value += int(self.lcd_enabled) << 7 + value += int(self.window.upper_tile_map_selected) << 6 + value += int(self.window.enabled) << 5 + value += int(self.background_and_window_lower_tile_data_selected) << 4 + value += int(self.background.upper_tile_map_selected) << 3 + value += int(self.big_sprite_size_selected) << 2 + value += int(self.sprites_enabled) << 1 + value += int(self.background.enabled) << 0 + return value + + def write(self, value): + self.lcd_enabled = bool(value & (1 << 7)) + self.window.upper_tile_map_selected = bool(value & (1 << 6)) + self.window.enabled = bool(value & (1 << 5)) + self.background_and_window_lower_tile_data_selected = \ + bool(value & (1 << 4)) + self.background.upper_tile_map_selected = bool(value & (1 << 3)) + self.big_sprite_size_selected = bool(value & (1 << 2)) + self.sprites_enabled = bool(value & (1 << 1)) + self.background.enabled = bool(value & (1 << 0)) + + + def get_selected_tile_data_space(self): + if self.window.upper_tile_map_selected: + return constants.VRAM_DATA_A + else: + return constants.VRAM_DATA_B Added: pypy/dist/pypy/lang/gameboy/video_sprite.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/video_sprite.py Sun Sep 28 00:39:24 2008 @@ -0,0 +1,231 @@ + +from pypy.lang.gameboy import constants +from pypy.lang.gameboy.constants import SPRITE_SIZE, GAMEBOY_SCREEN_WIDTH, \ + GAMEBOY_SCREEN_HEIGHT + +# ----------------------------------------------------------------------------- + + +class Sprite(object): + + def __init__(self, video): + self.video = video + self.big_size = False + self.reset() + + def reset(self): + self.x = 0 + self.y = 0 + self.tile = None + self.object_behind_background = False + self.x_flipped = False + self.y_flipped = False + self.palette_number = 0 + self.hidden = True + + + def get_data(self): + return [self.y, self.x, self.tile_number, self.get_attributes_and_flags()] + + def set_data(self, byte0=-1, byte1=-1, byte2=-1, byte3=-1): + """ + extracts the sprite data from an oam entry + """ + if byte0 is not -1: + self.extract_y_position(byte0) + if byte0 is not -1: + self.extract_x_position(byte1) + if byte0 is not -1: + self.extract_tile_number(byte2) + if byte0 is not -1: + self.extract_attributes_and_flags(byte3) + + def extract_y_position(self, data): + """ + extracts the Y Position + Specifies the sprites vertical position on the screen (minus 16). + An offscreen value (for example, Y=0 or Y>=160) hides the sprite. + """ + self.y = data # - 16 + self.hide_check() + + def extract_x_position(self, data): + """ + extracts the X Position + Specifies the sprites horizontal position on the screen (minus 8). + An offscreen value (X=0 or X>=168) hides the sprite, but the sprite + still affects the priority ordering - a better way to hide a sprite is + to set its Y-coordinate offscreen. + """ + self.x = data # - 8 + self.hide_check() + + def extract_tile_number(self, data): + """ + extracts the Tile/Pattern Number + Specifies the sprites Tile Number (00-FF). This (unsigned) value selects + a tile from memory at 8000h-8FFFh. In CGB Mode this could be either in + VRAM Bank 0 or 1, depending on Bit 3 of the following byte. + In 8x16 mode, the lower bit of the tile number is ignored. Ie. the + upper 8x8 tile is "NN AND FEh", and the lower 8x8 tile is "NN OR 01h". + """ + self.tile_number = data + + def extract_attributes_and_flags(self, data): + """ + extracts the Attributes/Flags: + Bit7 OBJ-to-BG Priority (0=OBJ Above BG, 1=OBJ Behind BG color 1-3) + (Used for both BG and Window. BG color 0 is always behind OBJ) + Bit6 Y flip (0=Normal, 1=Vertically mirrored) + Bit5 X flip (0=Normal, 1=Horizontally mirrored) + Bit4 Palette number **Non CGB Mode Only** (0=OBP0, 1=OBP1) + """ + self.object_behind_background = bool(data & (1 << 7)) + self.x_flipped = bool(data & (1 << 6)) + self.y_flipped = bool(data & (1 << 5)) + self.palette_number = bool(data & (1 << 3)) + + def get_attributes_and_flags(self): + value = 0 + value += int(self.object_behind_background) << 7 + value += int(self.x_flipped) << 6 + value += int(self.y_flipped) << 5 + value += int(self.palette_number ) << 3 + return value + + def hide_check(self): + if self.y <= 0 or self.y >= GAMEBOY_SCREEN_WIDTH: + self.hidden = True + elif self.x <= 0 or self.x >= GAMEBOY_SCREEN_WIDTH+SPRITE_SIZE: + self.hidden = True + else: + self.hidden = False + + def get_tile_number(self): + return self.tile.id + + def get_width(self): + return SPRITE_SIZE + + def get_height(self): + if self.big_size: + return 2*SPRITE_SIZE + else: + return SPRITE_SIZE + + def overlaps_on_line(self, sprite, line): + return False + + def intersects_line(self, line): + return line >= self.y and line <= self.y + self.get_height() + + def draw(self): + pass + + def draw_overlapped(self): + pass + +# ----------------------------------------------------------------------------- + + +class Tile(object): + + def __init__(self): + self.reset() + + def reset(self): + pass + + def set_tile_data(self, data): + pass + + def get_pixel_data(self): + return self.pixel_data + + def get_selected_tile_map_space(self): + pass +# ----------------------------------------------------------------------------- + +class Window(object): + + def __init__(self, video): + self.video = video + self.reset() + + def reset(self): + self.x = 0 + self.y = 0 + self.line_y = 0 + self.enabled = False + self.upper_tile_map_selected = False + + def update_line_y(self, data): + # don't draw window if it was not enabled and not being drawn before + if not self.enabled and (data & 0x20) != 0 and \ + self.line_y == 0 and self.video.line_y > self.y: + self.line_y = GAMEBOY_SCREEN_HEIGHT + + def get_tile_map_space(self): + #if (self.control.read() & mask) != 0: + if self.upper_tile_map_selected: + return constants.VRAM_MAP_B + else: + return constants.VRAM_MAP_A + + def draw_line(self, line_y): + if line_y < self.y or self.x >= 167 or \ + self.line_y >= GAMEBOY_SCREEN_HEIGHT: + return + else: + tile_map, tile_data = self.prepare_window_data() + self.video.draw_tiles(self.x + 1, tile_map, tile_data) + self.line_y += 1 + + def prepare_window_data(self): + tile_map = self.get_tile_map_space() + tile_map += (self.line_y >> 3) << 5 + tile_data = self.video.control.get_selected_tile_data_space() + tile_data += (self.line_y & 7) << 1 + return tile_map, tile_data; + +# ----------------------------------------------------------------------------- + +class Background(object): + + def __init__(self, video): + self.video = video + self.reset() + + def reset(self): + # SCROLLX and SCROLLY hold the coordinates of background to + # be displayed in the left upper corner of the screen. + self.scroll_x = 0 + self.scroll_y = 0 + self.enabled = True + self.upper_tile_map_selected = False + + def get_tile_map_space(self): + #if (self.control.read() & mask) != 0: + if self.upper_tile_map_selected: + return constants.VRAM_MAP_B + else: + return constants.VRAM_MAP_A + + def draw_clean_line(self, line_y): + for x in range(8+GAMEBOY_SCREEN_WIDTH+8): + self.video.line[x] = 0x00 + + def draw_line(self, line_y): + y = (self.scroll_y + line_y) & 0xFF + x = self.scroll_x & 0xFF + tile_map, tile_data = self.prepare_background_data(x, y) + self.video.draw_tiles(8 - (x & 7), tile_map, tile_data) + + def prepare_background_data(self, x, y): + tile_map = self.get_tile_map_space() + tile_map += ((y >> 3) << 5) + (x >> 3) + tile_data = self.video.control.get_selected_tile_data_space() + tile_data += (y & 7) << 1 + return tile_map, tile_data + + \ No newline at end of file From cami at codespeak.net Sun Sep 28 01:12:29 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 28 Sep 2008 01:12:29 +0200 (CEST) Subject: [pypy-svn] r58456 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080927231229.CE33016A1FD@codespeak.net> Author: cami Date: Sun Sep 28 01:12:27 2008 New Revision: 58456 Added: pypy/dist/pypy/lang/gameboy/test/test_video_mode.py Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py pypy/dist/pypy/lang/gameboy/video_mode.py pypy/dist/pypy/lang/gameboy/video_register.py Log: added test for video mode extended video register tests Added: pypy/dist/pypy/lang/gameboy/test/test_video_mode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/gameboy/test/test_video_mode.py Sun Sep 28 01:12:27 2008 @@ -0,0 +1,22 @@ +from pypy.lang.gameboy import constants +from pypy.lang.gameboy.video_sprite import Sprite +from pypy.lang.gameboy.video import Video +from pypy.lang.gameboy.test.test_video import get_video +import py + +# ------------------------------------------------------------------------------ + +def get_mode0(): + return Mode0(get_video()) + +def get_mode1(): + return Mode1(get_video()) + +def get_mode2(): + return Mode2(get_video()) + +def get_mode3(): + return Mode3(get_video()) + + +# ------------------------------------------------------------------------------ \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/test/test_video_registers.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_registers.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_registers.py Sun Sep 28 01:12:27 2008 @@ -22,17 +22,24 @@ control.write(0xFF) assert control.read() == 0xFF control.reset() - assert control.read() == 0x91 + assert control.read() == 0x91 def test_video_control_read_write_properties(): control = get_control_register() - for i in range(0xFF): control.write(i) assert control.read() == i - +def test_video_control_get_selected_tile_data_space(): + control = get_control_register() + + control.window.upper_tile_map_selected = True + assert control.get_selected_tile_data_space() == constants.VRAM_DATA_A + + control.window.upper_tile_map_selected = False + assert control.get_selected_tile_data_space() == constants.VRAM_DATA_B + # StatusRegister --------------------------------------------------------------- def test_video_status_reset(): @@ -52,9 +59,61 @@ def test_video_status_mode(): status = get_status_register() assert status.get_mode() == 2 - for i in range(3): status.set_mode(i) assert status.get_mode() == i + status.set_mode(4) assert status.get_mode() == 0 + +def test_video_status_mode_properties(): + status = get_status_register() + status.write(0x00, write_all=True) + + assert status.read(extend=True) == 0x00 + + status.mode0.h_blank_interrupt = True + assert status.read(extend=True) == (1 << 3) + status.mode0.h_blank_interrupt = False + + status.mode1.v_blank_interrupt = True + assert status.read(extend=True) == (1 << 4) + status.mode1.v_blank_interrupt = False + + status.mode2.oam_interrupt = True + assert status.read(extend=True) == (1 << 5) + status.mode2.oam_interrupt = False + +def test_video_status_get_mode(): + status = get_status_register() + status.current_mode = status.mode0 + assert status.get_mode() == 0 + + for i in range(0,4): + status.current_mode = status.modes[i] + assert status.get_mode() == status.modes[i].id() + +def test_video_status_set_mode(): + status = get_status_register() + for i in range(0,4): + status.set_mode(i) + assert status.current_mode == status.modes[i] + +def test_video_status_line_y_compare_check(): + status = get_status_register() + + status.line_y_compare_flag = False + status.line_y_compare_interrupt = False + assert status.line_y_compare_check() + + status.line_y_compare_flag = True + status.line_y_compare_interrupt = False + assert status.line_y_compare_check() + + status.line_y_compare_flag = False + status.line_y_compare_interrupt = True + assert status.line_y_compare_check() + + status.line_y_compare_flag = True + status.line_y_compare_interrupt = True + assert not status.line_y_compare_check() \ No newline at end of file Modified: pypy/dist/pypy/lang/gameboy/video_mode.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video_mode.py (original) +++ pypy/dist/pypy/lang/gameboy/video_mode.py Sun Sep 28 01:12:27 2008 @@ -48,6 +48,8 @@ if self.video.status.line_y_compare_interrupt: self.video.lcd_interrupt_flag.set_pending() +# ----------------------------------------------------------------------------- + class Mode0(Mode): """ Mode 0: The LCD controller is in the H-Blank period and @@ -61,14 +63,9 @@ def id(self): return 0 - def activate(self, previous_mode): - #if previous_mode.id() == 3: - self.video.cycles += constants.MODE_0_TICKS - self.h_blank_interrupt_check() - #else: - # video.reset_control() can be called in any position - # pass - #raise InvalidModeOrderException(self, previous_mode) + def activate(self): + self.video.cycles += constants.MODE_0_TICKS + self.h_blank_interrupt_check() def h_blank_interrupt_check(self): if self.h_blank_interrupt and \ @@ -98,7 +95,9 @@ self.video.display = False self.video.status.set_mode(1) self.video.v_blank = True - + +# ----------------------------------------------------------------------------- + class Mode1(Mode): """ Mode 1: The LCD contoller is in the V-Blank period (or the @@ -112,12 +111,8 @@ def id(self): return 1 - def activate(self, previous_mode): - #if previous_mode.id() == 0: - self.set_begin() - #else: - # pass - #raise InvalidModeOrderException(self, previous_mode) + def activate(self): + self.set_begin() def set_begin(self): self.video.cycles += constants.MODE_1_BEGIN_TICKS @@ -164,7 +159,9 @@ self.video.cycles += constants.MODE_1_TICKS else: self.set_end() - + +# ----------------------------------------------------------------------------- + class Mode2(Mode): """ Mode 2: The LCD controller is reading from OAM memory. @@ -178,13 +175,9 @@ def id(self): return 2 - def activate(self, previous_mode): - #if previous_mode.id() == 0 or previous_mode.id() == 1: - self.video.cycles += constants.MODE_2_TICKS - self.oam_interrupt_check() - #else: - # pass - #raise InvalidModeOrderException(self, previous_mode) + def activate(self): + self.video.cycles += constants.MODE_2_TICKS + self.oam_interrupt_check() def oam_interrupt_check(self): if self.oam_interrupt and \ @@ -197,6 +190,8 @@ def emulate_oam(self): self.video.status.set_mode(3) +# ----------------------------------------------------------------------------- + class Mode3(Mode): """ Mode 3: The LCD controller is reading from both OAM and VRAM, @@ -210,12 +205,8 @@ def id(self): return 3 - def activate(self, previous_mode): - #if previous_mode.id() == 2: - self.set_begin() - #else: - # pass - # #raise InvalidModeOrderException(self, previous_mode) + def activate(self): + self.set_begin() def set_begin(self): self.video.cycles += constants.MODE_3_BEGIN_TICKS Modified: pypy/dist/pypy/lang/gameboy/video_register.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video_register.py (original) +++ pypy/dist/pypy/lang/gameboy/video_register.py Sun Sep 28 01:12:27 2008 @@ -1,7 +1,9 @@ from pypy.lang.gameboy import constants from pypy.lang.gameboy.video_mode import Mode0, Mode1, Mode2, Mode3 + # ----------------------------------------------------------------------------- + class StatusRegister(object): """ Bit 6 - LYC=LY Coincidence Interrupt (1=Enable) (Read/Write) @@ -26,8 +28,6 @@ self.mode3 = Mode3(video) self.modes = [self.mode0, self.mode1, self.mode2, self.mode3] - - def reset(self): self.current_mode = self.mode2 self.line_y_compare_flag = False @@ -39,7 +39,6 @@ #self.mode_2_oam_interrupt = False self.status = True - def read(self, extend=False): value = self.get_mode() value += self.line_y_compare_flag << 2 @@ -51,9 +50,7 @@ value += int(self.status) << 7 return value - - def write(self, value, write_all=False, - keep_mode_0_h_blank_interrupt=False): + def write(self, value, write_all=False): if write_all: self.current_mode = self.modes[value & 0x03] self.line_y_compare_flag = bool(value & (1 << 2)) @@ -67,12 +64,11 @@ return self.current_mode.id() def set_mode(self, mode): - old_mode = self.current_mode self.current_mode = self.modes[mode & 0x03] - self.current_mode.activate(old_mode) + self.current_mode.activate() def line_y_compare_check(self): - return not self.line_y_compare_flag or not self.line_y_compare_interrupt + return not (self.line_y_compare_flag and self.line_y_compare_interrupt) # ----------------------------------------------------------------------------- From cami at codespeak.net Sun Sep 28 01:47:14 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 28 Sep 2008 01:47:14 +0200 (CEST) Subject: [pypy-svn] r58457 - pypy/dist/pypy/lang/gameboy/test Message-ID: <20080927234714.6CC731684A3@codespeak.net> Author: cami Date: Sun Sep 28 01:47:11 2008 New Revision: 58457 Modified: pypy/dist/pypy/lang/gameboy/test/test_video_mode.py Log: more test for the modes Modified: pypy/dist/pypy/lang/gameboy/test/test_video_mode.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_mode.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_mode.py Sun Sep 28 01:47:11 2008 @@ -2,10 +2,20 @@ from pypy.lang.gameboy.video_sprite import Sprite from pypy.lang.gameboy.video import Video from pypy.lang.gameboy.test.test_video import get_video +from pypy.lang.gameboy.video_mode import Mode0, Mode1, Mode2, Mode3 + import py # ------------------------------------------------------------------------------ + +class CallChecker(object): + def __init__(self): + self.called = False + + def __call__(self): + self.called = True + def get_mode0(): return Mode0(get_video()) @@ -19,4 +29,113 @@ return Mode3(get_video()) -# ------------------------------------------------------------------------------ \ No newline at end of file +# ------------------------------------------------------------------------------ + +def test_mode_emulate_hblank_line_y_compare(): + mode = get_mode0() + + mode.video.status.line_y_compare_flag == True + mode.video.line_y = 0 + mode.video.line_y_compare = 1 + mode.emulate_hblank_line_y_compare() + assert mode.video.status.line_y_compare_flag == False + + mode.video.line_y = 0 + mode.video.line_y_compare = 0 + mode.video.status.line_y_compare_flag = True + mode.emulate_hblank_line_y_compare(stat_check=True) + assert mode.video.status.line_y_compare_flag == True + + mode.video.status.line_y_compare_flag = True + mode.emulate_hblank_line_y_compare(stat_check=False) + assert mode.video.status.line_y_compare_flag + + mode.video.status.line_y_compare_flag = False + mode.emulate_hblank_line_y_compare(stat_check=False) + assert mode.video.status.line_y_compare_flag + + +def test_mode_line_y_line_y_compare_interrupt_check(): + mode = get_mode0() + + mode.video.status.line_y_compare_flag = False + mode.video.status.line_y_compare_interrupt = False + mode.line_y_line_y_compare_interrupt_check() + assert mode.video.status.line_y_compare_flag + assert mode.video.lcd_interrupt_flag.is_pending() == False + + mode.video.status.line_y_compare_flag = False + mode.video.status.line_y_compare_interrupt = True + mode.line_y_line_y_compare_interrupt_check() + assert mode.video.status.line_y_compare_flag + assert mode.video.lcd_interrupt_flag.is_pending() + +def test_mode_ids(): + assert get_mode0().id() == 0 + assert get_mode1().id() == 1 + assert get_mode2().id() == 2 + assert get_mode3().id() == 3 + +# ------------------------------------------------------------------------------ + +def test_mode0_activate(): + mode = get_mode0() + mode.video.cycles = 0 + mode.activate() + assert mode.video.cycles == constants.MODE_0_TICKS + +def test_mode0_h_blank_interrupt_check(): + mode = get_mode0() + assert not mode.video.lcd_interrupt_flag.is_pending() + mode.h_blank_interrupt_check(); + assert not mode.video.lcd_interrupt_flag.is_pending() + + mode.h_blank_interrupt = True + mode.video.status.line_y_compare_check = lambda: True; + mode.h_blank_interrupt_check(); + assert mode.video.lcd_interrupt_flag.is_pending() + + mode.h_blank_interrupt = False + mode.video.lcd_interrupt_flag.set_pending(False) + mode.video.status.line_y_compare_check = lambda: True; + mode.h_blank_interrupt_check(); + assert not mode.video.lcd_interrupt_flag.is_pending() + + mode.h_blank_interrupt = False + mode.video.status.line_y_compare_check = lambda: False; + mode.h_blank_interrupt_check(); + assert not mode.video.lcd_interrupt_flag.is_pending() + +def test_mode0_emulate_hblank(): + mode = get_mode0() + mode.video.line_y = 0 + mode.video.status.current_mode = mode + mode.emulate_hblank() + assert mode.video.line_y == 1 + assert mode.video.status.get_mode() == 2 + + mode.video.line_y = constants.GAMEBOY_SCREEN_HEIGHT-1 + mode.video.frames = 0 + mode.video.status.current_mode = mode + mode.video.frame_skip = 10 + mode.emulate_hblank() + assert mode.video.line_y == constants.GAMEBOY_SCREEN_HEIGHT + assert mode.video.frames == 1 + assert mode.video.v_blank == True + assert mode.video.display == False + assert mode.video.status.get_mode() == 1 + + mode.video.line_y = constants.GAMEBOY_SCREEN_HEIGHT-1 + mode.video.frames = 0 + mode.video.display = True + mode.video.draw_frame = CallChecker() + mode.video.status.current_mode = mode + mode.video.frame_skip = 10 + mode.emulate_hblank() + assert mode.video.draw_frame.called == True + assert mode.video.line_y == constants.GAMEBOY_SCREEN_HEIGHT + assert mode.video.frames == 1 + assert mode.video.v_blank == True + assert mode.video.display == False + assert mode.video.status.get_mode() == 1 + \ No newline at end of file From cami at codespeak.net Sun Sep 28 02:09:09 2008 From: cami at codespeak.net (cami at codespeak.net) Date: Sun, 28 Sep 2008 02:09:09 +0200 (CEST) Subject: [pypy-svn] r58458 - in pypy/dist/pypy/lang/gameboy: . test Message-ID: <20080928000909.1323F16A0D0@codespeak.net> Author: cami Date: Sun Sep 28 02:09:08 2008 New Revision: 58458 Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py pypy/dist/pypy/lang/gameboy/video.py pypy/dist/pypy/lang/gameboy/video_sprite.py Log: oam table acces runs directly on sprites now added new tests to cover the direct access made some changes for future direct acces on the vram table Modified: pypy/dist/pypy/lang/gameboy/test/test_video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video.py Sun Sep 28 02:09:08 2008 @@ -79,6 +79,15 @@ assert video.read(address) == value counted_value = (counted_value + 1 ) % 0xFF +def test_video_read_write_oam(): + video = get_video() + value = 0 + for i in range(constants.OAM_ADDR, constants.OAM_ADDR + constants.OAM_SIZE): + video.write(i, value) + assert video.read(i) == value + value = (value + 1) & 0xFF + + def test_read_write_control(): video = get_video() value = 0x2 Modified: pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py (original) +++ pypy/dist/pypy/lang/gameboy/test/test_video_sprite.py Sun Sep 28 02:09:08 2008 @@ -21,7 +21,7 @@ assert sprite.object_behind_background == False assert sprite.x_flipped == False assert sprite.y_flipped == False - assert sprite.palette_number == 0 + assert sprite.tile_number == 0 def test_reset(): @@ -54,6 +54,24 @@ sprite.reset() test_standard_values(sprite) +def test_video_sprite_read_write(): + sprite = get_sprite() + for i in range(0xFF): + sprite.set_data(byte0=i) + assert sprite.get_data()[0] == i + + for i in range(0xFF): + sprite.set_data(byte1=i) + assert sprite.get_data()[1] == i + + for i in range(0xFF): + sprite.set_data(byte2=i) + assert sprite.get_data()[2] == i + + for i in range(0xFF): + sprite.set_data(byte3=i) + assert sprite.get_data()[3] == i + def test_size(): sprite = get_sprite() sprite.big_size = False Modified: pypy/dist/pypy/lang/gameboy/video.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video.py (original) +++ pypy/dist/pypy/lang/gameboy/video.py Sun Sep 28 02:09:08 2008 @@ -414,12 +414,10 @@ the h-blank period. """ self.oam[address - constants.OAM_ADDR] = data & 0xFF - #self.update_sprites(address) self.update_sprite(address, data) def get_oam(self, address): - #return self.get_sprite(address).get_data()[address % 4]; - return self.oam[address - constants.OAM_ADDR] + return self.get_sprite(address).get_data_at(address); def set_vram(self, address, data): """ Modified: pypy/dist/pypy/lang/gameboy/video_sprite.py ============================================================================== --- pypy/dist/pypy/lang/gameboy/video_sprite.py (original) +++ pypy/dist/pypy/lang/gameboy/video_sprite.py Sun Sep 28 02:09:08 2008 @@ -5,7 +5,6 @@ # ----------------------------------------------------------------------------- - class Sprite(object): def __init__(self, video): @@ -20,10 +19,13 @@ self.object_behind_background = False self.x_flipped = False self.y_flipped = False - self.palette_number = 0 + self.tile_number = 0 self.hidden = True + self.rest_attributes_and_flags = 0 - + def get_data_at(self, address): + return self.get_data()[address % 4] + def get_data(self): return [self.y, self.x, self.tile_number, self.get_attributes_and_flags()] @@ -33,11 +35,11 @@ """ if byte0 is not -1: self.extract_y_position(byte0) - if byte0 is not -1: + if byte1 is not -1: self.extract_x_position(byte1) - if byte0 is not -1: + if byte2 is not -1: self.extract_tile_number(byte2) - if byte0 is not -1: + if byte3 is not -1: self.extract_attributes_and_flags(byte3) def extract_y_position(self, data): @@ -83,14 +85,16 @@ self.object_behind_background = bool(data & (1 << 7)) self.x_flipped = bool(data & (1 << 6)) self.y_flipped = bool(data & (1 << 5)) - self.palette_number = bool(data & (1 << 3)) + self.tile_number = bool(data & (1 << 4)) + self.rest_attributes_and_flags = data & (1+2+4+8) def get_attributes_and_flags(self): value = 0 value += int(self.object_behind_background) << 7 value += int(self.x_flipped) << 6 value += int(self.y_flipped) << 5 - value += int(self.palette_number ) << 3 + value += int(self.tile_number) << 4 + value += self.rest_attributes_and_flags return value def hide_check(self): @@ -144,6 +148,16 @@ def get_selected_tile_map_space(self): pass + + def get_data_at(self, address): + return self.get_data()[address % self.byte_size()] + + def get_data(): + return [] + + def byte_size(self): + return 0 + # ----------------------------------------------------------------------------- class Window(object): From witulski at codespeak.net Sun Sep 28 23:58:29 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Sun, 28 Sep 2008 23:58:29 +0200 (CEST) Subject: [pypy-svn] r58461 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080928215829.CD05B169F16@codespeak.net> Author: witulski Date: Sun Sep 28 23:58:28 2008 New Revision: 58461 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Log: Added IDIV + Test Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/assembler.py Sun Sep 28 23:58:28 2008 @@ -29,7 +29,7 @@ # This method wirtes the bitencodings into # the memory. The parameters are overwritten -# if one of the operands is an register +# if one of the operands is an register. # tttn isn't used yet # extra is an extra byte for long opcodes like IMUL def make_two_operand_instr(W = None, R = None, X = None, B = None, opcode =None, m = None, md1 = None, md2 = None, tttn = None, extra = None): @@ -78,7 +78,7 @@ # This method wirtes the bitencodings into # the memory. The parameters are overwritten -# if one of the operands is an register +# if one of the operands is an register. # tttn codes the flags and is only used by SETcc def make_one_operand_instr(W = None, R = None, X = None, B = None, opcode = None, m = None, md1 = None, md2 = None, tttn=None, extra = None): def quadreg_instr(self, arg1): @@ -240,6 +240,19 @@ def DEC(self, op1): method = getattr(self, "_DEC"+op1.to_string()) method(op1) + + def IDIV(self, op1): + method = getattr(self, "_IDIV"+op1.to_string()) + method(op1) + + def IMUL(self, op1, op2): + method = getattr(self, "_IMUL"+op1.to_string()+op2.to_string()) + # exchange the two arguments because + # the result is in the first register + if(op1.to_string()=="_QWREG" and op2.to_string()=="_QWREG"): + method(op2, op1) + else: + method(op1, op2) def INC(self, op1): method = getattr(self, "_INC"+op1.to_string()) @@ -276,20 +289,6 @@ def MOV(self, op1, op2): method = getattr(self, "_MOV"+op1.to_string()+op2.to_string()) method(op1, op2) - - - def IDIV(self, op1): - method = getattr(self, "_IDIV"+op1.to_string()) - method(op1) - - def IMUL(self, op1, op2): - method = getattr(self, "_IMUL"+op1.to_string()+op2.to_string()) - # exchange the two arguments because - # the result is in the first register - if(op1.to_string()=="_QWREG" and op2.to_string()=="_QWREG"): - method(op2, op1) - else: - method(op1, op2) def NEG(self, op1): method = getattr(self, "_NEG"+op1.to_string()) Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Sun Sep 28 23:58:28 2008 @@ -129,9 +129,11 @@ # FIXME: supports only RAX with QWREG def op_int_div(self, gv_x, gv_y): gv_z = self.allocate_register("rax") + gv_w = self.allocate_register("rdx") self.mc.MOV(gv_z, gv_x) + self.mc.XOR(gv_w, gv_w) self.mc.IDIV(gv_y) - return gv_y #FIXME: return gv_x? + return gv_z #FIXME: return gv_x? #FIXME: can only jump 32bit Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Sun Sep 28 23:58:28 2008 @@ -200,13 +200,17 @@ res = fnptr(12345,-9876) assert res == -121919220 - #Floating point exception - #def test_idiv(self): - # rgenop = self.RGenOp() - # div_function = make_div(rgenop) - # fnptr = self.cast(div_function,2) - # res = fnptr(100,2) - # assert res == 50 + #BUG ignores rdx + def test_idiv(self): + rgenop = self.RGenOp() + div_function = make_div(rgenop) + fnptr = self.cast(div_function,2) + res = fnptr(100,2) + assert res == 50 + res = fnptr(168,4) + assert res == 42 + res = fnptr(72057594037927935,5) + assert res == 14411518807585587 def test_greater(self): rgenop = self.RGenOp() @@ -282,8 +286,8 @@ rgenop = self.RGenOp() cmp_function = make_cmp(rgenop, "int_eq",42) fnptr = self.cast(cmp_function,1) - # res = fnptr(42) - # assert res == 1 + res = fnptr(42) + assert res == 1 res = fnptr(23) assert res == 0 cmp_function = make_cmp(rgenop, "int_eq") @@ -306,7 +310,7 @@ assert res == 0 res = fnptr(244,756) assert res == 0 - res = fnptr(-1,9223372036854775807) + res = fnptr(-1,9223372036854775807) #FFFF.... != 7FFF... assert res == 0 def test_not_equal(self): Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_simple.py Sun Sep 28 23:58:28 2008 @@ -30,11 +30,10 @@ builder.finish_and_return(token, genv_result) num = fp(1280, 20) assert num == 1300 - num = fp(1280, -80) - assert num == 1200 num = fp(1280, 1000) assert num == 2280 - print num + num = fp(1280, -80) + assert num == 1200 def test_add_twice(self): builder, fp, inputargs_gv, token = make_testbuilder(2) From witulski at codespeak.net Mon Sep 29 00:20:08 2008 From: witulski at codespeak.net (witulski at codespeak.net) Date: Mon, 29 Sep 2008 00:20:08 +0200 (CEST) Subject: [pypy-svn] r58463 - in pypy/branch/oo-jit/pypy/jit/codegen/x86_64: . test Message-ID: <20080928222008.AD22B169E82@codespeak.net> Author: witulski Date: Mon Sep 29 00:20:05 2008 New Revision: 58463 Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Log: added Mod + tests Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/rgenop.py Mon Sep 29 00:20:05 2008 @@ -133,7 +133,17 @@ self.mc.MOV(gv_z, gv_x) self.mc.XOR(gv_w, gv_w) self.mc.IDIV(gv_y) - return gv_z #FIXME: return gv_x? + return gv_z + + # IDIV RDX:RAX with QWREG + # FIXME: supports only RAX with QWREG + def op_int_mod(self, gv_x, gv_y): + gv_z = self.allocate_register("rax") + gv_w = self.allocate_register("rdx") + self.mc.MOV(gv_z, gv_x) + self.mc.XOR(gv_w, gv_w) + self.mc.IDIV(gv_y) + return gv_w #FIXME: can only jump 32bit Modified: pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/x86_64/test/test_rgenop.py Mon Sep 29 00:20:05 2008 @@ -22,7 +22,7 @@ builder.end() return gv_push_pop -#FIXME: Jumps ever +#FIXME: Jumps ever (CMP BUG) def make_jne(rgenop): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) builder, gv_jne, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "jne") @@ -94,24 +94,6 @@ builder.finish_and_return(sigtoken, gv_result) builder.end() return gv_cmp - -def make_mul(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) - builder, gv_mul, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "mul") - builder.start_writing() - gv_result = builder.genop2("int_mul", gv_x, gv_y) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_mul - -def make_div(rgenop): - sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) - builder, gv_div, [gv_x, gv_y] = rgenop.newgraph(sigtoken, "div") - builder.start_writing() - gv_result = builder.genop2("int_div", gv_x, gv_y) - builder.finish_and_return(sigtoken, gv_result) - builder.end() - return gv_div def make_mul_imm(rgenop, num): sigtoken = rgenop.sigToken(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) @@ -179,9 +161,8 @@ # res = fnptr(2) # assert res == int("123456789",16)*2 - def test_imul(self): - rgenop = self.RGenOp() - mul_function = make_mul(rgenop) + def test_imul(self): + mul_function = make_two_op_instr(self.RGenOp(), "int_mul") fnptr = self.cast(mul_function,2) res = fnptr(1200,300) assert res == 360000 @@ -200,17 +181,35 @@ res = fnptr(12345,-9876) assert res == -121919220 - #BUG ignores rdx + #FIXME: ignores rdx and signs def test_idiv(self): - rgenop = self.RGenOp() - div_function = make_div(rgenop) + div_function = make_two_op_instr(self.RGenOp(), "int_div") fnptr = self.cast(div_function,2) + res = fnptr(100,3) + assert res == 33 # integer div res = fnptr(100,2) assert res == 50 res = fnptr(168,4) assert res == 42 res = fnptr(72057594037927935,5) assert res == 14411518807585587 + res = fnptr(-50,-5) + assert res == 10 + + #FIXME: ignores rdx and signs + def test_mod(self): + mod_function = make_two_op_instr(self.RGenOp(), "int_mod") + fnptr = self.cast(mod_function,2) + res = fnptr(100,3) + assert res == 1 + res = fnptr(4321,3) + assert res == 1 + res = fnptr(12345,7) + assert res == 4 + res = fnptr(-42,2) + assert res == 0 + res = fnptr(-12345,2) + assert res == 1 def test_greater(self): rgenop = self.RGenOp() From antocuni at codespeak.net Mon Sep 29 10:59:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 29 Sep 2008 10:59:26 +0200 (CEST) Subject: [pypy-svn] r58464 - in pypy/branch/oo-jit/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli Message-ID: <20080929085926.75854169EFA@codespeak.net> Author: antocuni Date: Mon Sep 29 10:59:24 2008 New Revision: 58464 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/translator/cli/opcodes.py Log: put more meaningful skip messages for some tests; some other tests pass out of the box Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Mon Sep 29 10:59:24 2008 @@ -317,7 +317,7 @@ return '_'.join(parts) def is_comparison(opname): - if opname in ('ooisnull', 'oononnull'): + if opname in ('ooisnull', 'oononnull', 'oois'): return True suffixes = '_lt _le _eq _ne _gt _ge'.split() for suffix in suffixes: Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 29 10:59:24 2008 @@ -424,6 +424,7 @@ return builder, graph.gv_entrypoint, graph.inputargs_gv[:] def replay(self, label): + print 'Replay!' raise NotImplementedError Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 29 10:59:24 2008 @@ -97,16 +97,15 @@ def test_indirect_red_call_with_exc(self): py.test.skip("replay: NotImplementedError") - def test_indirect_gray_call(self): - py.test.skip('mono 1.2 crashes, try again with a newer version') - def test_red_int_add_ovf(self): py.test.skip("TODO: exceptions") - test_learn_nonzeroness = skip - test_freeze_booleffects_correctly = skip - test_ptrequality = skip - test_void_args = skip + def test_learn_nonzeroness(self): + py.test.skip("replay: NotImplementedError") + + def test_freeze_booleffects_correctly(self): + py.test.skip("replay: NotImplementedError") + test_red_isinstance = skip test_red_isinstance_degenerated = skip test_simple_array = skip Modified: pypy/branch/oo-jit/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/opcodes.py Mon Sep 29 10:59:24 2008 @@ -106,7 +106,6 @@ 'ullong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], 'ullong_invert': 'not', - 'oois': 'ceq', 'ooisnull': [PushAllArgs, 'nop', 'nop', 'nop', 'ldnull', 'ceq'], 'oononnull': [PushAllArgs, 'nop', 'nop', 'nop', 'nop', 'ldnull', 'ceq']+Not, @@ -251,6 +250,8 @@ 'ullong_ge': _not('clt.un'), 'ullong_lshift': [PushAllArgs, 'conv.u4', 'shl'], 'ullong_rshift': [PushAllArgs, 'conv.i4', 'shr'], + + 'oois': 'ceq', } opcodes = misc_ops.copy() From arigo at codespeak.net Mon Sep 29 11:44:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 29 Sep 2008 11:44:02 +0200 (CEST) Subject: [pypy-svn] r58465 - in pypy/dist/pypy/rlib: . rstruct test Message-ID: <20080929094402.64C5B169FD5@codespeak.net> Author: arigo Date: Mon Sep 29 11:43:59 2008 New Revision: 58465 Added: pypy/dist/pypy/rlib/rstruct/__init__.py pypy/dist/pypy/rlib/rstruct/runpack.py - copied unchanged from r58464, pypy/dist/pypy/rlib/rstruct/__init__.py Modified: pypy/dist/pypy/rlib/rzipfile.py pypy/dist/pypy/rlib/test/test_rstruct.py Log: Don't hide unexpected pieces of code in __init__ files. Added: pypy/dist/pypy/rlib/rstruct/__init__.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rstruct/__init__.py Mon Sep 29 11:43:59 2008 @@ -0,0 +1 @@ +# Empty Modified: pypy/dist/pypy/rlib/rzipfile.py ============================================================================== --- pypy/dist/pypy/rlib/rzipfile.py (original) +++ pypy/dist/pypy/rlib/rzipfile.py Mon Sep 29 11:43:59 2008 @@ -1,7 +1,7 @@ from zipfile import ZIP_STORED, ZIP_DEFLATED from pypy.rlib.streamio import open_file_as_stream -from pypy.rlib.rstruct import runpack +from pypy.rlib.rstruct.runpack import runpack import os from pypy.rlib import rzlib from pypy.rlib.rarithmetic import r_uint, intmask Modified: pypy/dist/pypy/rlib/test/test_rstruct.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rstruct.py (original) +++ pypy/dist/pypy/rlib/test/test_rstruct.py Mon Sep 29 11:43:59 2008 @@ -1,6 +1,6 @@ from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rlib.rstruct import runpack +from pypy.rlib.rstruct.runpack import runpack import struct class BaseTestRStruct(BaseRtypingTest): From antocuni at codespeak.net Mon Sep 29 12:00:23 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 29 Sep 2008 12:00:23 +0200 (CEST) Subject: [pypy-svn] r58466 - in pypy/branch/oo-jit/pypy: jit/codegen jit/codegen/cli jit/codegen/cli/test jit/rainbow translator/cli Message-ID: <20080929100023.CFB62169F5E@codespeak.net> Author: antocuni Date: Mon Sep 29 12:00:22 2008 New Revision: 58466 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py pypy/branch/oo-jit/pypy/jit/codegen/model.py pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py pypy/branch/oo-jit/pypy/translator/cli/database.py Log: implement genop_instanceof; two tests pass Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/operation.py Mon Sep 29 12:00:22 2008 @@ -260,6 +260,23 @@ if self.restype() is not None: self.storeResult() +class InstanceOf(Operation): + + def __init__(self, meth, gv_obj, alloctoken): + self.meth = meth + self.gv_obj = gv_obj + self.clitype = alloctoken.getCliType() + + def restype(self): + return typeof(System.Boolean) + + def emit(self): + self.gv_obj.load(self.meth) + self.meth.il.Emit(OpCodes.Isinst, self.clitype) + self.meth.il.Emit(OpCodes.Ldnull) + self.meth.il.Emit(OpCodes.Cgt_Un) + self.storeResult() + def mark(il, s): il.Emit(OpCodes.Ldstr, s) il.Emit(OpCodes.Pop) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 29 12:00:22 2008 @@ -732,6 +732,11 @@ self.appendop(op) return op.gv_res() + def genop_instanceof(self, gv_obj, alloctoken): + op = ops.InstanceOf(self.meth, gv_obj, alloctoken) + self.appendop(op) + return op.gv_res() + def enter_next_block(self, args_gv): seen = {} for i in range(len(args_gv)): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 29 12:00:22 2008 @@ -106,8 +106,6 @@ def test_freeze_booleffects_correctly(self): py.test.skip("replay: NotImplementedError") - test_red_isinstance = skip - test_red_isinstance_degenerated = skip test_simple_array = skip test_arraysize = skip test_setarrayitem = skip Modified: pypy/branch/oo-jit/pypy/jit/codegen/model.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/model.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/model.py Mon Sep 29 12:00:22 2008 @@ -98,7 +98,8 @@ ## def genop_oosend(self, methtoken, gv_self, args_gv): ## def genop_oononnull(self, gv_obj): ## def genop_ooisnull(self, gv_obj): - +## def genop_instanceof(self, gv_obj, alloctoken): + ## def genop_oogetfield(self, fieldtoken, gv_obj): ## def genop_oosetfield(self, fieldtoken, gv_obj, gv_value): Modified: pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py Mon Sep 29 12:00:22 2008 @@ -1105,6 +1105,7 @@ @arguments("red", "structtypedesc", returns="red") def opimpl_red_instanceof(self, objbox, typedesc): + assert isinstance(objbox, rvalue.AbstractPtrRedBox) if objbox.content is None: return rtimeshift.geninstanceof(self.jitstate, objbox, typedesc) Modified: pypy/branch/oo-jit/pypy/translator/cli/database.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/database.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/database.py Mon Sep 29 12:00:22 2008 @@ -130,7 +130,7 @@ return name def class_or_record_name(self, TYPE): - if isinstance(TYPE, ootype.Instance): + if TYPE is not ootype.ROOT and isinstance(TYPE, ootype.Instance): return self.class_name(TYPE) elif isinstance(TYPE, ootype.Record): return self.get_record_name(TYPE) From antocuni at codespeak.net Mon Sep 29 12:05:01 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 29 Sep 2008 12:05:01 +0200 (CEST) Subject: [pypy-svn] r58467 - pypy/branch/oo-jit/pypy/jit/codegen/cli/test Message-ID: <20080929100501.DFC0A169F6C@codespeak.net> Author: antocuni Date: Mon Sep 29 12:05:01 2008 New Revision: 58467 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: I love tests that pass out of the box :-) Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 29 12:05:01 2008 @@ -106,10 +106,6 @@ def test_freeze_booleffects_correctly(self): py.test.skip("replay: NotImplementedError") - test_simple_array = skip - test_arraysize = skip - test_setarrayitem = skip - test_red_array = skip test_degenerated_before_return = skip test_degenerated_before_return_2 = skip test_degenerated_at_return = skip From arigo at codespeak.net Mon Sep 29 12:22:26 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 29 Sep 2008 12:22:26 +0200 (CEST) Subject: [pypy-svn] r58468 - pypy/dist/pypy/doc Message-ID: <20080929102226.0C077169F96@codespeak.net> Author: arigo Date: Mon Sep 29 12:22:24 2008 New Revision: 58468 Modified: pypy/dist/pypy/doc/maemo.txt Log: I wasted enough time for today on this. Modified: pypy/dist/pypy/doc/maemo.txt ============================================================================== --- pypy/dist/pypy/doc/maemo.txt (original) +++ pypy/dist/pypy/doc/maemo.txt Mon Sep 29 12:22:24 2008 @@ -20,6 +20,18 @@ I had to manually edit /scratchbox/devkits/debian-etch/etc/environment to add ARCH=armel, otherwise things did not work +XXX things don't work for me (arigo); I cannot get a Python +more recent than 2.3. I edited /etc/apt/sources.list to contain:: + + deb ftp://ftp.fi.debian.org/debian/ lenny main contrib non-free + deb-src ftp://ftp.fi.debian.org/debian/ lenny main contrib non-free + +Then apt-get update gives a minor error (possibly just a warning), +but sb-install-base-packages crashes with no more explanation than:: + + E: Internal Error, Could not perform immediate configuration (2) on libc6 + + Translating pypy ---------------- From fijal at codespeak.net Mon Sep 29 13:22:52 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 29 Sep 2008 13:22:52 +0200 (CEST) Subject: [pypy-svn] r58471 - pypy/branch/gc-experiments/pypy/config Message-ID: <20080929112252.EFF5E169EEA@codespeak.net> Author: fijal Date: Mon Sep 29 13:22:50 2008 New Revision: 58471 Modified: pypy/branch/gc-experiments/pypy/config/translationoption.py Log: Add mark&compact translation option Modified: pypy/branch/gc-experiments/pypy/config/translationoption.py ============================================================================== --- pypy/branch/gc-experiments/pypy/config/translationoption.py (original) +++ pypy/branch/gc-experiments/pypy/config/translationoption.py Mon Sep 29 13:22:50 2008 @@ -45,7 +45,7 @@ # gc ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", - "generation", "hybrid", "none"], + "generation", "hybrid", "markcompact", "none"], "ref", requires={ "ref": [("translation.rweakref", False), # XXX ("translation.gctransformer", "ref")], @@ -57,6 +57,7 @@ "generation": [("translation.gctransformer", "framework")], "hybrid": [("translation.gctransformer", "framework")], "boehm": [("translation.gctransformer", "boehm")], + "markcompact": [("translation.gctransformer", "framework")], }, cmdline="--gc"), ChoiceOption("gctransformer", "GC transformer that is used - internal", From fijal at codespeak.net Mon Sep 29 13:24:06 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 29 Sep 2008 13:24:06 +0200 (CEST) Subject: [pypy-svn] r58472 - pypy/branch/gc-experiments/pypy/rpython/lltypesystem Message-ID: <20080929112406.75661169EDD@codespeak.net> Author: fijal Date: Mon Sep 29 13:24:05 2008 New Revision: 58472 Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py Log: make this a bit more annotation-friendly Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py Mon Sep 29 13:24:05 2008 @@ -736,6 +736,11 @@ assert lltype.typeOf(dest) == Address size.raw_memcopy(source, dest) +def raw_memmove(source, dest, size): + # for now let's assume that raw_memmove is the same as raw_memcopy, + # when run on top of fake addresses + raw_memcopy(source, dest, size) + def cast_any_ptr(EXPECTED_TYPE, ptr): # this is a generalization of the various cast_xxx_ptr() functions. PTRTYPE = lltype.typeOf(ptr) From fijal at codespeak.net Mon Sep 29 13:26:15 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 29 Sep 2008 13:26:15 +0200 (CEST) Subject: [pypy-svn] r58474 - in pypy/branch/gc-experiments/pypy/rpython/memory: gc test Message-ID: <20080929112615.5A94C169F41@codespeak.net> Author: fijal Date: Mon Sep 29 13:26:14 2008 New Revision: 58474 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py pypy/branch/gc-experiments/pypy/rpython/memory/test/test_transformed_gc.py Log: Separate semispace and markcompact, this was insane markcompact works more or less Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Mon Sep 29 13:26:14 2008 @@ -488,6 +488,7 @@ "semispace": "semispace.SemiSpaceGC", "generation": "generation.GenerationGC", "hybrid": "hybrid.HybridGC", + "markcompact" : "markcompact.MarkCompactGC", } try: modulename, classname = classes[config.translation.gc].split('.') Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Mon Sep 29 13:26:14 2008 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.gc.base import MovingGCBase, \ - TYPEID_MASK, GCFLAG_FINALIZATION_ORDERING + TYPEID_MASK, GCFLAG_FINALIZATION_ORDERING, GCFLAG_EXTERNAL from pypy.rlib.debug import ll_assert from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -33,10 +33,14 @@ # using memmove to another view of the same space, hence compacting # everything +# before compacting, we update forward references to pointers + class MarkCompactGC(MovingGCBase): HDR = lltype.Struct('header', ('tid', lltype.Signed), ('forward_ptr', llmemory.Address)) + TRANSLATION_PARAMS = {'space_size': 16*1024*1024} # XXX adjust + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=16*(1024**2)): # space_size should be maximal available virtual memory. # this way we'll never need to copy anything nor implement @@ -49,11 +53,14 @@ self.space = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.space), "couldn't allocate arena") self.spaceptr = self.space + self.toaddr = self.space MovingGCBase.setup(self) + self.finalizers_to_run = self.AddressDeque() def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.tid = typeid | flags + hdr.forward_ptr = NULL def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): @@ -102,24 +109,63 @@ def collect(self): self.debug_check_consistency() - self.mark() - if self.run_finalizers.non_empty(): - self.update_run_finalizers() + toaddr = llarena.arena_new_view(self.space) + self.mark_roots_recursively() + self.debug_check_consistency() + #if self.run_finalizers.non_empty(): + # self.update_run_finalizers() if self.objects_with_finalizers.non_empty(): - self.deal_with_objects_with_finalizers(None) + self.mark_objects_with_finalizers() + if self.finalizers_to_run.non_empty(): + self.execute_finalizers() + self.debug_check_consistency() + finaladdr = self.update_forward_pointers(toaddr) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() - self.debug_check_consistency() - toaddr = llarena.arena_new_view(self.space) - self.debug_check_consistency() - self.compact(toaddr) + self.update_objects_with_id() + self.compact() self.space = toaddr + self.spaceptr = finaladdr self.debug_check_consistency() def get_type_id(self, addr): return self.header(addr).tid & TYPEID_MASK - def compact(self, toaddr): + def mark_roots_recursively(self): + self.root_walker.walk_roots( + MarkCompactGC._mark_root_recursively, # stack roots + MarkCompactGC._mark_root_recursively, # static in prebuilt non-gc structures + MarkCompactGC._mark_root_recursively) # static in prebuilt gc objects + + def _mark_root_recursively(self, root): + self.trace_and_mark(root.address[0]) + + def mark(self, obj): + self.header(obj).tid |= GCFLAG_MARKBIT + + def marked(self, obj): + return self.header(obj).tid & GCFLAG_MARKBIT + + def trace_and_mark(self, obj): + if self.marked(obj): + return + self.mark(obj) + to_see = self.AddressStack() + to_see.append(obj) + while to_see.non_empty(): + obj = to_see.pop() + self.trace(obj, self._mark_obj, to_see) + to_see.delete() + + def _mark_obj(self, pointer, to_see): + obj = pointer.address[0] + if obj != NULL: + if self.marked(obj): + return + self.mark(obj) + to_see.append(obj) + + def update_forward_pointers(self, toaddr): fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header while fromaddr < self.spaceptr: @@ -127,108 +173,198 @@ obj = fromaddr + size_gc_header objsize = self.get_size(obj) totalsize = size_gc_header + objsize - if not hdr.tid & GCFLAG_MARKBIT: - # this object dies, clear arena - llarena.arena_reset(fromaddr, totalsize, True) + if not self.marked(obj): + pass else: - if hdr.tid & GCFLAG_FORWARDED: - # this object needs to be copied somewhere - # first approach - copy object if possible, otherwise - # copy it somewhere else and keep track of that - if toaddr + totalsize > fromaddr: - # this is the worst possible scenario: object does - # not fit inside space to copy - xxx - else: - # object fits there: copy - hdr.tid &= ~(GCFLAG_MARKBIT|GCFLAG_FORWARDED) - #llop.debug_print(lltype.Void, fromaddr, "copied to", toaddr, - # "tid", self.header(obj).tid, - # "size", totalsize) - llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize) - llarena.arena_reset(fromaddr, totalsize, True) - else: - hdr.tid &= ~(GCFLAG_MARKBIT|GCFLAG_FORWARDED) - # XXX this is here only to make llarena happier, makes no - # sense whatsoever, need to disable it when translated - llarena.arena_reserve(toaddr, totalsize) - llmemory.raw_memcopy(obj - size_gc_header, toaddr, totalsize) - toaddr += size_gc_header + objsize - fromaddr += size_gc_header + objsize - self.spaceptr = toaddr + llarena.arena_reserve(toaddr, totalsize) + self.set_forwarding_address(obj, toaddr) + toaddr += totalsize + fromaddr += totalsize - def update_forward_refs(self): + # now update references self.root_walker.walk_roots( - MarkCompactGC._trace_copy, # stack roots - MarkCompactGC._trace_copy, # static in prebuilt non-gc structures - MarkCompactGC._trace_copy) # static in prebuilt gc objects - ptr = self.space - size_gc_header = self.gcheaderbuilder.size_gc_header - while ptr < self.spaceptr: - obj = ptr + size_gc_header + MarkCompactGC._update_root, # stack roots + MarkCompactGC._update_root, # static in prebuilt non-gc structures + MarkCompactGC._update_root) # static in prebuilt gc objects + fromaddr = self.space + while fromaddr < self.spaceptr: + hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) + obj = fromaddr + size_gc_header objsize = self.get_size(obj) totalsize = size_gc_header + objsize - self.trace(obj, self._trace_copy, None) - ptr += totalsize + if not self.marked(obj): + pass + else: + self.trace(obj, self._update_ref, None) + fromaddr += totalsize + return toaddr + + def _update_root(self, pointer): + if pointer.address[0] != NULL: + pointer.address[0] = self.get_forwarding_address(pointer.address[0]) + + def _update_ref(self, pointer, ignore): + if pointer.address[0] != NULL: + pointer.address[0] = self.get_forwarding_address(pointer.address[0]) + + def is_forwarded(self, addr): + return self.header(addr).forward_ptr != NULL + + def _is_external(self, obj): + # XXX might change + return self.header(obj).tid & GCFLAG_EXTERNAL - def _trace_copy(self, pointer, ignored=None): - addr = pointer.address[0] - size_gc_header = self.gcheaderbuilder.size_gc_header - if addr != NULL: - hdr = llmemory.cast_adr_to_ptr(addr - size_gc_header, - lltype.Ptr(self.HDR)) - pointer.address[0] = hdr.forward_ptr + def get_forwarding_address(self, obj): + if self._is_external(obj): + return obj + else: + return self.header(obj).forward_ptr + self.size_gc_header() - def mark(self): - self.root_walker.walk_roots( - MarkCompactGC._mark_object, # stack roots - MarkCompactGC._mark_object, # static in prebuilt non-gc structures - MarkCompactGC._mark_object) # static in prebuilt gc objects + def set_forwarding_address(self, obj, newaddr): + self.header(obj).forward_ptr = newaddr - def _mark_object(self, pointer, ignored=None): - obj = pointer.address[0] - if obj != NULL: - self.header(obj).tid |= GCFLAG_MARKBIT - self.trace(obj, self._mark_object, None) + def surviving(self, obj): + return self.header(obj).forward_ptr != NULL - def deal_with_objects_with_finalizers(self, ignored): - new_with_finalizer = self.AddressDeque() - while self.objects_with_finalizers.non_empty(): - obj = self.objects_with_finalizers.popleft() - if self.surviving(obj): - new_with_finalizer.append(obj) - break - finalizers_to_run.append(obj) - xxxx + def compact(self): + fromaddr = self.space + size_gc_header = self.gcheaderbuilder.size_gc_header + while fromaddr < self.spaceptr: + hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) + obj = fromaddr + size_gc_header + objsize = self.get_size(obj) + totalsize = size_gc_header + objsize + if not self.surviving(obj): + # this object dies, clear arena + llarena.arena_reset(fromaddr, totalsize, True) + else: + ll_assert(self.is_forwarded(obj), "not forwarded, surviving obj") + forward_ptr = hdr.forward_ptr + hdr.forward_ptr = NULL + hdr.tid &= ~(GCFLAG_MARKBIT&GCFLAG_FINALIZATION_ORDERING) + #llop.debug_print(lltype.Void, fromaddr, "copied to", forward_ptr, + # "\ntid", self.header(obj).tid, + # "\nsize", totalsize) + llmemory.raw_memmove(fromaddr, forward_ptr, totalsize) + llarena.arena_reset(fromaddr, totalsize, False) + assert llmemory.cast_adr_to_ptr(forward_ptr, lltype.Ptr(self.HDR)).tid + fromaddr += totalsize def debug_check_object(self, obj): - # XXX write it down + # not sure what to check here pass - def surviving(self, obj): - hdr = self.header(obj) - return hdr.tid & GCFLAG_MARKBIT + + def id(self, ptr): + obj = llmemory.cast_ptr_to_adr(ptr) + if self.header(obj).tid & GCFLAG_EXTERNAL: + result = self._compute_id_for_external(obj) + else: + result = self._compute_id(obj) + return llmemory.cast_adr_to_int(result) + + def _next_id(self): + # return an id not currently in use (as an address instead of an int) + if self.id_free_list.non_empty(): + result = self.id_free_list.pop() # reuse a dead id + else: + # make up a fresh id number + result = llmemory.cast_int_to_adr(self.next_free_id) + self.next_free_id += 2 # only odd numbers, to make lltype + # and llmemory happy and to avoid + # clashes with real addresses + return result + + def _compute_id(self, obj): + # look if the object is listed in objects_with_id + result = self.objects_with_id.get(obj) + if not result: + result = self._next_id() + self.objects_with_id.setitem(obj, result) + return result + + def _compute_id_for_external(self, obj): + # For prebuilt objects, we can simply return their address. + # This method is overriden by the HybridGC. + return obj + + def update_objects_with_id(self): + old = self.objects_with_id + new_objects_with_id = self.AddressDict(old.length()) + old.foreach(self._update_object_id_FAST, new_objects_with_id) + old.delete() + self.objects_with_id = new_objects_with_id + + def _update_object_id(self, obj, id, new_objects_with_id): + # safe version (used by subclasses) + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.setitem(newobj, id) + else: + self.id_free_list.append(id) + + def _update_object_id_FAST(self, obj, id, new_objects_with_id): + # unsafe version, assumes that the new_objects_with_id is large enough + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.insertclean(newobj, id) + else: + self.id_free_list.append(id) def _finalization_state(self, obj): - hdr = self.header(obj) if self.surviving(obj): + hdr = self.header(obj) if hdr.tid & GCFLAG_FINALIZATION_ORDERING: return 2 else: return 3 else: + hdr = self.header(obj) if hdr.tid & GCFLAG_FINALIZATION_ORDERING: return 1 else: return 0 - def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): - # it's enough to leave mark bit here - hdr = self.header(obj) - if hdr.tid & GCFLAG_MARKBIT: - return # cycle - self.header(obj).tid |= GCFLAG_MARKBIT - self.trace(obj, self._mark_object, None) - - def get_forwarding_address(self, obj): - return obj + def mark_objects_with_finalizers(self): + new_with_finalizers = self.AddressDeque() + finalizers_to_run = self.finalizers_to_run + while self.objects_with_finalizers.non_empty(): + x = self.objects_with_finalizers.popleft() + if self.marked(x): + new_with_finalizers.append(x) + else: + finalizers_to_run.append(x) + self.trace_and_mark(x) + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizers + + def execute_finalizers(self): + while self.finalizers_to_run.non_empty(): + obj = self.finalizers_to_run.popleft() + finalizer = self.getfinalizer(self.get_type_id(obj)) + finalizer(obj) + self.finalizers_to_run.delete() + self.finalizers_to_run = self.AddressDeque() + + def invalidate_weakrefs(self): + # walk over list of objects that contain weakrefs + # if the object it references survives then update the weakref + # otherwise invalidate the weakref + new_with_weakref = self.AddressStack() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.surviving(obj): + continue # weakref itself dies + newobj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.get_type_id(obj)) + pointing_to = (obj + offset).address[0] + # XXX I think that pointing_to cannot be NULL here + if pointing_to: + if self.surviving(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + new_with_weakref.append(newobj) + else: + (obj + offset).address[0] = NULL + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/semispace.py Mon Sep 29 13:26:14 2008 @@ -371,5 +371,229 @@ stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj + def deal_with_objects_with_finalizers(self, scan): + # walk over list of objects with finalizers + # if it is not copied, add it to the list of to-be-called finalizers + # and copy it, to me make the finalizer runnable + # We try to run the finalizers in a "reasonable" order, like + # CPython does. The details of this algorithm are in + # pypy/doc/discussion/finalizer-order.txt. + new_with_finalizer = self.AddressDeque() + marked = self.AddressDeque() + pending = self.AddressStack() + self.tmpstack = self.AddressStack() + while self.objects_with_finalizers.non_empty(): + x = self.objects_with_finalizers.popleft() + ll_assert(self._finalization_state(x) != 1, + "bad finalization state 1") + if self.surviving(x): + new_with_finalizer.append(self.get_forwarding_address(x)) + continue + marked.append(x) + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 0: + self._bump_finalization_state_from_0_to_1(y) + self.trace(y, self._append_if_nonnull, pending) + elif state == 2: + self._recursively_bump_finalization_state_from_2_to_3(y) + scan = self._recursively_bump_finalization_state_from_1_to_2( + x, scan) + + while marked.non_empty(): + x = marked.popleft() + state = self._finalization_state(x) + ll_assert(state >= 2, "unexpected finalization state < 2") + newx = self.get_forwarding_address(x) + if state == 2: + self.run_finalizers.append(newx) + # we must also fix the state from 2 to 3 here, otherwise + # we leave the GCFLAG_FINALIZATION_ORDERING bit behind + # which will confuse the next collection + self._recursively_bump_finalization_state_from_2_to_3(x) + else: + new_with_finalizer.append(newx) + + self.tmpstack.delete() + pending.delete() + marked.delete() + self.objects_with_finalizers.delete() + self.objects_with_finalizers = new_with_finalizer + return scan + + def _append_if_nonnull(pointer, stack): + if pointer.address[0] != NULL: + stack.append(pointer.address[0]) + _append_if_nonnull = staticmethod(_append_if_nonnull) + + def _finalization_state(self, obj): + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 2 + else: + return 3 + else: + hdr = self.header(obj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 1 + else: + return 0 + + def _bump_finalization_state_from_0_to_1(self, obj): + ll_assert(self._finalization_state(obj) == 0, + "unexpected finalization state != 0") + hdr = self.header(obj) + hdr.tid |= GCFLAG_FINALIZATION_ORDERING + + def _recursively_bump_finalization_state_from_2_to_3(self, obj): + ll_assert(self._finalization_state(obj) == 2, + "unexpected finalization state != 2") + newobj = self.get_forwarding_address(obj) + pending = self.tmpstack + ll_assert(not pending.non_empty(), "tmpstack not empty") + pending.append(newobj) + while pending.non_empty(): + y = pending.pop() + hdr = self.header(y) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + self.trace(y, self._append_if_nonnull, pending) + + def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): + # recursively convert objects from state 1 to state 2. + # Note that copy() copies all bits, including the + # GCFLAG_FINALIZATION_ORDERING. The mapping between + # state numbers and the presence of this bit was designed + # for the following to work :-) + self.copy(obj) + return self.scan_copied(scan) + + def invalidate_weakrefs(self): + # walk over list of objects that contain weakrefs + # if the object it references survives then update the weakref + # otherwise invalidate the weakref + new_with_weakref = self.AddressStack() + while self.objects_with_weakrefs.non_empty(): + obj = self.objects_with_weakrefs.pop() + if not self.surviving(obj): + continue # weakref itself dies + obj = self.get_forwarding_address(obj) + offset = self.weakpointer_offset(self.get_type_id(obj)) + pointing_to = (obj + offset).address[0] + # XXX I think that pointing_to cannot be NULL here + if pointing_to: + if self.surviving(pointing_to): + (obj + offset).address[0] = self.get_forwarding_address( + pointing_to) + new_with_weakref.append(obj) + else: + (obj + offset).address[0] = NULL + self.objects_with_weakrefs.delete() + self.objects_with_weakrefs = new_with_weakref + + def update_run_finalizers(self): + # we are in an inner collection, caused by a finalizer + # the run_finalizers objects need to be copied + new_run_finalizer = self.AddressDeque() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.popleft() + new_run_finalizer.append(self.copy(obj)) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizer + + def execute_finalizers(self): + self.finalizer_lock_count += 1 + try: + while self.run_finalizers.non_empty(): + #print "finalizer" + if self.finalizer_lock_count > 1: + # the outer invocation of execute_finalizers() will do it + break + obj = self.run_finalizers.popleft() + finalizer = self.getfinalizer(self.get_type_id(obj)) + finalizer(obj) + finally: + self.finalizer_lock_count -= 1 + + def id(self, ptr): + obj = llmemory.cast_ptr_to_adr(ptr) + if self.header(obj).tid & GCFLAG_EXTERNAL: + result = self._compute_id_for_external(obj) + else: + result = self._compute_id(obj) + return llmemory.cast_adr_to_int(result) + + def _next_id(self): + # return an id not currently in use (as an address instead of an int) + if self.id_free_list.non_empty(): + result = self.id_free_list.pop() # reuse a dead id + else: + # make up a fresh id number + result = llmemory.cast_int_to_adr(self.next_free_id) + self.next_free_id += 2 # only odd numbers, to make lltype + # and llmemory happy and to avoid + # clashes with real addresses + return result + + def _compute_id(self, obj): + # look if the object is listed in objects_with_id + result = self.objects_with_id.get(obj) + if not result: + result = self._next_id() + self.objects_with_id.setitem(obj, result) + return result + + def _compute_id_for_external(self, obj): + # For prebuilt objects, we can simply return their address. + # This method is overriden by the HybridGC. + return obj + + def update_objects_with_id(self): + old = self.objects_with_id + new_objects_with_id = self.AddressDict(old.length()) + old.foreach(self._update_object_id_FAST, new_objects_with_id) + old.delete() + self.objects_with_id = new_objects_with_id + + def _update_object_id(self, obj, id, new_objects_with_id): + # safe version (used by subclasses) + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.setitem(newobj, id) + else: + self.id_free_list.append(id) + + def _update_object_id_FAST(self, obj, id, new_objects_with_id): + # unsafe version, assumes that the new_objects_with_id is large enough + if self.surviving(obj): + newobj = self.get_forwarding_address(obj) + new_objects_with_id.insertclean(newobj, id) + else: + self.id_free_list.append(id) + + def debug_check_object(self, obj): + """Check the invariants about 'obj' that should be true + between collections.""" + tid = self.header(obj).tid + if tid & GCFLAG_EXTERNAL: + ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded") + ll_assert(not (self.tospace <= obj < self.free), + "external flag but object inside the semispaces") + else: + ll_assert(not (tid & GCFLAG_FORWARDED), "bug: !external+forwarded") + ll_assert(self.tospace <= obj < self.free, + "!external flag but object outside the semispaces") + ll_assert(not (tid & GCFLAG_FINALIZATION_ORDERING), + "unexpected GCFLAG_FINALIZATION_ORDERING") + + def debug_check_can_copy(self, obj): + ll_assert(not (self.tospace <= obj < self.free), + "copy() on already-copied object") + STATISTICS_NUMBERS = 0 + Modified: pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/test/test_gc.py Mon Sep 29 13:26:14 2008 @@ -458,6 +458,19 @@ class TestMarkCompactGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + def test_finalizer_order(self): + py.test.skip("Not implemented yet") + + def test_finalizer_calls_malloc(self): + py.test.skip("Not implemented yet") + + def test_finalizer_calls_malloc_during_minor_collect(self): + py.test.skip("Not implemented yet") + + def test_weakref_to_object_with_finalizer(self): + py.test.skip("Not implemented yet") + + class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass GC_CANNOT_MALLOC_NONMOVABLE = False Modified: pypy/branch/gc-experiments/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/test/test_transformed_gc.py Mon Sep 29 13:26:14 2008 @@ -798,6 +798,14 @@ GC_PARAMS = {'space_size': 2048} root_stack_depth = 200 +class TestMarkCompactGC(GenericMovingGCTests): + gcname = 'markcompact' + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + root_stack_depth = 200 + class TestGenerationGC(GenericMovingGCTests): gcname = "generation" From arigo at codespeak.net Mon Sep 29 13:41:49 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 29 Sep 2008 13:41:49 +0200 (CEST) Subject: [pypy-svn] r58475 - pypy/dist/pypy/doc Message-ID: <20080929114149.812E7169F98@codespeak.net> Author: arigo Date: Mon Sep 29 13:41:49 2008 New Revision: 58475 Modified: pypy/dist/pypy/doc/maemo.txt Log: Worked in the end; update with fijal's suggestions that unstuck me. Modified: pypy/dist/pypy/doc/maemo.txt ============================================================================== --- pypy/dist/pypy/doc/maemo.txt (original) +++ pypy/dist/pypy/doc/maemo.txt Mon Sep 29 13:41:49 2008 @@ -18,18 +18,18 @@ (and not a rootstrap). I had to manually edit /scratchbox/devkits/debian-etch/etc/environment -to add ARCH=armel, otherwise things did not work - -XXX things don't work for me (arigo); I cannot get a Python -more recent than 2.3. I edited /etc/apt/sources.list to contain:: +to add ARCH=armel, otherwise apt-get did not work. I also edited +/etc/apt/sources.list to contain:: deb ftp://ftp.fi.debian.org/debian/ lenny main contrib non-free deb-src ftp://ftp.fi.debian.org/debian/ lenny main contrib non-free -Then apt-get update gives a minor error (possibly just a warning), -but sb-install-base-packages crashes with no more explanation than:: - - E: Internal Error, Could not perform immediate configuration (2) on libc6 +Then run ``apt-get update``, get an error, ignore it, and run +``fakeroot apt-get install python-dev``. Doesn't work out of +the box for me; I have actually to install a few pieces at a +time, starting from libc6, sometimes with and sometimes +without the ``fakeroot``. It's quite a mess but in the end I +get a ``python2.5``. Translating pypy From antocuni at codespeak.net Mon Sep 29 13:51:53 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 29 Sep 2008 13:51:53 +0200 (CEST) Subject: [pypy-svn] r58476 - in pypy/branch/oo-jit/pypy/jit/codegen/cli: . test Message-ID: <20080929115153.77E3B169FA9@codespeak.net> Author: antocuni Date: Mon Sep 29 13:51:52 2008 New Revision: 58476 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: more tests passing Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 29 13:51:52 2008 @@ -206,6 +206,8 @@ def revealconst(self, T): if T is ootype.Object: return ootype.NULL # XXX? + elif isinstance(T, ootype.OOType): + return ootype.null(T) # XXX return lltype.cast_primitive(T, self.value) def getCliType(self): Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 29 13:51:52 2008 @@ -106,8 +106,6 @@ def test_freeze_booleffects_correctly(self): py.test.skip("replay: NotImplementedError") - test_degenerated_before_return = skip - test_degenerated_before_return_2 = skip test_degenerated_at_return = skip test_degenerated_via_substructure = skip test_red_subclass = skip From antocuni at codespeak.net Mon Sep 29 14:13:08 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 29 Sep 2008 14:13:08 +0200 (CEST) Subject: [pypy-svn] r58479 - in pypy/branch/oo-jit/pypy/jit/codegen/cli: . test Message-ID: <20080929121308.7648D169EAA@codespeak.net> Author: antocuni Date: Mon Sep 29 14:13:08 2008 New Revision: 58479 Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Log: make kindToken returning the actual class of T instead of Object; more tests pass Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/rgenop.py Mon Sep 29 14:13:08 2008 @@ -398,7 +398,7 @@ elif T is ootype.Char: return cChar elif isinstance(T, ootype.OOType): - return cObject # XXX? + return ootype.runtimeClass(T) else: assert False Modified: pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/codegen/cli/test/test_gencli_interpreter.py Mon Sep 29 14:13:08 2008 @@ -106,6 +106,5 @@ def test_freeze_booleffects_correctly(self): py.test.skip("replay: NotImplementedError") - test_degenerated_at_return = skip - test_degenerated_via_substructure = skip - test_red_subclass = skip + def test_degenerated_at_return(self): + py.test.skip('test framework does not support getattr on returned instances') From antocuni at codespeak.net Tue Sep 30 10:50:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 30 Sep 2008 10:50:17 +0200 (CEST) Subject: [pypy-svn] r58492 - pypy/branch/oo-jit/pypy/translator/cli Message-ID: <20080930085017.6B2FE169E9A@codespeak.net> Author: antocuni Date: Tue Sep 30 10:50:15 2008 New Revision: 58492 Modified: pypy/branch/oo-jit/pypy/translator/cli/query.py Log: make sure that arrays whose items are valuetype are treated as valuetype as well Modified: pypy/branch/oo-jit/pypy/translator/cli/query.py ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/query.py (original) +++ pypy/branch/oo-jit/pypy/translator/cli/query.py Tue Sep 30 10:50:15 2008 @@ -112,6 +112,7 @@ desc.AssemblyQualifiedName = name # XXX desc.BaseType = 'System.Array' desc.ElementType = itemdesc.FullName + desc.IsValueType = itemdesc.IsValueType desc.IsArray = True desc.Methods = [ ('Get', ['ootype.Signed', ], itemdesc.FullName), From hpk at codespeak.net Tue Sep 30 12:00:37 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 30 Sep 2008 12:00:37 +0200 (CEST) Subject: [pypy-svn] r58499 - pypy/build/benchmem/benchmark Message-ID: <20080930100037.55707169FAE@codespeak.net> Author: hpk Date: Tue Sep 30 12:00:36 2008 New Revision: 58499 Modified: pypy/build/benchmem/benchmark/sizes.py Log: adding a "null" benchmark Modified: pypy/build/benchmem/benchmark/sizes.py ============================================================================== --- pypy/build/benchmem/benchmark/sizes.py (original) +++ pypy/build/benchmem/benchmark/sizes.py Tue Sep 30 12:00:36 2008 @@ -1,4 +1,7 @@ +def bench_0nothing(iter1): + pass + def bench_list_of_None(iter1): l = [None for i in range(iter1)] checkpoint(collect=True) From fijal at codespeak.net Tue Sep 30 13:59:01 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 30 Sep 2008 13:59:01 +0200 (CEST) Subject: [pypy-svn] r58501 - pypy/branch/gc-experiments/pypy/rpython/memory/gc Message-ID: <20080930115901.671BA16A01C@codespeak.net> Author: fijal Date: Tue Sep 30 13:58:58 2008 New Revision: 58501 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Log: This is code repetition now Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/base.py Tue Sep 30 13:58:58 2008 @@ -252,231 +252,6 @@ size = llarena.round_up_for_allocation(size) return size - - - def deal_with_objects_with_finalizers(self, scan): - # walk over list of objects with finalizers - # if it is not copied, add it to the list of to-be-called finalizers - # and copy it, to me make the finalizer runnable - # We try to run the finalizers in a "reasonable" order, like - # CPython does. The details of this algorithm are in - # pypy/doc/discussion/finalizer-order.txt. - new_with_finalizer = self.AddressDeque() - marked = self.AddressDeque() - pending = self.AddressStack() - self.tmpstack = self.AddressStack() - while self.objects_with_finalizers.non_empty(): - x = self.objects_with_finalizers.popleft() - ll_assert(self._finalization_state(x) != 1, - "bad finalization state 1") - if self.surviving(x): - new_with_finalizer.append(self.get_forwarding_address(x)) - continue - marked.append(x) - pending.append(x) - while pending.non_empty(): - y = pending.pop() - state = self._finalization_state(y) - if state == 0: - self._bump_finalization_state_from_0_to_1(y) - self.trace(y, self._append_if_nonnull, pending) - elif state == 2: - self._recursively_bump_finalization_state_from_2_to_3(y) - scan = self._recursively_bump_finalization_state_from_1_to_2( - x, scan) - - while marked.non_empty(): - x = marked.popleft() - state = self._finalization_state(x) - ll_assert(state >= 2, "unexpected finalization state < 2") - newx = self.get_forwarding_address(x) - if state == 2: - self.run_finalizers.append(newx) - # we must also fix the state from 2 to 3 here, otherwise - # we leave the GCFLAG_FINALIZATION_ORDERING bit behind - # which will confuse the next collection - self._recursively_bump_finalization_state_from_2_to_3(x) - else: - new_with_finalizer.append(newx) - - self.tmpstack.delete() - pending.delete() - marked.delete() - self.objects_with_finalizers.delete() - self.objects_with_finalizers = new_with_finalizer - return scan - - def _append_if_nonnull(pointer, stack): - if pointer.address[0] != NULL: - stack.append(pointer.address[0]) - _append_if_nonnull = staticmethod(_append_if_nonnull) - - def _finalization_state(self, obj): - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - hdr = self.header(newobj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: - return 2 - else: - return 3 - else: - hdr = self.header(obj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: - return 1 - else: - return 0 - - def _bump_finalization_state_from_0_to_1(self, obj): - ll_assert(self._finalization_state(obj) == 0, - "unexpected finalization state != 0") - hdr = self.header(obj) - hdr.tid |= GCFLAG_FINALIZATION_ORDERING - - def _recursively_bump_finalization_state_from_2_to_3(self, obj): - ll_assert(self._finalization_state(obj) == 2, - "unexpected finalization state != 2") - newobj = self.get_forwarding_address(obj) - pending = self.tmpstack - ll_assert(not pending.non_empty(), "tmpstack not empty") - pending.append(newobj) - while pending.non_empty(): - y = pending.pop() - hdr = self.header(y) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 - self.trace(y, self._append_if_nonnull, pending) - - def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): - # recursively convert objects from state 1 to state 2. - # Note that copy() copies all bits, including the - # GCFLAG_FINALIZATION_ORDERING. The mapping between - # state numbers and the presence of this bit was designed - # for the following to work :-) - self.copy(obj) - return self.scan_copied(scan) - - def invalidate_weakrefs(self): - # walk over list of objects that contain weakrefs - # if the object it references survives then update the weakref - # otherwise invalidate the weakref - new_with_weakref = self.AddressStack() - while self.objects_with_weakrefs.non_empty(): - obj = self.objects_with_weakrefs.pop() - if not self.surviving(obj): - continue # weakref itself dies - obj = self.get_forwarding_address(obj) - offset = self.weakpointer_offset(self.get_type_id(obj)) - pointing_to = (obj + offset).address[0] - # XXX I think that pointing_to cannot be NULL here - if pointing_to: - if self.surviving(pointing_to): - (obj + offset).address[0] = self.get_forwarding_address( - pointing_to) - new_with_weakref.append(obj) - else: - (obj + offset).address[0] = NULL - self.objects_with_weakrefs.delete() - self.objects_with_weakrefs = new_with_weakref - - def update_run_finalizers(self): - # we are in an inner collection, caused by a finalizer - # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressDeque() - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.popleft() - new_run_finalizer.append(self.copy(obj)) - self.run_finalizers.delete() - self.run_finalizers = new_run_finalizer - - def execute_finalizers(self): - self.finalizer_lock_count += 1 - try: - while self.run_finalizers.non_empty(): - #print "finalizer" - if self.finalizer_lock_count > 1: - # the outer invocation of execute_finalizers() will do it - break - obj = self.run_finalizers.popleft() - finalizer = self.getfinalizer(self.get_type_id(obj)) - finalizer(obj) - finally: - self.finalizer_lock_count -= 1 - - def id(self, ptr): - obj = llmemory.cast_ptr_to_adr(ptr) - if self.header(obj).tid & GCFLAG_EXTERNAL: - result = self._compute_id_for_external(obj) - else: - result = self._compute_id(obj) - return llmemory.cast_adr_to_int(result) - - def _next_id(self): - # return an id not currently in use (as an address instead of an int) - if self.id_free_list.non_empty(): - result = self.id_free_list.pop() # reuse a dead id - else: - # make up a fresh id number - result = llmemory.cast_int_to_adr(self.next_free_id) - self.next_free_id += 2 # only odd numbers, to make lltype - # and llmemory happy and to avoid - # clashes with real addresses - return result - - def _compute_id(self, obj): - # look if the object is listed in objects_with_id - result = self.objects_with_id.get(obj) - if not result: - result = self._next_id() - self.objects_with_id.setitem(obj, result) - return result - - def _compute_id_for_external(self, obj): - # For prebuilt objects, we can simply return their address. - # This method is overriden by the HybridGC. - return obj - - def update_objects_with_id(self): - old = self.objects_with_id - new_objects_with_id = self.AddressDict(old.length()) - old.foreach(self._update_object_id_FAST, new_objects_with_id) - old.delete() - self.objects_with_id = new_objects_with_id - - def _update_object_id(self, obj, id, new_objects_with_id): - # safe version (used by subclasses) - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - new_objects_with_id.setitem(newobj, id) - else: - self.id_free_list.append(id) - - def _update_object_id_FAST(self, obj, id, new_objects_with_id): - # unsafe version, assumes that the new_objects_with_id is large enough - if self.surviving(obj): - newobj = self.get_forwarding_address(obj) - new_objects_with_id.insertclean(newobj, id) - else: - self.id_free_list.append(id) - - def debug_check_object(self, obj): - """Check the invariants about 'obj' that should be true - between collections.""" - tid = self.header(obj).tid - if tid & GCFLAG_EXTERNAL: - ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded") - ll_assert(not (self.tospace <= obj < self.free), - "external flag but object inside the semispaces") - else: - ll_assert(not (tid & GCFLAG_FORWARDED), "bug: !external+forwarded") - ll_assert(self.tospace <= obj < self.free, - "!external flag but object outside the semispaces") - ll_assert(not (tid & GCFLAG_FINALIZATION_ORDERING), - "unexpected GCFLAG_FINALIZATION_ORDERING") - - def debug_check_can_copy(self, obj): - ll_assert(not (self.tospace <= obj < self.free), - "copy() on already-copied object") - def choose_gc_from_config(config): """Return a (GCClass, GC_PARAMS) from the given config object. """ From fijal at codespeak.net Tue Sep 30 14:02:44 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 30 Sep 2008 14:02:44 +0200 (CEST) Subject: [pypy-svn] r58502 - pypy/branch/gc-experiments/pypy/rpython/memory/gc Message-ID: <20080930120244.799AE16A01D@codespeak.net> Author: fijal Date: Tue Sep 30 14:02:41 2008 New Revision: 58502 Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Log: Comment Modified: pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/memory/gc/markcompact.py Tue Sep 30 14:02:41 2008 @@ -39,6 +39,10 @@ HDR = lltype.Struct('header', ('tid', lltype.Signed), ('forward_ptr', llmemory.Address)) + # XXX probably we need infinite (ie 4G) amount of memory here + # and we'll keep all pages shared. The question is what we do + # with tests which create all llarenas + TRANSLATION_PARAMS = {'space_size': 16*1024*1024} # XXX adjust def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=16*(1024**2)): From fijal at codespeak.net Tue Sep 30 14:11:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 30 Sep 2008 14:11:47 +0200 (CEST) Subject: [pypy-svn] r58503 - in pypy/branch/gc-experiments/pypy: rpython/lltypesystem translator/c/src translator/c/test Message-ID: <20080930121147.35160169FF1@codespeak.net> Author: fijal Date: Tue Sep 30 14:11:46 2008 New Revision: 58503 Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py pypy/branch/gc-experiments/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-experiments/pypy/translator/c/src/mem.h pypy/branch/gc-experiments/pypy/translator/c/test/test_lladdresses.py Log: Support for raw_memmove Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/lltypesystem/llmemory.py Tue Sep 30 14:11:46 2008 @@ -790,3 +790,19 @@ setattr(dest._obj, name, llvalue) else: raise TypeError(T) + +from pypy.rpython.extregistry import ExtRegistryEntry + +class RawMemmoveEntry(ExtRegistryEntry): + _about_ = raw_memmove + + def compute_result_annotation(self, s_from, s_to, s_size): + from pypy.annotation.model import SomeAddress, SomeInteger + assert isinstance(s_from, SomeAddress) + assert isinstance(s_to, SomeAddress) + assert isinstance(s_size, SomeInteger) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_list = hop.inputargs(Address, Address, lltype.Signed) + return hop.genop('raw_memmove', v_list) Modified: pypy/branch/gc-experiments/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-experiments/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-experiments/pypy/rpython/lltypesystem/lloperation.py Tue Sep 30 14:11:46 2008 @@ -368,6 +368,7 @@ 'raw_free': LLOp(), 'raw_memclear': LLOp(), 'raw_memcopy': LLOp(), + 'raw_memmove': LLOp(), 'raw_load': LLOp(sideeffects=False), 'raw_store': LLOp(), 'stack_malloc': LLOp(), # mmh Modified: pypy/branch/gc-experiments/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/gc-experiments/pypy/translator/c/src/mem.h (original) +++ pypy/branch/gc-experiments/pypy/translator/c/src/mem.h Tue Sep 30 14:11:46 2008 @@ -90,6 +90,7 @@ if (r != NULL) memset((void*) r, 0, size); #define OP_RAW_MEMCOPY(x,y,size,r) memcpy(y,x,size); +#define OP_RAW_MEMMOVE(x,y,size,r) memmove(y,x,size); /************************************************************/ Modified: pypy/branch/gc-experiments/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/gc-experiments/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/gc-experiments/pypy/translator/c/test/test_lladdresses.py Tue Sep 30 14:11:46 2008 @@ -96,6 +96,19 @@ res = fc() assert res +def test_raw_memmove(): + def f(): + addr = raw_malloc(100) + addr.signed[0] = 12 + (addr + 10).signed[0] = 42 + raw_memmove(addr, addr + 4, 96) + result = (addr + 4).signed[0] + (addr + 14).signed[0] + raw_free(addr) + return result + fc = compile(f, []) + res = fc() + assert res + def test_pointer_comparison(): def f(): result = 0 From fijal at codespeak.net Tue Sep 30 14:13:16 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 30 Sep 2008 14:13:16 +0200 (CEST) Subject: [pypy-svn] r58504 - pypy/dist/pypy/config Message-ID: <20080930121316.D3924169F02@codespeak.net> Author: fijal Date: Tue Sep 30 14:13:16 2008 New Revision: 58504 Modified: pypy/dist/pypy/config/translationoption.py Log: using mark'n'sweep here is fairly pointless Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Tue Sep 30 14:13:16 2008 @@ -299,7 +299,7 @@ '0': 'boehm nobackendopt', '1': 'boehm lowinline', 'size': 'boehm lowinline remove_asserts', - 'mem': 'marksweep lowinline remove_asserts', + 'mem': 'generation lowinline remove_asserts', '2': 'hybrid extraopts', '3': 'hybrid extraopts remove_asserts', } From arigo at codespeak.net Tue Sep 30 15:08:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 30 Sep 2008 15:08:03 +0200 (CEST) Subject: [pypy-svn] r58506 - in pypy/build/benchmem: . benchmark Message-ID: <20080930130803.01CF616A04B@codespeak.net> Author: arigo Date: Tue Sep 30 15:08:02 2008 New Revision: 58506 Modified: pypy/build/benchmem/benchmark/sizes.py pypy/build/benchmem/runbench.py Log: Systematize the list of benchmarks. Remove a couple of (in my opinion) meaningless ones. Modified: pypy/build/benchmem/benchmark/sizes.py ============================================================================== --- pypy/build/benchmem/benchmark/sizes.py (original) +++ pypy/build/benchmem/benchmark/sizes.py Tue Sep 30 15:08:02 2008 @@ -1,4 +1,6 @@ +glob = 0 + def bench_0nothing(iter1): pass @@ -10,26 +12,98 @@ l = [int(i+1000) for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_emptytuple(iter1): - l = [() for i in range(iter1)] +def bench_list_of_float(iter1): + l = [float(i+1000) for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + +def bench_list_of_str_len2(iter1): + l = [('%02d' % i)[:2] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len3(iter1): + l = [('%03d' % i)[:3] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len4(iter1): + l = [('%04d' % i)[:4] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len5(iter1): + l = [('%05d' % i)[:5] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len6(iter1): + l = [('%06d' % i)[:6] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len7(iter1): + l = [('%07d' % i)[:7] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_str_len8(iter1): + l = [('%08d' % i)[:8] for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + +def bench_list_of_tuple_len1(iter1): + l = [(glob,) for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_tuple_len2(iter1): + l = [(glob, glob) for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_tuple_len3(iter1): + l = [(glob, glob, glob) for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_tupleofNone(iter1): - l = [(None,) for i in range(iter1)] +def bench_list_of_tuple_len4(iter1): + l = [(glob, glob, glob, glob) for i in range(iter1)] checkpoint(collect=True) +# ____________________________________________________________ + def bench_list_of_emptylist(iter1): l = [[] for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_list_with_int(iter1): - l = [[i] for i in range(iter1)] +def bench_list_of_list_len1(iter1): + l = [[glob] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_len2(iter1): + l = [[glob, glob] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_len3(iter1): + l = [[glob, glob, glob] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_len4(iter1): + l = [[glob, glob, glob, glob] for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_list_with_float(iter1): - l = [[float(i)] for i in range(iter1)] +def bench_list_of_list_len5(iter1): + l = [[glob, glob, glob, glob, glob] for i in range(iter1)] checkpoint(collect=True) +def bench_list_of_list_len6(iter1): + l = [[glob, glob, glob, glob, glob, glob] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_len7(iter1): + l = [[glob, glob, glob, glob, glob, glob, glob] for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_list_len8(iter1): + l = [[glob, glob, glob, glob, glob, glob, glob, glob] for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + def bench_list_of_emptydict(iter1): l = [{} for i in range(iter1)] checkpoint(collect=True) @@ -38,18 +112,224 @@ l = [{i:None} for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_empty_instances_old(iter1): +def bench_list_of_dict_strkey_len1(iter1): + l = [{'%07d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len2(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len3(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len4(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None, 'd%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len5(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None, 'd%06d'%i:None, + 'e%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len6(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None, 'd%06d'%i:None, + 'e%06d'%i:None, 'f%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len7(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None, 'd%06d'%i:None, + 'e%06d'%i:None, 'f%06d'%i:None, + 'g%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_dict_strkey_len8(iter1): + l = [{'a%06d'%i:None, 'b%06d'%i:None, + 'c%06d'%i:None, 'd%06d'%i:None, + 'e%06d'%i:None, 'f%06d'%i:None, + 'g%06d'%i:None, 'h%06d'%i:None} for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + +def bench_list_of_instances_old_attr1(iter1): + class A: + def __init__(self): + self.a = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr2(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr3(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr4(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr5(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr6(iter1): class A: - pass + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 x = [A() for i in range(iter1)] checkpoint(collect=True) -def bench_list_of_empty_instances_new(iter1): +def bench_list_of_instances_old_attr7(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 + self.g = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_old_attr8(iter1): + class A: + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 + self.g = 1 + self.h = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + +def bench_list_of_instances_new_attr1(iter1): class A(object): - pass + def __init__(self): + self.a = 1 x = [A() for i in range(iter1)] checkpoint(collect=True) +def bench_list_of_instances_new_attr2(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr3(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr4(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr5(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr6(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr7(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 + self.g = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +def bench_list_of_instances_new_attr8(iter1): + class A(object): + def __init__(self): + self.a = 1 + self.b = 1 + self.c = 1 + self.d = 1 + self.e = 1 + self.f = 1 + self.g = 1 + self.h = 1 + x = [A() for i in range(iter1)] + checkpoint(collect=True) + +# ____________________________________________________________ + def bench_linked_instances(iter1): class A(object): def __init__(self, prev): @@ -74,3 +354,19 @@ for i in range(iter1): x = (x,) checkpoint(collect=True) + +# ____________________________________________________________ + +from email.Message import Message + +def bench_list_of_messages(iter1): + x = [] + for i in range(iter1): + # make block-looking 40-lines, 72-columns e-mails + msg = Message() + msg.set_payload(('%072d' % i) * 40) + msg['Subject'] = '%60d' % i + msg['From'] = '%15d' % i + msg['To'] = '%20d' % i + x.append(msg) + checkpoint(collect=True) Modified: pypy/build/benchmem/runbench.py ============================================================================== --- pypy/build/benchmem/runbench.py (original) +++ pypy/build/benchmem/runbench.py Tue Sep 30 15:08:02 2008 @@ -69,6 +69,7 @@ for name, obj in vars(self.benchpath.pyimport()).items(): if name.startswith("bench") and callable(obj): l.append(name) + l.sort() return l def run(self):